Gun: first cut

This commit is contained in:
Mitra Ardron 2018-07-06 20:41:45 -07:00
parent 144e173274
commit bab61a9e7a
9 changed files with 266 additions and 10 deletions

26
API.md
View File

@ -30,6 +30,7 @@ There are a set of classes:
* *TransportIPFS*: Connects to IPFS, currently (April 2018) via WebSocketsStar (WSS) * *TransportIPFS*: Connects to IPFS, currently (April 2018) via WebSocketsStar (WSS)
* *TransportYJS*: Implements shared lists, and dictionaries. Uses IPFS for transport * *TransportYJS*: Implements shared lists, and dictionaries. Uses IPFS for transport
* *TransportWEBTORRENT*: Integrates to Feross's WebTorrent library * *TransportWEBTORRENT*: Integrates to Feross's WebTorrent library
* *TransportGUN*: Integrates to the Gun DB
* *Transports*: manages the list of conencted transports, and directs api calls to them. * *Transports*: manages the list of conencted transports, and directs api calls to them.
Calls are generally made through the Transports class which knows how to route them to underlying connections. Calls are generally made through the Transports class which knows how to route them to underlying connections.
@ -323,6 +324,11 @@ returns instance of TransportIPFS if connected
returns instance of TransportWEBTORRENT if connected returns instance of TransportWEBTORRENT if connected
``` ```
##### static gun(verbose)
```
returns instance of TransportGUN if connected
```
##### static async p_resolveNames(urls) ##### static async p_resolveNames(urls)
See Naming below See Naming below
``` ```
@ -345,7 +351,7 @@ t: Add a Transport instance to _transports
##### static setup0(transports, options, verbose, cb) ##### static setup0(transports, options, verbose, cb)
Calls setup0 for each transport based on its short name. Specially handles LOCAL as a transport pointing at a local http server (for testing). Calls setup0 for each transport based on its short name. Specially handles LOCAL as a transport pointing at a local http server (for testing).
``` ```
transports Array of short names of transports e.g. [IPFS,HTTP,ORBITDB] transports Array of short names of transports e.g. [IPFS,HTTP,GUN]
options Passed to setup0 on each transport options Passed to setup0 on each transport
cb Callback to be called each time status changes cb Callback to be called each time status changes
Returns: Array of transport instances Returns: Array of transport instances
@ -518,6 +524,14 @@ supportFunctions:
supportFeatures: supportFeatures:
fetch.range Not supported (currently April 2018) fetch.range Not supported (currently April 2018)
## TransportGUN
A subclass of Transport for handling GUN connections (decentralized database)
supportURLS = `gun:*` (TODO: may in the future support `dweb:/gun/*`)
supportFunctions
`add, list, listmonitor, newlisturls, connection, get, set, getall, keys, newdatabase, newtable, monitor`
supportFeatures:
## Naming ## Naming
Independently from the transport, the Transport library can resolve names if provided an appropriate callback. Independently from the transport, the Transport library can resolve names if provided an appropriate callback.
See p_resolveNames(urls) and resolveNamesWith(cb) See p_resolveNames(urls) and resolveNamesWith(cb)
@ -533,3 +547,13 @@ The format of names currently (April 2018) is under development but its likely t
`dweb:/arc/archive.org/details/foo` `dweb:/arc/archive.org/details/foo`
to allow smooth integration with existing HTTP urls that are moving to decentralization. to allow smooth integration with existing HTTP urls that are moving to decentralization.
## Adding a Transport
The following steps are needed to add a transport.
* Add a line to package.json/dependencies for any packages needed
* Add lines to index.js;
* Add function to return the instance to Transports.js
* Add to list in Transports.p_connect()
* Add to API.md
* Look for any "SEE-OTHER-ADDTRANSPORT" in case not on this list
* Edit a copy of the closest Transport to what you are building

View File

@ -15,7 +15,7 @@ class Transport {
Fields: Fields:
statuselement: If set is an HTML Element that should be adjusted to indicate status (this is managed by Transports, just stored on Transport) statuselement: If set is an HTML Element that should be adjusted to indicate status (this is managed by Transports, just stored on Transport)
statuscb: Callback when status changes statuscb: Callback when status changes
name: Short name of element e.g. HTTP IPFS WEBTORRENT name: Short name of element e.g. HTTP IPFS WEBTORRENT GUN
*/ */
} }

217
TransportGUN Normal file
View File

@ -0,0 +1,217 @@
/*
This Transport layers uses GUN.
*/
const Url = require('url');
const Gun = require('gun')
// Utility packages (ours) And one-liners
function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})}
// Other Dweb modules
const errors = require('./Errors'); // Standard Dweb Errors
const Transport = require('./Transport.js'); // Base class for TransportXyz
const Transports = require('./Transports'); // Manage all Transports that are loaded
const utils = require('./utils'); // Utility functions
let defaultoptions = {
}
class TransportGUN extends Transport {
/*
GUN specific transport - over IPFS
Fields:
gun: object returned when starting GUN
*/
constructor(options, verbose) {
super(options, verbose);
this.options = options; // Dictionary of options { ipfs: {...}, "yarrays", yarray: {...} }
this.gun = undefined;
this.name = "GUN"; // For console log etc
this.supportURLs = ['gun'];
#TODO-GUN doesnt really support lists yet, its "set" function only handles other gun objects.
this.supportFunctions = ['connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor']; // TODO-GUN check this: ['fetch', 'add', 'list', 'listmonitor', 'newlisturls',]
this.status = Transport.STATUS_LOADED;
}
async p_connection(url, verbose) {
/*
Utility function to get Gun object for this URL
url: URL string to find list of
resolves: Gun a connection to use for get's etc.
*/
if (typeof url === "string")
url = Url.parse(url);
patharray = url.pathstring.split('/') //[ 'gun', database, table ]
patharray.shift; // Loose "gun"
g = this.gun;
while (patharray.length) {
g = g.get(patharray.shift()); //TODO-GUN-TEST will this work if database never initialized
}
return g;
}
static setup0(options, verbose) {
/*
First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.
options: { gun: { }, } Set of options - "gun" is used for those to pass direct to Gun
*/
let combinedoptions = Transport.mergeoptions(defaultoptions, options);
console.log("GUN options %o", combinedoptions); // Log even if !verbose
let t = new TransportGUN(combinedoptions, verbose); // Note doesnt start IPFS or OrbitDB
Dweb.Transports.addtransport(t);
return t;
}
async p_setup1(verbose, cb) {
/*
This sets up for GUN.
Throws: TODO-GUN document errors that can occur
*/
try {
this.status = Dweb.Transport.STATUS_STARTING; // Should display, but probably not refreshed in most case
if (cb) cb(this);
this.gun = new Gun(this.options.gun); // TODO-GUN how do I know if this succeeded
await this.p_status(verbose);
} catch(err) {
console.error(this.name,"failed to start",err);
this.status = Transport.STATUS_FAILED;
}
if (cb) cb(this);
return this;
}
async p_status(verbose) {
/*
Return a string for the status of a transport. No particular format, but keep it short as it will probably be in a small area of the screen.
*/
this.status = Dweb.Transport.STATUS_CONNECTED; //TODO-GUN how do I know if/when I'm connected (see comment on p_setup1 as well)
return this.status;
}
async p_newdatabase(pubkey, {verbose=false}={}) {
/*
Request a new database
For GUN it doesnt actually create anything, just generates the URLs
TODO-GUN how to make globally accessible if normalized to my UUID
TODO-GUN how to get a private Url that enables me to write to it?
returns: {publicurl: "gun:/gun/<publickey>", privateurl: "gun:/gun/<publickey>">
*/
if (pubkey.hasOwnProperty("keypair"))
pubkey = pubkey.keypair.signingexport()
// By this point pubkey should be an export of a public key of form xyz:abc where xyz
// specifies the type of public key (NACL VERIFY being the only kind we expect currently)
let u = `gun:/gun/${encodeURIComponent(pubkey)}`;
return {"publicurl": u, "privateurl": u};
}
async p_newtable(pubkey, table, {verbose=false}={}) {
/*
Request a new table
For GUN it doesnt actually create anything, just generates the URLs
returns: {publicurl: "gun:/gun/<publickey>/<table>", privateurl: "gun:/gun/<publickey>/<table>">
*/
if (!pubkey) throw new errors.CodingError("p_newtable currently requires a pubkey");
let database = await this.p_newdatabase(pubkey, {verbose});
// If have use cases without a database, then call p_newdatabase first
return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it
}
async p_set(url, keyvalues, value, {verbose=false}={}) { // url = yjs:/yjs/database/table
/*
Set key values
keyvalues: string (key) in which case value should be set there OR
object in which case value is ignored
*/
let table = await this.p_connection(url, verbose);
if (typeof keyvalues === "string") {
table.path(keyvalues).put(JSON.stringify(value));
} else {
table.put(keyvalues); // Store all key-value pairs without destroying any other key/value pairs previously set
}
}
async p_get(url, keys, {verbose=false}={}) {
let table = await this.p_connection(url, verbose);
if (Array.isArray(keys)) {
return keys.reduce(function(previous, key) {
let val = table.get(key);
previous[key] = typeof val === "string" ? JSON.parse(val) : val; // Handle undefined
return previous;
}, {});
} else {
let val = table.get(keys);
return typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync
}
}
async p_delete(url, keys, {verbose=false}={}) {
let table = await this.p_connection(url, verbose);
if (typeof keys === "string") {
table.path(keys).put(null);
} else {
keys.map((key) => table.path(key).put(null)); // This looks like it is sync
}
}
_p_once(table) {
return new Promise((resolve, reject) => table.once(resolve));
}
async p_keys(url, {verbose=false}={}) {
kvs = await this.p_getall(url, {verbose});
return Object.keys(kvs);
}
async p_getall(url, {verbose=false}={}) {
let table = await this.p_connection(url, verbose);
return this._p_once(table));
}
async monitor(url, callback, verbose) {
/*
Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall.
Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => GUN.monitor)(b: dispatchEvent)
:param url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
:param callback: function({type, key, value}) Callback for each new item added to the list
:param verbose: boolean - true for debugging output
*/
url = typeof url === "string" ? url : url.href;
let conn = this.p_connection(url, verbose);
# See https://github.com/amark/gun/wiki/API#map for why this
# What we really want is to have the callback called once for each changed BUT
# conn.map().on(cb) will also get called for each initial value
# conn.on(cb) and then throwing away initial call would be ok, except it streams so cb might be called with first half of data and then rest
# TODO-GUN - waiting on an option for the above to have compliant monitor, for now just make sure to ignore dupes and note that GUN doesnt support list/add/listmonitor anyway
conn.map().on((v, k) => callback("set", k, JSON.parse(v)));
}
static async test(transport, verbose) { //TODO-GUN rewrite this
if (verbose) {console.log("TransportGUN.test")}
try {
let testurl = "1114"; // Just a predictable number can work with
let res = await transport.p_rawlist(testurl, verbose);
let listlen = res.length; // Holds length of list run intermediate
if (verbose) console.log("rawlist returned ", ...Dweb.utils.consolearr(res));
transport.listmonitor(testurl, (obj) => console.log("Monitored", obj), verbose);
let sig = new Dweb.Signature({urls: ["123"], date: new Date(Date.now()), signature: "Joe Smith", signedby: [testurl]}, verbose);
await transport.p_rawadd(testurl, sig, verbose);
if (verbose) console.log("TransportIPFS.p_rawadd returned ");
res = await transport.p_rawlist(testurl, verbose);
if (verbose) console.log("rawlist returned ", ...Dweb.utils.consolearr(res)); // Note not showing return
await delay(500);
res = await transport.p_rawlist(testurl, verbose);
console.assert(res.length === listlen + 1, "Should have added one item");
} catch(err) {
console.log("Exception thrown in TransportORBITDB.test:", err.message);
throw err;
}
}
}
Transports._transportclasses["GUN"] = TransportGUN;
TransportGUN.GUN = GUN; // Allow node tests to find it
exports = module.exports = TransportGUN;

View File

@ -119,7 +119,7 @@ class TransportIPFS extends Transport {
await this.p_ipfsstart(verbose); // Throws Error("websocket error") and possibly others. await this.p_ipfsstart(verbose); // Throws Error("websocket error") and possibly others.
this.status = await this.p_status(verbose); this.status = await this.p_status(verbose);
} catch(err) { } catch(err) {
console.error("IPFS failed to connect",err); console.error(this.name, "failed to connect", err);
this.status = Transport.STATUS_FAILED; this.status = Transport.STATUS_FAILED;
} }
if (cb) cb(this); if (cb) cb(this);

View File

@ -73,7 +73,7 @@ class TransportWEBTORRENT extends Transport {
await this.p_webtorrentstart(verbose); await this.p_webtorrentstart(verbose);
await this.p_status(verbose); await this.p_status(verbose);
} catch(err) { } catch(err) {
console.error("WebTorrent failed to connect",err); console.error(this.name, "failed to connect", err);
this.status = Transport.STATUS_FAILED; this.status = Transport.STATUS_FAILED;
} }
if (cb) cb(this); if (cb) cb(this);

View File

@ -123,7 +123,7 @@ class TransportYJS extends Transport {
this.yarrays = {}; this.yarrays = {};
await this.p_status(verbose); await this.p_status(verbose);
} catch(err) { } catch(err) {
console.error("YJS failed to start",err); console.error(this.name,"failed to start",err);
this.status = Transport.STATUS_FAILED; this.status = Transport.STATUS_FAILED;
} }
if (cb) cb(this); if (cb) cb(this);
@ -231,8 +231,6 @@ class TransportYJS extends Transport {
} }
// Support for Key-Value pairs as per
// https://docs.google.com/document/d/1yfmLRqKPxKwB939wIy9sSaa7GKOzM5PrCZ4W1jRGW6M/edit#
async p_newdatabase(pubkey, {verbose=false}={}) { async p_newdatabase(pubkey, {verbose=false}={}) {
//if (pubkey instanceof Dweb.PublicPrivate) //if (pubkey instanceof Dweb.PublicPrivate)
if (pubkey.hasOwnProperty("keypair")) if (pubkey.hasOwnProperty("keypair"))
@ -252,7 +250,12 @@ class TransportYJS extends Transport {
return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it
} }
async p_set(url, keyvalues, value, {verbose=false}={}) { // url = yjs:/yjs/database/table/key async p_set(url, keyvalues, value, {verbose=false}={}) { // url = yjs:/yjs/database/table
/*
Set key values
keyvalues: string (key) in which case value should be set there OR
object in which case value is ignored
*/
let y = await this.p_connection(url, verbose); let y = await this.p_connection(url, verbose);
if (typeof keyvalues === "string") { if (typeof keyvalues === "string") {
y.share.map.set(keyvalues, JSON.stringify(value)); y.share.map.set(keyvalues, JSON.stringify(value));
@ -302,7 +305,7 @@ class TransportYJS extends Transport {
} }
async monitor(url, callback, verbose) { async monitor(url, callback, verbose) {
/* /*
Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist. Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall.
Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => YJS.monitor)(b: dispatchEvent) Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => YJS.monitor)(b: dispatchEvent)
:param url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd :param url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd

View File

@ -71,6 +71,9 @@ class Transports {
// Need a async version of this for serviceworker and TransportsProxy // Need a async version of this for serviceworker and TransportsProxy
return this.validFor(urls, func, options).map((ut) => ut[0]); return this.validFor(urls, func, options).map((ut) => ut[0]);
} }
// SEE-OTHER-ADDTRANSPORT
static http(verbose) { static http(verbose) {
// Find an http transport if it exists, so for example YJS can use it. // Find an http transport if it exists, so for example YJS can use it.
return Transports._connected().find((t) => t.name === "HTTP") return Transports._connected().find((t) => t.name === "HTTP")
@ -85,6 +88,12 @@ class Transports {
return Transports._connected().find((t) => t.name === "WEBTORRENT") return Transports._connected().find((t) => t.name === "WEBTORRENT")
} }
static gun(verbose) {
// Find a GUN transport if it exists
return Transports._connected().find((t) => t.name === "GUN")
}
static async p_resolveNames(urls) { static async p_resolveNames(urls) {
/* If and only if TransportNAME was loaded (it might not be as it depends on higher level classes like Domain and SmartDict) /* If and only if TransportNAME was loaded (it might not be as it depends on higher level classes like Domain and SmartDict)
then resolve urls that might be names, returning a modified array. then resolve urls that might be names, returning a modified array.
@ -535,7 +544,7 @@ class Transports {
let tabbrevs = options.transports; // Array of transport abbreviations let tabbrevs = options.transports; // Array of transport abbreviations
this._optionspaused = (options.paused || []).map(n => n.toUpperCase()); // Array of transports paused - defaults to none, upper cased this._optionspaused = (options.paused || []).map(n => n.toUpperCase()); // Array of transports paused - defaults to none, upper cased
if (!(tabbrevs && tabbrevs.length)) { tabbrevs = options.defaulttransports || [] } if (!(tabbrevs && tabbrevs.length)) { tabbrevs = options.defaulttransports || [] }
if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT"]; } if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT", "GUN"]; } // SEE-OTHER-ADDTRANSPORT
tabbrevs = tabbrevs.map(n => n.toUpperCase()); tabbrevs = tabbrevs.map(n => n.toUpperCase());
let transports = this.setup0(tabbrevs, options, verbose); let transports = this.setup0(tabbrevs, options, verbose);
if (options.statuscb) { if (options.statuscb) {

View File

@ -1,9 +1,11 @@
// Order is significant as should search earlier ones first // Order is significant as should search earlier ones first
// put IPFS before Webtorrent for showcasing, as Webtorrent works in some cases IPFS doesnt so that way we exercise both // put IPFS before Webtorrent for showcasing, as Webtorrent works in some cases IPFS doesnt so that way we exercise both
const DwebTransports = require("./Transports.js"); const DwebTransports = require("./Transports.js");
// SEE-OTHER-ADDTRANSPORT
require("./TransportHTTP.js"); // Can access via window.DwebTransports._transportclasses["HTTP"] require("./TransportHTTP.js"); // Can access via window.DwebTransports._transportclasses["HTTP"]
require("./TransportIPFS.js"); require("./TransportIPFS.js");
require("./TransportYJS.js"); require("./TransportYJS.js");
require("./TransportWEBTORRENT.js"); require("./TransportWEBTORRENT.js");
require("./TransportGUN.js");
if (typeof window !== "undefined") { window.DwebTransports = DwebTransports; } if (typeof window !== "undefined") { window.DwebTransports = DwebTransports; }
exports = module.exports = DwebTransports; exports = module.exports = DwebTransports;

View File

@ -15,6 +15,7 @@
"ipfs": "latest", "ipfs": "latest",
"ipfs-unixfs": "^0.1.15", "ipfs-unixfs": "^0.1.15",
"node-fetch": "latest", "node-fetch": "latest",
"gun": "latest",
"readable-stream": "latest", "readable-stream": "latest",
"webtorrent": "^0.99.3", "webtorrent": "^0.99.3",
"y-array": "latest", "y-array": "latest",