diff --git a/API.md b/API.md index 734e14f..b033f43 100644 --- a/API.md +++ b/API.md @@ -30,6 +30,7 @@ There are a set of classes: * *TransportIPFS*: Connects to IPFS, currently (April 2018) via WebSocketsStar (WSS) * *TransportYJS*: Implements shared lists, and dictionaries. Uses IPFS for transport * *TransportWEBTORRENT*: Integrates to Feross's WebTorrent library +* *TransportGUN*: Integrates to the Gun DB * *Transports*: manages the list of conencted transports, and directs api calls to them. Calls are generally made through the Transports class which knows how to route them to underlying connections. @@ -323,6 +324,11 @@ returns instance of TransportIPFS if connected returns instance of TransportWEBTORRENT if connected ``` +##### static gun(verbose) +``` +returns instance of TransportGUN if connected +``` + ##### static async p_resolveNames(urls) See Naming below ``` @@ -345,7 +351,7 @@ t: Add a Transport instance to _transports ##### static setup0(transports, options, verbose, cb) Calls setup0 for each transport based on its short name. Specially handles ‘LOCAL’ as a transport pointing at a local http server (for testing). ``` -transports Array of short names of transports e.g. [‘IPFS’,’HTTP’,’ORBITDB’] +transports Array of short names of transports e.g. [‘IPFS’,’HTTP’,’GUN’] options Passed to setup0 on each transport cb Callback to be called each time status changes Returns: Array of transport instances @@ -518,6 +524,14 @@ supportFunctions: supportFeatures: fetch.range Not supported (currently April 2018) +## TransportGUN +A subclass of Transport for handling GUN connections (decentralized database) + +supportURLS = `gun:*` (TODO: may in the future support `dweb:/gun/*`) +supportFunctions + `add, list, listmonitor, newlisturls, connection, get, set, getall, keys, newdatabase, newtable, monitor` +supportFeatures: + ## Naming Independently from the transport, the Transport library can resolve names if provided an appropriate callback. See p_resolveNames(urls) and resolveNamesWith(cb) @@ -533,3 +547,13 @@ The format of names currently (April 2018) is under development but its likely t `dweb:/arc/archive.org/details/foo` to allow smooth integration with existing HTTP urls that are moving to decentralization. +## Adding a Transport +The following steps are needed to add a transport. + +* Add a line to package.json/dependencies for any packages needed +* Add lines to index.js; +* Add function to return the instance to Transports.js +* Add to list in Transports.p_connect() +* Add to API.md +* Look for any "SEE-OTHER-ADDTRANSPORT" in case not on this list +* Edit a copy of the closest Transport to what you are building \ No newline at end of file diff --git a/Transport.js b/Transport.js index d004299..18c797b 100644 --- a/Transport.js +++ b/Transport.js @@ -15,7 +15,7 @@ class Transport { Fields: statuselement: If set is an HTML Element that should be adjusted to indicate status (this is managed by Transports, just stored on Transport) statuscb: Callback when status changes - name: Short name of element e.g. HTTP IPFS WEBTORRENT + name: Short name of element e.g. HTTP IPFS WEBTORRENT GUN */ } diff --git a/TransportGUN b/TransportGUN new file mode 100644 index 0000000..3a0e129 --- /dev/null +++ b/TransportGUN @@ -0,0 +1,217 @@ +/* +This Transport layers uses GUN. +*/ +const Url = require('url'); +const Gun = require('gun') + +// Utility packages (ours) And one-liners +function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})} + +// Other Dweb modules +const errors = require('./Errors'); // Standard Dweb Errors +const Transport = require('./Transport.js'); // Base class for TransportXyz +const Transports = require('./Transports'); // Manage all Transports that are loaded +const utils = require('./utils'); // Utility functions + +let defaultoptions = { +} + +class TransportGUN extends Transport { + /* + GUN specific transport - over IPFS + + Fields: + gun: object returned when starting GUN + */ + + constructor(options, verbose) { + super(options, verbose); + this.options = options; // Dictionary of options { ipfs: {...}, "yarrays", yarray: {...} } + this.gun = undefined; + this.name = "GUN"; // For console log etc + this.supportURLs = ['gun']; + #TODO-GUN doesnt really support lists yet, its "set" function only handles other gun objects. + this.supportFunctions = ['connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor']; // TODO-GUN check this: ['fetch', 'add', 'list', 'listmonitor', 'newlisturls',] + this.status = Transport.STATUS_LOADED; + } + + async p_connection(url, verbose) { + /* + Utility function to get Gun object for this URL + url: URL string to find list of + resolves: Gun a connection to use for get's etc. + */ + if (typeof url === "string") + url = Url.parse(url); + patharray = url.pathstring.split('/') //[ 'gun', database, table ] + patharray.shift; // Loose "gun" + g = this.gun; + while (patharray.length) { + g = g.get(patharray.shift()); //TODO-GUN-TEST will this work if database never initialized + } + return g; + } + + static setup0(options, verbose) { + /* + First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections. + options: { gun: { }, } Set of options - "gun" is used for those to pass direct to Gun + */ + let combinedoptions = Transport.mergeoptions(defaultoptions, options); + console.log("GUN options %o", combinedoptions); // Log even if !verbose + let t = new TransportGUN(combinedoptions, verbose); // Note doesnt start IPFS or OrbitDB + Dweb.Transports.addtransport(t); + return t; + } + + async p_setup1(verbose, cb) { + /* + This sets up for GUN. + Throws: TODO-GUN document errors that can occur + */ + try { + this.status = Dweb.Transport.STATUS_STARTING; // Should display, but probably not refreshed in most case + if (cb) cb(this); + this.gun = new Gun(this.options.gun); // TODO-GUN how do I know if this succeeded + await this.p_status(verbose); + } catch(err) { + console.error(this.name,"failed to start",err); + this.status = Transport.STATUS_FAILED; + } + if (cb) cb(this); + return this; + } + + async p_status(verbose) { + /* + Return a string for the status of a transport. No particular format, but keep it short as it will probably be in a small area of the screen. + */ + this.status = Dweb.Transport.STATUS_CONNECTED; //TODO-GUN how do I know if/when I'm connected (see comment on p_setup1 as well) + return this.status; + } + + async p_newdatabase(pubkey, {verbose=false}={}) { + /* + Request a new database + For GUN it doesnt actually create anything, just generates the URLs + TODO-GUN how to make globally accessible if normalized to my UUID + TODO-GUN how to get a private Url that enables me to write to it? + + returns: {publicurl: "gun:/gun/", privateurl: "gun:/gun/"> + */ + if (pubkey.hasOwnProperty("keypair")) + pubkey = pubkey.keypair.signingexport() + // By this point pubkey should be an export of a public key of form xyz:abc where xyz + // specifies the type of public key (NACL VERIFY being the only kind we expect currently) + let u = `gun:/gun/${encodeURIComponent(pubkey)}`; + return {"publicurl": u, "privateurl": u}; + } + + async p_newtable(pubkey, table, {verbose=false}={}) { + /* + Request a new table + For GUN it doesnt actually create anything, just generates the URLs + + returns: {publicurl: "gun:/gun//", privateurl: "gun:/gun//
"> + */ + if (!pubkey) throw new errors.CodingError("p_newtable currently requires a pubkey"); + let database = await this.p_newdatabase(pubkey, {verbose}); + // If have use cases without a database, then call p_newdatabase first + return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it + } + + async p_set(url, keyvalues, value, {verbose=false}={}) { // url = yjs:/yjs/database/table + /* + Set key values + keyvalues: string (key) in which case value should be set there OR + object in which case value is ignored + */ + let table = await this.p_connection(url, verbose); + if (typeof keyvalues === "string") { + table.path(keyvalues).put(JSON.stringify(value)); + } else { + table.put(keyvalues); // Store all key-value pairs without destroying any other key/value pairs previously set + } + } + async p_get(url, keys, {verbose=false}={}) { + let table = await this.p_connection(url, verbose); + if (Array.isArray(keys)) { + return keys.reduce(function(previous, key) { + let val = table.get(key); + previous[key] = typeof val === "string" ? JSON.parse(val) : val; // Handle undefined + return previous; + }, {}); + } else { + let val = table.get(keys); + return typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync + } + } + + async p_delete(url, keys, {verbose=false}={}) { + let table = await this.p_connection(url, verbose); + if (typeof keys === "string") { + table.path(keys).put(null); + } else { + keys.map((key) => table.path(key).put(null)); // This looks like it is sync + } + } + + _p_once(table) { + return new Promise((resolve, reject) => table.once(resolve)); + } + async p_keys(url, {verbose=false}={}) { + kvs = await this.p_getall(url, {verbose}); + return Object.keys(kvs); + } + async p_getall(url, {verbose=false}={}) { + let table = await this.p_connection(url, verbose); + return this._p_once(table)); + } + + async monitor(url, callback, verbose) { + /* + Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall. + Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => GUN.monitor)(b: dispatchEvent) + + :param url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd + :param callback: function({type, key, value}) Callback for each new item added to the list + + :param verbose: boolean - true for debugging output + */ + url = typeof url === "string" ? url : url.href; + let conn = this.p_connection(url, verbose); + # See https://github.com/amark/gun/wiki/API#map for why this + # What we really want is to have the callback called once for each changed BUT + # conn.map().on(cb) will also get called for each initial value + # conn.on(cb) and then throwing away initial call would be ok, except it streams so cb might be called with first half of data and then rest + # TODO-GUN - waiting on an option for the above to have compliant monitor, for now just make sure to ignore dupes and note that GUN doesnt support list/add/listmonitor anyway + conn.map().on((v, k) => callback("set", k, JSON.parse(v))); + } + + + static async test(transport, verbose) { //TODO-GUN rewrite this + if (verbose) {console.log("TransportGUN.test")} + try { + let testurl = "1114"; // Just a predictable number can work with + let res = await transport.p_rawlist(testurl, verbose); + let listlen = res.length; // Holds length of list run intermediate + if (verbose) console.log("rawlist returned ", ...Dweb.utils.consolearr(res)); + transport.listmonitor(testurl, (obj) => console.log("Monitored", obj), verbose); + let sig = new Dweb.Signature({urls: ["123"], date: new Date(Date.now()), signature: "Joe Smith", signedby: [testurl]}, verbose); + await transport.p_rawadd(testurl, sig, verbose); + if (verbose) console.log("TransportIPFS.p_rawadd returned "); + res = await transport.p_rawlist(testurl, verbose); + if (verbose) console.log("rawlist returned ", ...Dweb.utils.consolearr(res)); // Note not showing return + await delay(500); + res = await transport.p_rawlist(testurl, verbose); + console.assert(res.length === listlen + 1, "Should have added one item"); + } catch(err) { + console.log("Exception thrown in TransportORBITDB.test:", err.message); + throw err; + } + } + +} +Transports._transportclasses["GUN"] = TransportGUN; +TransportGUN.GUN = GUN; // Allow node tests to find it +exports = module.exports = TransportGUN; diff --git a/TransportIPFS.js b/TransportIPFS.js index 095bbb7..1162142 100644 --- a/TransportIPFS.js +++ b/TransportIPFS.js @@ -119,7 +119,7 @@ class TransportIPFS extends Transport { await this.p_ipfsstart(verbose); // Throws Error("websocket error") and possibly others. this.status = await this.p_status(verbose); } catch(err) { - console.error("IPFS failed to connect",err); + console.error(this.name, "failed to connect", err); this.status = Transport.STATUS_FAILED; } if (cb) cb(this); diff --git a/TransportWEBTORRENT.js b/TransportWEBTORRENT.js index e9b9d3b..689b01a 100644 --- a/TransportWEBTORRENT.js +++ b/TransportWEBTORRENT.js @@ -73,7 +73,7 @@ class TransportWEBTORRENT extends Transport { await this.p_webtorrentstart(verbose); await this.p_status(verbose); } catch(err) { - console.error("WebTorrent failed to connect",err); + console.error(this.name, "failed to connect", err); this.status = Transport.STATUS_FAILED; } if (cb) cb(this); diff --git a/TransportYJS.js b/TransportYJS.js index 99cfc8c..7ffe3fd 100644 --- a/TransportYJS.js +++ b/TransportYJS.js @@ -123,7 +123,7 @@ class TransportYJS extends Transport { this.yarrays = {}; await this.p_status(verbose); } catch(err) { - console.error("YJS failed to start",err); + console.error(this.name,"failed to start",err); this.status = Transport.STATUS_FAILED; } if (cb) cb(this); @@ -231,8 +231,6 @@ class TransportYJS extends Transport { } - // Support for Key-Value pairs as per - // https://docs.google.com/document/d/1yfmLRqKPxKwB939wIy9sSaa7GKOzM5PrCZ4W1jRGW6M/edit# async p_newdatabase(pubkey, {verbose=false}={}) { //if (pubkey instanceof Dweb.PublicPrivate) if (pubkey.hasOwnProperty("keypair")) @@ -252,7 +250,12 @@ class TransportYJS extends Transport { return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it } - async p_set(url, keyvalues, value, {verbose=false}={}) { // url = yjs:/yjs/database/table/key + async p_set(url, keyvalues, value, {verbose=false}={}) { // url = yjs:/yjs/database/table + /* + Set key values + keyvalues: string (key) in which case value should be set there OR + object in which case value is ignored + */ let y = await this.p_connection(url, verbose); if (typeof keyvalues === "string") { y.share.map.set(keyvalues, JSON.stringify(value)); @@ -302,7 +305,7 @@ class TransportYJS extends Transport { } async monitor(url, callback, verbose) { /* - Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist. + Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall. Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => YJS.monitor)(b: dispatchEvent) :param url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd diff --git a/Transports.js b/Transports.js index abf1f73..f36ea76 100644 --- a/Transports.js +++ b/Transports.js @@ -71,6 +71,9 @@ class Transports { // Need a async version of this for serviceworker and TransportsProxy return this.validFor(urls, func, options).map((ut) => ut[0]); } + + // SEE-OTHER-ADDTRANSPORT + static http(verbose) { // Find an http transport if it exists, so for example YJS can use it. return Transports._connected().find((t) => t.name === "HTTP") @@ -85,6 +88,12 @@ class Transports { return Transports._connected().find((t) => t.name === "WEBTORRENT") } + static gun(verbose) { + // Find a GUN transport if it exists + return Transports._connected().find((t) => t.name === "GUN") + } + + static async p_resolveNames(urls) { /* If and only if TransportNAME was loaded (it might not be as it depends on higher level classes like Domain and SmartDict) then resolve urls that might be names, returning a modified array. @@ -535,7 +544,7 @@ class Transports { let tabbrevs = options.transports; // Array of transport abbreviations this._optionspaused = (options.paused || []).map(n => n.toUpperCase()); // Array of transports paused - defaults to none, upper cased if (!(tabbrevs && tabbrevs.length)) { tabbrevs = options.defaulttransports || [] } - if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT"]; } + if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT", "GUN"]; } // SEE-OTHER-ADDTRANSPORT tabbrevs = tabbrevs.map(n => n.toUpperCase()); let transports = this.setup0(tabbrevs, options, verbose); if (options.statuscb) { diff --git a/index.js b/index.js index 7f90895..4be1ac7 100644 --- a/index.js +++ b/index.js @@ -1,9 +1,11 @@ // Order is significant as should search earlier ones first // put IPFS before Webtorrent for showcasing, as Webtorrent works in some cases IPFS doesnt so that way we exercise both const DwebTransports = require("./Transports.js"); +// SEE-OTHER-ADDTRANSPORT require("./TransportHTTP.js"); // Can access via window.DwebTransports._transportclasses["HTTP"] require("./TransportIPFS.js"); require("./TransportYJS.js"); require("./TransportWEBTORRENT.js"); +require("./TransportGUN.js"); if (typeof window !== "undefined") { window.DwebTransports = DwebTransports; } exports = module.exports = DwebTransports; diff --git a/package.json b/package.json index d123449..188009a 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "ipfs": "latest", "ipfs-unixfs": "^0.1.15", "node-fetch": "latest", + "gun": "latest", "readable-stream": "latest", "webtorrent": "^0.99.3", "y-array": "latest",