Gun: first cut

API.md

@@ -30,6 +30,7 @@ There are a set of classes:
 * *TransportIPFS*: Connects to IPFS, currently (April 2018) via WebSocketsStar (WSS)
 * *TransportYJS*: Implements shared lists, and dictionaries. Uses IPFS for transport
 * *TransportWEBTORRENT*: Integrates to Feross's WebTorrent library
+* *TransportGUN*: Integrates to the Gun DB
 * *Transports*: manages the list of conencted transports, and directs api calls to them. 
 
 Calls are generally made through the Transports class which knows how to route them to underlying connections.
@@ -323,6 +324,11 @@ returns instance of TransportIPFS if connected
 returns instance of TransportWEBTORRENT if connected
 ```
 
+##### static gun(verbose)
+```
+returns instance of TransportGUN if connected
+```
+
 ##### static async p_resolveNames(urls)
 See Naming below
 ```
@@ -345,7 +351,7 @@ t:        Add a Transport instance to _transports
 ##### static setup0(transports, options, verbose, cb)
 Calls setup0 for each transport based on its short name. Specially handles ‘LOCAL’ as a transport pointing at a local http server (for testing).
 ```
-transports      Array of short names of transports e.g. [‘IPFS’,’HTTP’,’ORBITDB’]
+transports      Array of short names of transports e.g. [‘IPFS’,’HTTP’,’GUN’]
 options         Passed to setup0 on each transport
 cb              Callback to be called each time status changes
 Returns:        Array of transport instances
@@ -518,6 +524,14 @@ supportFunctions:
 supportFeatures: 
     fetch.range Not supported (currently April 2018)
 
+## TransportGUN
+A subclass of Transport for handling GUN connections (decentralized database)
+
+supportURLS = `gun:*` (TODO: may in the future support `dweb:/gun/*`)
+supportFunctions 
+    `add, list, listmonitor, newlisturls, connection, get, set, getall, keys, newdatabase, newtable, monitor`
+supportFeatures: 
+
 ## Naming
 Independently from the transport, the Transport library can resolve names if provided an appropriate callback. 
 See p_resolveNames(urls) and resolveNamesWith(cb)
@@ -533,3 +547,13 @@ The format of names currently (April 2018) is under development but its likely t
 `dweb:/arc/archive.org/details/foo` 
 to allow smooth integration with existing HTTP urls that are moving to decentralization. 
 
+## Adding a Transport
+The following steps are needed to add a transport. 
+
+* Add a line to package.json/dependencies for any packages needed
+* Add lines to index.js; 
+* Add function to return the instance to Transports.js 
+* Add to list in Transports.p_connect()
+* Add to API.md
+* Look for any "SEE-OTHER-ADDTRANSPORT" in case not on this list
+* Edit a copy of the closest Transport to what you are building

Transport.js

@@ -15,7 +15,7 @@ class Transport {
         Fields:
         statuselement:  If set is an HTML Element that should be adjusted to indicate status (this is managed by Transports, just stored on Transport)
         statuscb: Callback when status changes
-        name:   Short name of element e.g. HTTP IPFS WEBTORRENT
+        name:   Short name of element e.g. HTTP IPFS WEBTORRENT GUN
         */
     }
 

TransportGUN

@@ -0,0 +1,217 @@
+/*
+This Transport layers uses GUN.
+*/
+const Url = require('url');
+const Gun = require('gun')
+
+// Utility packages (ours) And one-liners
+function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})}
+
+// Other Dweb modules
+const errors = require('./Errors'); // Standard Dweb Errors
+const Transport = require('./Transport.js'); // Base class for TransportXyz
+const Transports = require('./Transports'); // Manage all Transports that are loaded
+const utils = require('./utils'); // Utility functions
+
+let defaultoptions = {
+}
+
+class TransportGUN extends Transport {
+    /*
+    GUN specific transport - over IPFS
+
+    Fields:
+    gun: object returned when starting GUN
+     */
+
+    constructor(options, verbose) {
+        super(options, verbose);
+        this.options = options;         // Dictionary of options { ipfs: {...}, "yarrays", yarray: {...} }
+        this.gun = undefined;
+        this.name = "GUN";          // For console log etc
+        this.supportURLs = ['gun'];
+        #TODO-GUN doesnt really support lists yet, its "set" function only handles other gun objects.
+        this.supportFunctions = ['connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor']; // TODO-GUN check this:  ['fetch', 'add', 'list', 'listmonitor', 'newlisturls',]
+        this.status = Transport.STATUS_LOADED;
+    }
+
+    async p_connection(url, verbose) {
+        /*
+        Utility function to get Gun object for this URL
+        url:        URL string to find list of
+        resolves:   Gun a connection to use for get's etc.
+        */
+        if (typeof url === "string")
+            url = Url.parse(url);
+        patharray = url.pathstring.split('/')   //[ 'gun', database, table ]
+        patharray.shift;    // Loose "gun"
+        g = this.gun;
+        while (patharray.length) {
+            g = g.get(patharray.shift());   //TODO-GUN-TEST will this work if database never initialized
+        }
+        return g;
+    }
+
+    static setup0(options, verbose) {
+        /*
+            First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.
+            options: { gun: { }, }   Set of options - "gun" is used for those to pass direct to Gun
+        */
+        let combinedoptions = Transport.mergeoptions(defaultoptions, options);
+        console.log("GUN options %o", combinedoptions); // Log even if !verbose
+        let t = new TransportGUN(combinedoptions, verbose);   // Note doesnt start IPFS or OrbitDB
+        Dweb.Transports.addtransport(t);
+        return t;
+    }
+
+    async p_setup1(verbose, cb) {
+        /*
+        This sets up for GUN.
+        Throws: TODO-GUN document errors that can occur
+        */
+        try {
+            this.status = Dweb.Transport.STATUS_STARTING;   // Should display, but probably not refreshed in most case
+            if (cb) cb(this);
+            this.gun = new Gun(this.options.gun);   // TODO-GUN how do I know if this succeeded
+            await this.p_status(verbose);
+        } catch(err) {
+            console.error(this.name,"failed to start",err);
+            this.status = Transport.STATUS_FAILED;
+        }
+        if (cb) cb(this);
+        return this;
+    }
+
+    async p_status(verbose) {
+        /*
+        Return a string for the status of a transport. No particular format, but keep it short as it will probably be in a small area of the screen.
+         */
+        this.status = Dweb.Transport.STATUS_CONNECTED;  //TODO-GUN how do I know if/when I'm connected (see comment on p_setup1 as well)
+        return this.status;
+    }
+
+    async p_newdatabase(pubkey, {verbose=false}={}) {
+        /*
+        Request a new database
+        For GUN it doesnt actually create anything, just generates the URLs
+        TODO-GUN how to make globally accessible if normalized to my UUID
+        TODO-GUN how to get a private Url that enables me to write to it?
+
+        returns: {publicurl: "gun:/gun/<publickey>", privateurl:  "gun:/gun/<publickey>">
+        */
+        if (pubkey.hasOwnProperty("keypair"))
+            pubkey = pubkey.keypair.signingexport()
+        // By this point pubkey should be an export of a public key of form xyz:abc where xyz
+        // specifies the type of public key (NACL VERIFY being the only kind we expect currently)
+        let u =  `gun:/gun/${encodeURIComponent(pubkey)}`;
+        return {"publicurl": u, "privateurl": u};
+    }
+
+    async p_newtable(pubkey, table, {verbose=false}={}) {
+        /*
+        Request a new table
+        For GUN it doesnt actually create anything, just generates the URLs
+
+        returns: {publicurl: "gun:/gun/<publickey>/<table>", privateurl:  "gun:/gun/<publickey>/<table>">
+        */
+        if (!pubkey) throw new errors.CodingError("p_newtable currently requires a pubkey");
+        let database = await this.p_newdatabase(pubkey, {verbose});
+        // If have use cases without a database, then call p_newdatabase first
+        return { privateurl: `${database.privateurl}/${table}`,  publicurl: `${database.publicurl}/${table}`}  // No action required to create it
+    }
+
+    async p_set(url, keyvalues, value, {verbose=false}={}) {  // url = yjs:/yjs/database/table
+        /*
+        Set key values
+        keyvalues:  string (key) in which case value should be set there OR
+                object in which case value is ignored
+         */
+        let table = await this.p_connection(url, verbose);
+        if (typeof keyvalues === "string") {
+            table.path(keyvalues).put(JSON.stringify(value));
+        } else {
+            table.put(keyvalues);   // Store all key-value pairs without destroying any other key/value pairs previously set
+        }
+    }
+    async p_get(url, keys, {verbose=false}={}) {
+        let table = await this.p_connection(url, verbose);
+        if (Array.isArray(keys)) {
+            return keys.reduce(function(previous, key) {
+                let val = table.get(key);
+                previous[key] = typeof val === "string" ? JSON.parse(val) : val;    // Handle undefined
+                return previous;
+            }, {});
+        } else {
+            let val = table.get(keys);
+            return typeof val === "string" ? JSON.parse(val) : val;  // This looks like it is sync
+        }
+    }
+
+    async p_delete(url, keys, {verbose=false}={}) {
+        let table = await this.p_connection(url, verbose);
+        if (typeof keys === "string") {
+            table.path(keys).put(null);
+        } else {
+            keys.map((key) => table.path(key).put(null));  // This looks like it is sync
+        }
+    }
+
+    _p_once(table) {
+        return new Promise((resolve, reject) => table.once(resolve));
+    }
+    async p_keys(url, {verbose=false}={}) {
+        kvs = await this.p_getall(url, {verbose});
+        return Object.keys(kvs);
+    }
+    async p_getall(url, {verbose=false}={}) {
+        let table = await this.p_connection(url, verbose);
+        return this._p_once(table));
+    }
+
+    async monitor(url, callback, verbose) {
+        /*
+         Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall.
+         Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => GUN.monitor)(b: dispatchEvent)
+
+         :param url:         string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
+         :param callback:    function({type, key, value})  Callback for each new item added to the list
+
+         :param verbose:     boolean - true for debugging output
+          */
+        url = typeof url === "string" ? url : url.href;
+        let conn = this.p_connection(url, verbose);
+        # See https://github.com/amark/gun/wiki/API#map for why this
+        # What we really want is to have the callback called once for each changed BUT
+        # conn.map().on(cb) will also get called for each initial value
+        # conn.on(cb) and then throwing away initial call would be ok, except it streams so cb might be called with first half of data and then rest
+        # TODO-GUN - waiting on an option for the above to have compliant monitor, for now just make sure to ignore dupes and note that GUN doesnt support list/add/listmonitor anyway
+        conn.map().on((v, k) => callback("set", k, JSON.parse(v)));
+    }
+
+
+    static async test(transport, verbose) { //TODO-GUN rewrite this
+        if (verbose) {console.log("TransportGUN.test")}
+        try {
+            let testurl = "1114";  // Just a predictable number can work with
+            let res = await transport.p_rawlist(testurl, verbose);
+            let listlen = res.length;   // Holds length of list run intermediate
+            if (verbose) console.log("rawlist returned ", ...Dweb.utils.consolearr(res));
+            transport.listmonitor(testurl, (obj) => console.log("Monitored", obj), verbose);
+            let sig = new Dweb.Signature({urls: ["123"], date: new Date(Date.now()), signature: "Joe Smith", signedby: [testurl]}, verbose);
+            await transport.p_rawadd(testurl, sig, verbose);
+            if (verbose) console.log("TransportIPFS.p_rawadd returned ");
+            res = await transport.p_rawlist(testurl, verbose);
+            if (verbose) console.log("rawlist returned ", ...Dweb.utils.consolearr(res)); // Note not showing return
+            await delay(500);
+            res = await transport.p_rawlist(testurl, verbose);
+            console.assert(res.length === listlen + 1, "Should have added one item");
+        } catch(err) {
+            console.log("Exception thrown in TransportORBITDB.test:", err.message);
+            throw err;
+        }
+    }
+
+}
+Transports._transportclasses["GUN"] = TransportGUN;
+TransportGUN.GUN = GUN; // Allow node tests to find it
+exports = module.exports = TransportGUN;

TransportIPFS.js

@@ -119,7 +119,7 @@ class TransportIPFS extends Transport {
             await this.p_ipfsstart(verbose);    // Throws Error("websocket error") and possibly others.
             this.status = await this.p_status(verbose);
         } catch(err) {
-            console.error("IPFS failed to connect",err);
+            console.error(this.name, "failed to connect", err);
             this.status = Transport.STATUS_FAILED;
         }
         if (cb) cb(this);

TransportWEBTORRENT.js

@@ -73,7 +73,7 @@ class TransportWEBTORRENT extends Transport {
             await this.p_webtorrentstart(verbose);
             await this.p_status(verbose);
         } catch(err) {
-            console.error("WebTorrent failed to connect",err);
+            console.error(this.name, "failed to connect", err);
             this.status = Transport.STATUS_FAILED;
         }
         if (cb) cb(this);

TransportYJS.js

@@ -123,7 +123,7 @@ class TransportYJS extends Transport {
             this.yarrays = {};
             await this.p_status(verbose);
         } catch(err) {
-            console.error("YJS failed to start",err);
+            console.error(this.name,"failed to start",err);
             this.status = Transport.STATUS_FAILED;
         }
         if (cb) cb(this);
@@ -231,8 +231,6 @@ class TransportYJS extends Transport {
     }
 
 
-    // Support for Key-Value pairs as per
-    // https://docs.google.com/document/d/1yfmLRqKPxKwB939wIy9sSaa7GKOzM5PrCZ4W1jRGW6M/edit#
     async p_newdatabase(pubkey, {verbose=false}={}) {
         //if (pubkey instanceof Dweb.PublicPrivate)
         if (pubkey.hasOwnProperty("keypair"))
@@ -252,7 +250,12 @@ class TransportYJS extends Transport {
         return { privateurl: `${database.privateurl}/${table}`,  publicurl: `${database.publicurl}/${table}`}  // No action required to create it
     }
 
-    async p_set(url, keyvalues, value, {verbose=false}={}) {  // url = yjs:/yjs/database/table/key
+    async p_set(url, keyvalues, value, {verbose=false}={}) {  // url = yjs:/yjs/database/table
+        /*
+        Set key values
+        keyvalues:  string (key) in which case value should be set there OR
+                object in which case value is ignored
+         */
         let y = await this.p_connection(url, verbose);
         if (typeof keyvalues === "string") {
             y.share.map.set(keyvalues, JSON.stringify(value));
@@ -302,7 +305,7 @@ class TransportYJS extends Transport {
     }
     async monitor(url, callback, verbose) {
         /*
-         Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist.
+         Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall.
          Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => YJS.monitor)(b: dispatchEvent)
 
          :param url:         string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd

Transports.js

@@ -71,6 +71,9 @@ class Transports {
         // Need a async version of this for serviceworker and TransportsProxy
         return this.validFor(urls, func, options).map((ut) => ut[0]);
     }
+
+    // SEE-OTHER-ADDTRANSPORT
+
     static http(verbose) {
         // Find an http transport if it exists, so for example YJS can use it.
         return Transports._connected().find((t) => t.name === "HTTP")
@@ -85,6 +88,12 @@ class Transports {
         return Transports._connected().find((t) => t.name === "WEBTORRENT")
     }
 
+    static gun(verbose) {
+        // Find a GUN transport if it exists
+        return Transports._connected().find((t) => t.name === "GUN")
+    }
+
+
     static async p_resolveNames(urls) {
         /* If and only if TransportNAME was loaded (it might not be as it depends on higher level classes like Domain and SmartDict)
             then resolve urls that might be names, returning a modified array.
@@ -535,7 +544,7 @@ class Transports {
             let tabbrevs = options.transports;    // Array of transport abbreviations
             this._optionspaused = (options.paused || []).map(n => n.toUpperCase());       // Array of transports paused - defaults to none, upper cased
             if (!(tabbrevs && tabbrevs.length)) { tabbrevs = options.defaulttransports || [] }
-            if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT"]; }
+            if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT", "GUN"]; } // SEE-OTHER-ADDTRANSPORT
             tabbrevs = tabbrevs.map(n => n.toUpperCase());
             let transports = this.setup0(tabbrevs, options, verbose);
             if (options.statuscb) {

index.js

@@ -1,9 +1,11 @@
 // Order is significant as should search earlier ones first
 // put IPFS before Webtorrent for showcasing, as Webtorrent works in some cases IPFS doesnt so that way we exercise both
 const DwebTransports = require("./Transports.js");
+// SEE-OTHER-ADDTRANSPORT
 require("./TransportHTTP.js");   // Can access via window.DwebTransports._transportclasses["HTTP"]
 require("./TransportIPFS.js");
 require("./TransportYJS.js");
 require("./TransportWEBTORRENT.js");
+require("./TransportGUN.js");
 if (typeof window !== "undefined") { window.DwebTransports = DwebTransports; }
 exports = module.exports = DwebTransports;

package.json

@@ -15,6 +15,7 @@
     "ipfs": "latest",
     "ipfs-unixfs": "^0.1.15",
     "node-fetch": "latest",
+    "gun": "latest",
     "readable-stream": "latest",
     "webtorrent": "^0.99.3",
     "y-array": "latest",