Minimum demoable mirror (commute)

Transports.js

@@ -94,17 +94,19 @@ class Transports {
         return Transports._connected().find((t) => t.name === "GUN")
     }
 
-
     static async p_resolveNames(urls) {
         /* If and only if TransportNAME was loaded (it might not be as it depends on higher level classes like Domain and SmartDict)
             then resolve urls that might be names, returning a modified array.
          */
-        if (this.namingcb) {
+        if (this.mirror) {
+            return Array.isArray(urls) ? urls.map(url=>this.gatewayUrl(url)) : this.gatewayUrl(url);
+        } else if (this.namingcb) {
             return await this.namingcb(urls);  // Array of resolved urls
         } else {
             return urls;
         }
     }
+
     static resolveNamesWith(cb) {
         // Set a callback for p_resolveNames
         this.namingcb = cb;
@@ -551,7 +553,7 @@ class Transports {
         returns array of transport instances
          */
         // "IPFS" or "IPFS,LOCAL,HTTP"
-        let localoptions = {http: {urlbase: "http://localhost:4244"}};
+        let localoptions = {http: {urlbase: "http://localhost:4244"}}; //TODO-MIRROR "localoptions" may not be needed any more
         return tabbrevs.map((tabbrev) => {
             let transportclass = this._transportclasses[ (tabbrev === "LOCAL") ? "HTTP" : tabbrev ];
             if (!transportclass) {
@@ -611,9 +613,7 @@ class Transports {
             if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT", "GUN"]; } // SEE-OTHER-ADDTRANSPORT
             tabbrevs = tabbrevs.map(n => n.toUpperCase());
             let transports = this.setup0(tabbrevs, options);
-            if (options.statuscb) {
+            ["statuscb", "mirror"].forEach(k => { if (options[k]) this[k] = options[k];} )
-                this.statuscb = options.statuscb;
-            }
             if (!!options.statuselement) {
                 let statuselement = options.statuselement;
                 while (statuselement.lastChild) {statuselement.removeChild(statuselement.lastChild); }   // Remove any exist status
@@ -661,52 +661,63 @@ class Transports {
         return Transports.http()._url(urls.find(u => (u.startsWith("contenthash") || u.startsWith("http") )), "content/rawfetch");
     }
 
-    static async test() {
+    static canonicalName(url, options={}) {
-        console.log("Transports.test")
+        /*
-        try {
+        Utility function to convert a variety of missentered, or presumed names into a canonical result that can be resolved or passed to a transport
-            /* Could convert this - copied fom YJS to do a test at the "Transports" level
+         */
-            let testurl = "yjs:/yjs/THISATEST";  // Just a predictable number can work with
+        if (typeof url !== "string") url = Url.parse(url).href;
-            let res = await transport.p_rawlist(testurl);
+        // In patterns below http or https; and  :/ or :// are treated the same
-            let listlen = res.length;   // Holds length of list run intermediate
+        const gateways = ["dweb.me", "ipfs.io"]; // Kniwn gateways, may dynamically load this at some point
-            console.log("rawlist returned ", ...utils.consolearr(res));
+        const protocols = ["ipfs","gun","magnet","yjs","arc", "contenthash", "http", "https"];
-            transport.listmonitor(testurl, (obj) => console.log("Monitored", obj));
+        const protocolsWantingDomains = ["arc", "http", "https"];
-            let sig = new Dweb.Signature({urls: ["123"], date: new Date(Date.now()), signature: "Joe Smith", signedby: [testurl]});
+        const gatewaypatts = [ // Must be before patts because gateway names often start with a valid proto
-            await transport.p_rawadd(testurl, sig);
+            /^http[s]?:[/]+([^/]+)[/](\w+)[/](.*)/i,   // https://(gateway)/proto/(internal)  + gateway in list (IPFS gateways. dweb.me)
-            console.log("TransportIPFS.p_rawadd returned ");
+        ]
-            res = await transport.p_rawlist(testurl);
+        const patts = [ // No overlap between patts & arcpatts, so order unimportant
-            console.log("rawlist returned ", ...utils.consolearr(res)); // Note not showing return
+            /^dweb:[/]+(\w+)[/]+(.*)/i,                         // dweb://(proto)/(internal)
-            await delay(500);
+            /^\w+:[/]+(\w+)[/](.*)/i,                           // proto1://proto2//(internal) - maybe only match if proto1=proto2 (must be before proto:/internal)
-            res = await transport.p_rawlist(testurl);
+            /^(\w+):[/]+(.*)/i,                                 // (proto)://(internal) # must be after proto1://proto2
-            console.assert(res.length === listlen + 1, "Should have added one item");
+            /^[/]*(\w+)[/](.*)/i,                               // /(proto)//(internal) - maybe only match if proto1=proto2
-            */
+            /^[/]*dweb[/]*(\w+)[/](.*)/i,                       // /dweb/(proto)//(internal)
-            //console.log("TransportYJS test complete");
+        ]
-            /* TODO-KEYVALUE reenable these tests,s but catch http examples
+        const arcpatts = [ // No overlap between patts & arcpatts, so order unimportant
-            let db = await this.p_newdatabase("TESTNOTREALLYAKEY");    // { privateurls, publicurls }
+            /^http[s]?:[/]+[^/]+[/](archive).(org)[/]*(.*)/i,   // https://localhost;123/(archive.org)/(internal)
-            console.assert(db.privateurls[0] === "yjs:/yjs/TESTNOTREALLYAKEY");
+            /^http[s]?:[/]+dweb.(\w+)[.]([^/]+)[/]*(.*)/i,      // https://dweb.(proto).(dom.ain)/(internal) # Before dweb.dom.ain
-            let table = await this.p_newtable("TESTNOTREALLYAKEY","TESTTABLE");         // { privateurls, publicurls }
+            // /^http[s]?:[/]+dweb.([^/]+[.][^/]+[/]*.*)/i,     // https://dweb.(dom.ain)/internal) or https://dweb.(domain) Handled by coe on recognizing above
-            let mapurls = table.publicurls;
+            /^(http[s])?:[/]+([^/]+)[/]+(.*)/i,                 // https://dom.ain/pa/th
-            console.assert(mapurls[0] === "yjs:/yjs/TESTNOTREALLYAKEY/TESTTABLE");
+        ]
-            await this.p_set(mapurls, "testkey", "testvalue");
-            let res = await this.p_get(mapurls, "testkey");
-            console.assert(res === "testvalue");
-            await this.p_set(mapurls, "testkey2", {foo: "bar"});
-            res = await this.p_get(mapurls, "testkey2");
-            console.assert(res.foo === "bar");
-            await this.p_set(mapurls, "testkey3", [1,2,3]);
-            res = await this.p_get(mapurls, "testkey3");
-            console.assert(res[1] === 2);
-            res = await this.p_keys(mapurls);
-            console.assert(res.length === 3 && res.includes("testkey3"));
-            res = await this.p_getall(mapurls);
-            console.assert(res.testkey2.foo === "bar");
-            */
 
-        } catch(err) {
+        for (let patt of gatewaypatts)  {
-            console.log("Exception thrown in Transports.test:", err.message);
+            let rr = url.match(patt);
-            throw err;
+            if (rr && gateways.includes(rr[1]) && protocols.includes(rr[2]))
+                return {proto: rr[2], internal: rr[3]};
         }
+        for (let patt of arcpatts)  {
+            let rr = url.match(patt);
+            if (rr) {
+                if (protocols.includes(rr[1])) {
+                    // arc (and possibly others) want the domain as part of the internal
+                    return {proto: rr[1], internal: (protocolsWantingDomains.includes(rr[1]) ? [rr[2], rr[3]].join('/') : rr[3])};
+                } else {
+                    return {proto: "arc", internal: [[rr[1], rr[2]].join('.'), rr[3]].join('/')};
+                }
+            }
+        };
+        for (let patt of patts)  {
+            let rr = url.match(patt);
+            if (rr && protocols.includes(rr[1]))
+                return {proto: rr[1], internal: rr[2]};
+        };
+        return undefined;
+    }
+    static canonicalUrl(url, options={}) {
+        let o = this.canonicalName(url, options);
+        return o.protocol + ":/" + o.internal;
+    }
+    static gatewayUrl(url) {
+        let o = Transports.canonicalName(url);
+        return [this.mirror, o.proto, o.internal].join('/');
     }
-
 }
 Transports._transports = [];    // Array of transport instances connected
 Transports.namingcb = undefined;    // Will be defined by the naming component (turns URLs for names into URLs for transport)

test.js

@@ -3,11 +3,83 @@ const DwebTransports = require('./index.js');
 async function p_test({transport=["GUN"]}={}) {
     if (Array.isArray(transport)) {
         for (tname of transport) {
-            await p_test({ transport: tname});   // Note this is going to run in parallel
+            await p_test({transport: tname});   // Note this is going to run in parallel
         }
     } else {
         let tclass = DwebTransports._transportclasses[transport];
         await tclass.p_test();
     }
 }
-p_test();
+async function test_transports() {
+    console.log("Transports.test")
+    try {
+        /* Could convert this - copied fom YJS to do a test at the "Transports" level
+        let testurl = "yjs:/yjs/THISATEST";  // Just a predictable number can work with
+        let res = await transport.p_rawlist(testurl);
+        let listlen = res.length;   // Holds length of list run intermediate
+        console.log("rawlist returned ", ...utils.consolearr(res));
+        transport.listmonitor(testurl, (obj) => console.log("Monitored", obj));
+        let sig = new Dweb.Signature({urls: ["123"], date: new Date(Date.now()), signature: "Joe Smith", signedby: [testurl]});
+        await transport.p_rawadd(testurl, sig);
+        console.log("TransportIPFS.p_rawadd returned ");
+        res = await transport.p_rawlist(testurl);
+        console.log("rawlist returned ", ...utils.consolearr(res)); // Note not showing return
+        await delay(500);
+        res = await transport.p_rawlist(testurl);
+        console.assert(res.length === listlen + 1, "Should have added one item");
+        */
+        //console.log("TransportYJS test complete");
+        /* TODO-KEYVALUE reenable these tests,s but catch http examples
+        let db = await this.p_newdatabase("TESTNOTREALLYAKEY");    // { privateurls, publicurls }
+        console.assert(db.privateurls[0] === "yjs:/yjs/TESTNOTREALLYAKEY");
+        let table = await this.p_newtable("TESTNOTREALLYAKEY","TESTTABLE");         // { privateurls, publicurls }
+        let mapurls = table.publicurls;
+        console.assert(mapurls[0] === "yjs:/yjs/TESTNOTREALLYAKEY/TESTTABLE");
+        await this.p_set(mapurls, "testkey", "testvalue");
+        let res = await this.p_get(mapurls, "testkey");
+        console.assert(res === "testvalue");
+        await this.p_set(mapurls, "testkey2", {foo: "bar"});
+        res = await this.p_get(mapurls, "testkey2");
+        console.assert(res.foo === "bar");
+        await this.p_set(mapurls, "testkey3", [1,2,3]);
+        res = await this.p_get(mapurls, "testkey3");
+        console.assert(res[1] === 2);
+        res = await this.p_keys(mapurls);
+        console.assert(res.length === 3 && res.includes("testkey3"));
+        res = await this.p_getall(mapurls);
+        console.assert(res.testkey2.foo === "bar");
+        */
+
+    } catch(err) {
+        console.log("Exception thrown in Transports.test:", err.message);
+        throw err;
+    }
+}
+
+function canonicalNameTests() {
+    // Test the regexps
+    [   // Each test shows the URL and the expected return of protocol and internal string, it doesnt test failing cases
+        ["https://dweb.me/ipfs/internal", "ipfs", "internal"],
+        ["https://dweb.ipfs.foo.bar/internal", "ipfs", "internal"],
+        ["dweb://ipfs/internal", "ipfs", "internal"],
+        ["ipfs://internal", "ipfs", "internal"],
+        ["gun://ipfs/internal", "ipfs", "internal"],
+        ["/ipfs/internal", "ipfs", "internal"],
+        ["/dweb/ipfs/internal", "ipfs", "internal"],
+        ["http://dweb.dom.ain/internal", "arc", "dom.ain/internal"],
+        ["http://localhost:123/archive.org/internal", "arc", "archive.org/internal"],
+        ["https://dweb.arc.dom.ain/internal", "arc", "dom.ain/internal"],
+        ["https://foo.bar/baz/splat", "https", "foo.bar/baz/splat"]
+    ].forEach((t) => {
+        console.log(t[0])
+        let res = DwebTransports.canonicalName(t[0]);
+        if (!(res && res["proto"] === t[1] && res["internal"] === t[2])) {
+            console.log("ERROR", t, res);
+        }
+    })
+}
+
+/*
+p_test()
+.then(() => test_transports);
+*/