Merge branch 'master' into split

# Conflicts:
#	dist/dweb-transports-bundle.js

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 data.json
 /radata/
+yarn-error.log

DOCUMENTINDEX.md

@@ -1,13 +1,14 @@
 # Dweb Index and Links
 Mitra Ardron - last update 15th September 2018
 
+NOTE THIS DOC IS OUT OF DATE, SOME OF THE LINKS ARE PROBABLY STILL USEFUL BUT ITS NOT COMPREHENSIVE
+
 This is a short document of links relevant to the Dweb Project at the Internet Archive.
 
 This doc will gradually replace the myriad, mostly out of date, links sections in each of the doc files.
 
 ## Sites
 * [dweb.archive.org](https://dweb.archive.org) - The Internet Archive but decentralized - front page of IA
-* [Examples](https://dweb.me/examples) - index of our examples demonstrating libraries and archive site
 * [Decentralizedweb.net](https://Decentralizedweb.net) - Webpage for the Decentralized Web Summits of 2016 & 2018
 
 # Repositories
@@ -20,11 +21,10 @@ This doc will gradually replace the myriad, mostly out of date, links sections i
     * Webtorrent Seeder & Tracker - that collaborate to provide access to all IA torrents
     * URI-forwarding: .[MD](https://github.com/internetarchive/dweb-transport/blob/master/URL-forwards.md) .[JPG](https://github.com/internetarchive/dweb-transport/blob/master/URL-forwards.jpg) Documentation of URI forwarding in various tools
 * [Dweb-Serviceworker](https://github.com/internetarchive/dweb-serviceworker) - [README](https://github.com/internetarchive/dweb-serviceworker/blob/master/README.md) - Experimental (incomplete, unused) service worker proxy for Dweb-Transports
-* [Dweb-Mirror](https://github.com/internetarchive/dweb-mirror) - [README](https://github.com/internetarchive/dweb-mirror/blob/master/README.md) Mirroring and serving subsets of the Archive - builds on dweb-transports and dweb-objects and dweb-archive
+* [Dweb-Mirror](https://github.com/internetarchive/dweb-mirror) - [README](https://github.com/internetarchive/dweb-mirror/blob/master/README.md) Offline Archive - Mirroring and serving subsets of the Archive - builds on dweb-transports and dweb-objects and dweb-archive
 * [Dweb-Universal](https://github.com/mitra42/dweb-universal) - [README](https://github.com//mitra42/dweb-universal/blob/master/README.md)  Overview repo for the 2019 “universal” project - making the IA more accessible where the internet is poor. 
     * [Internet Archive Dweb overview.md](https://github.com/mitra42/dweb-universal/blob/master/Internet Archive Dweb overview.md) Higher level overview of the Dweb projects and repos.
     * [Dweb Universal architecture.pdf](https://github.com/mitra42/dweb-universal/blob/master/Dweb%20Universal%20architecture.pdf) Diagram 
-    * [Naming](https://github.com/mitra42/dweb-universal/blob/master/naming.md)  - proposal for naming in the dweb
     * [URI structure for HTTP Server](https://github.com/mitra42/dweb-universal/blob/master/uri%20structure%20for%20http%20server.md)
     * [Xyz but decentralized](https://github.com/mitra42/dweb-universal/blob/master/xyz%20but%20decentralized.md)
 * [Dweb-Ext](https://github.com/abhidas17695/dweb-ext/) - [README](https://github.com/abhidas17695/dweb-ext/blob/master/README.md) - Browser extension for booting into the dweb

Naming.js

@@ -0,0 +1,94 @@
+const debug = require('debug')('dweb-transports:naming');
+
+const domains = {
+  arc: {
+    "archive.org": {
+      ".": ["https://dweb.me/archive/archive.html"],
+      "about": ["https://archive.org/about/"],
+      "details": ["https://dweb.me/archive/archive.html?item="],
+      "examples": ["https://dweb.me/archive/examples/"],
+      "images": ["https://dweb.me/archive/images/"],
+      "serve": ["https://dweb.archive.org/download/"],
+      "metadata": [
+        "wolk://dweb.archive.org/metadata/",
+        "gun:/gun/arc/archive.org/metadata/",
+        "https://dweb.me/arc/archive.org/metadata/"],
+      "search.php": ["https://dweb.me/archive/archive.html?query="],
+      "search": ["https://dweb.me/archive/archive.html?query="],
+    },
+  },
+  ipfs: [ "http://ipfs.io/ipfs/", "https://dweb.me/ipfs/"],
+}
+
+
+function expand(partialUrl, remainder) {
+  return partialUrl.endsWith("html")
+  ? [partialUrl, remainder.join('/')] // THis might always be an error.
+  : partialUrl.endsWith("=")
+  ? partialUrl + remainder.join('/')
+  : (partialUrl.endsWith("/"))
+  ? partialUrl+remainder.join('/')
+  : undefined;
+}
+function resolve(parent, table, path) {
+  /**
+   * parent = STRING "a/b/c" path matched so far
+   * table = { key: url || [url]}
+   * path = "d/e/f"
+   * returns [ url || [url,remainder]] || undefined
+   */
+  //debug("Resolving %o in %s", path, parent);
+  const remainder = Array.isArray(path) ? path : path.split('/');
+  const name = remainder.shift();
+  const found = table[name] || table["."]
+  if (found) {
+    if (Array.isArray(found)) {
+      return (found.map(partialUrl => expand(partialUrl, remainder)).filter(url => !!url)); // [url || [url, remainder]]
+    } else if (typeof found === "object") {
+      return resolve([parent, name].join('/'), found, remainder);
+    } else if (typeof found === "string") {
+      return [ expand(found, remainder) ]
+    }
+  } else {
+    debug("WARNING unable to resolve %s in %s", name, parent.join('/') || '/' )
+    return undefined; // Remainder not found
+  }
+}
+
+function resolveName(url) {
+  return url.startsWith("dweb:/") ? resolve([], domains, url.slice(6)) : url; //
+}
+function naming(names) {
+    return [].concat(...names.map(n => resolveName(n)))
+}
+async function p_namingcb(names) {
+  return new Promise((resolve, reject) => { try { const res = naming(names); resolve(res); } catch(err) {reject(err)}}); // Promisify pattern v2b (no CB)
+}
+
+/*
+//TODO find in DM where its catching http://dweb.me and heading back to http://localhost:4244
+const testdata = {
+  "dweb:/arc/archive.org/metadata/foo": [
+    "https://dweb.me/arc/archive.org/metadata/foo",
+    "gun:/gun/arc/archive.org/metadata/foo",
+    "wolk://dweb.archive.org/metadata/foo" ],
+  "dweb:/arc/archive.org/details/foo": [
+    "https://dweb.me/archive/archive.html?item=foo"],
+}
+
+function test() {
+  Object.entries(testdata).forEach(kv => {
+    const res = resolveName(kv[0]);
+    if ((!res
+      || res.length !== kv[1].length)
+      || res.some(r => !kv[1].includes(r))) {
+    debug("%s => %s expect %s", kv[0], res, kv[1]);
+  }});
+  p_namingcb(["dweb:/arc/archive.org/details/foo","foo://bar.baz"])
+  .then(res => debug("Got %o", res))
+  .catch(err => debug("Fail %o", err.message));
+}
+test();
+*/
+
+exports = module.exports = {naming, p_namingcb};

README.md

@@ -126,6 +126,7 @@ See [Dweb document index](./DOCUMENTINDEX.md) for a list of the repos that make
 
 ### Release Notes
 
+* 0.1.63: Move naming internal
 * 0.1.62: Upgrade dependencies
 * 0.1.61: loopguard to return correct error from queuedFetch; add info to statuses 
 * 0.1.60: Back on master release of webtorrent

Transports.js

@@ -5,6 +5,7 @@ const debug = require('debug')('dweb-transports');
 const httptools = require('./httptools');
 const each = require('async/each');
 const map = require('async/map');
+const {p_namingcb, naming} = require('./Naming.js')
 
 class Transports {
     /*
@@ -858,7 +859,8 @@ class Transports {
     }
 }
 Transports._transports = [];    // Array of transport instances connected
-Transports.namingcb = undefined;    // Will be defined by the naming component (turns URLs for names into URLs for transport)
+Transports.naming = naming;
+Transports.namingcb = p_namingcb;    // Will be defined by the naming component (turns URLs for names into URLs for transport)
 Transports._transportclasses = {};  // Pointers to classes whose code is loaded.
 Transports.httptools = httptools;   // Static http tools
 exports = module.exports = Transports;

package.json

@@ -15,7 +15,7 @@
     "cids": "^0.7.1",
     "debug": "^4.1.1",
     "gun": "^0.2019.726",
-    "ipfs": "^0.37.0",
+    "ipfs": "^0.35.0",
     "ipfs-http-client": "^33.1.1",
     "ipfs-unixfs": "^0.1.16",
     "node-fetch": "^2.3.0",
@@ -53,5 +53,5 @@
     "test": "cd src; node ./test.js",
     "help": "echo 'test (test it)'; echo 'build (creates dweb-transports-bundle)'"
   },
-  "version": "0.1.62"
+  "version": "0.1.63"
 }