Support for noCache (reload) partial

API.md

@@ -317,7 +317,7 @@ Finds an array or Transports that are STARTED and can support this URL.
 urls:       Array of urls
 func:       Function to check support for: fetch, store, add, list, listmonitor, reverse
             - see supportFunctions on each Transport class
-options     For future use
+options     checks supportFeatures
 Returns:    Array of pairs of url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
 ```
 
@@ -497,7 +497,7 @@ Option|Default|Meaning
 urlbase|https://dweb.me:443|Connect to dweb.me for contenthash urls
 supportURLS|http:*, https:*, contenthash:*}| (TODO: may in the future support `dweb:/contenthash/*`)
 supportFunctions|fetch, store, add, list, reverse, newlisturls, get, set, keys, getall, delete, newtable, newdatabase|
-supportFeatures|fetch.range|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
+supportFeatures|fetch.range, noCache|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
 
 
 ## TransportIPFS
@@ -525,7 +525,8 @@ supportURLS = `ipfs:*` (TODO: may in the future support `dweb:/ipfs/*`)
 SupportFunctions (note YJS uses IPFS and supports some other functions): 
     `fetch, store`
 SupportFeatures: 
-    fetch.range Not supported (currently April 2018))
+    fetch.range Not supported (currently May 2019))
+    noCache:    Not actually supported, but immutable
     
 Currently there is code for p_f_createReadStream. It works but because IPFS cannot return an error even if it 
 cannot open the stream, IPFS is usually set as the last choice transport for streams.
@@ -538,8 +539,9 @@ supportURLS = `yjs:*` (TODO: may in the future support `dweb:/yjs/*`)
 supportFunctions (note YJS uses IPFS and supports some other functions): 
     `fetch, add, list, listmonitor, newlisturls, connection, get, set, getall, keys, newdatabase, newtable, monitor`
 supportFeatures: 
-    fetch.range Not supported (currently April 2018)
+    fetch.range Not supported (currently May 2019)
-
+    noCache:    Not supported (currently May 2019)
+    
 ## TransportWEBTORRENT
 A subclass of Transport for handling WEBTORRENT connections (similar to, with interworking with BitTorrent)
 
@@ -553,7 +555,8 @@ supportFunctions:
     `fetch`, `createReadStream`
 
 supportFeatures: 
-    fetch.range Not supported (currently April 2018)
+    fetch.range Not supported (currently May 2019)
+    noCache:    Not actually supported, but immutable
 
 ## TransportGUN
 A subclass of Transport for handling GUN connections (decentralized database)
@@ -562,6 +565,7 @@ supportURLS = `gun:*` (TODO: may in the future support `dweb:/gun/*`)
 
 supportFunctions = `add`, `list`, `listmonitor`, `newlisturls`, `connection`, `get`, `set`, `getall`, `keys`, `newdatabase`, `newtable`, `monitor`
 supportFeatures: 
+    noCache:    Not supported (and cache flushing is hard)
 
 ## TransportWOLK 
 A subclass of Transport for handling the WOLK transport layer (decentralized, block chain based, incentivised storage)

Transport.js

@@ -96,11 +96,12 @@ class Transport {
         // True if connected (status==STATUS_CONNECTED==0) should not need subclassing
         return ! this.status;
     }
-    supports(url, func) { //TODO-API
+    supports(url, func, {noCache=undefined}={}) { //TODO-API
         /*
         Determine if this transport supports a certain set of URLs and a func
 
         :param url: String or parsed URL
+        :param opts:    { noCache }  check against supportFeatures
         :return:    true if this protocol supports these URLs and this func
         :throw:     TransportError if invalid URL
          */
@@ -112,13 +113,15 @@ class Transport {
         } //Should be TransportError but out of scope here
         // noinspection Annotator  supportURLs is defined in subclasses
         return (    (!url || this.supportURLs.includes(url.protocol.slice(0, -1)))
-            && (!func || this.supportFunctions.includes(func)))
+            && (!func || this.supportFunctions.includes(func))
+            && (!noCache || this.supportFeatures.includes("noCache"))
+        )
     }
 
-    validFor(url, func) { //TODO-API
+    validFor(url, func, opts) { //TODO-API
         // By default a transport can handle a url and a func if its connected and supports that url/func
         // This shouldnt need subclassing, an exception is HTTP which only applies "connected" against urls heading for the gateway
-        return this.connected() && this.supports(url, func);
+        return this.connected() && this.supports(url, func, opts);
     }
 
 

TransportGUN.js

@@ -73,6 +73,7 @@ class TransportGUN extends Transport {
         this.supportFunctions = [ 'fetch', //'store'
                                  'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor',
                                  'add', 'list', 'listmonitor', 'newlisturls'];
+        this.supportFeatures = []; // Doesnt support noCache and is mutable
         this.status = Transport.STATUS_LOADED;
     }
 

TransportHTTP.js

@@ -32,12 +32,13 @@ class TransportHTTP extends Transport {
         this.urlbase = options.urlbase; // e.g. https://dweb.me
         this.supportURLs = ['contenthash', 'http','https'];
         this.supportFunctions = ['fetch', 'store', 'add', 'list', 'reverse', 'newlisturls', "get", "set", "keys", "getall", "delete", "newtable", "newdatabase"]; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller
+        this.supportFeatures = ['noCache'];
         if (typeof window === "undefined") {
             // running in node, can support createReadStream,  (browser can't - see createReadStream below)
             this.supportFunctions.push("createReadStream");
         }
         // noinspection JSUnusedGlobalSymbols
-        this.supportFeatures = ['fetch.range'];
+        this.supportFeatures = ['fetch.range', 'noCache'];
         this.name = "HTTP";             // For console log etc
         this.status = Transport.STATUS_LOADED;
     }
@@ -87,10 +88,10 @@ class TransportHTTP extends Transport {
         return url;
     }
 
-    validFor(url, func) {
+    validFor(url, func, opts) {
         // Overrides Transport.prototype.validFor because HTTP's connection test is only really for dweb.me
         // in particular this allows urls like https://be-api.us.archive.org
-        return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func);
+        return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func, opts);
     }
     // noinspection JSCheckFunctionSignatures
     async p_rawfetch(url, opts={}) {
@@ -98,7 +99,7 @@ class TransportHTTP extends Transport {
         Fetch from underlying transport,
         Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have
         url: Of resource - which is turned into the HTTP url in p_httpfetch
-        opts: {start, end, retries} see p_GET for documentation
+        opts: {start, end, retries, noCache} see p_GET for documentation
         throws: TransportError if fails
          */
         //if (!(url && url.includes(':') ))

TransportIPFS.js

@@ -69,6 +69,7 @@ class TransportIPFS extends Transport {
         this.name = "IPFS";             // For console log etc
         this.supportURLs = ['ipfs'];
         this.supportFunctions = ['fetch', 'store', 'seed', 'createReadStream'];   // Does not support reverse
+        this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
         this.status = Transport.STATUS_LOADED;
     }
 

TransportWEBTORRENT.js

@@ -35,6 +35,7 @@ class TransportWEBTORRENT extends Transport {
         this.name = "WEBTORRENT";       // For console log etc
         this.supportURLs = ['magnet'];
         this.supportFunctions = ['fetch', 'createReadStream', "seed"];
+        this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
         this.status = Transport.STATUS_LOADED;
     }
 

TransportWOLK.js

@@ -30,6 +30,8 @@ class TransportWOLK extends Transport {
       this.name = "WOLK";          // For console log etc
       this.supportURLs = ['wolk'];
       this.supportFunctions = [ 'fetch',  'connection', 'get', 'set',  ]; // 'store' - requires chunkdata; 'createReadStream' not implemented
+      this.supportFeatures = []; // Doesnt support noCache and is mutable
+
       this.status = Transport.STATUS_LOADED;
     }
 

TransportYJS.js

@@ -50,6 +50,7 @@ class TransportYJS extends Transport {
         this.supportURLs = ['yjs'];
         this.supportFunctions = ['fetch', 'add', 'list', 'listmonitor', 'newlisturls',
             'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor'];   // Only does list functions, Does not support reverse,
+        this.supportFeatures = []; // Doesnt support noCache and is mutable
         this.status = Transport.STATUS_LOADED;
     }
 

Transports.js

@@ -44,7 +44,7 @@ class Transports {
         const res = this._transports.map((t) => { return {"name": t.name, "status": t.status}})
         if (cb) { cb(null, res)} else { return new Promise((resolve, reject) => resolve(res))}
     }
-    static validFor(urls, func, options) {
+    static validFor(urls, func, opts) { //TODO-RELOAD check for noCache support
         /*
         Finds an array or Transports that can support this URL.
 
@@ -52,7 +52,7 @@ class Transports {
 
         urls:       Array of urls
         func:       Function to check support for: fetch, store, add, list, listmonitor, reverse - see supportFunctions on each Transport class
-        options:    For future use
+        opts:       Passed to each Transport, esp for supportFeatures
         returns:    Array of pairs of Url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
         throws:     CodingError if urls empty or [undefined...]
          */
@@ -62,19 +62,19 @@ class Transports {
             return [];
         }
         if (!(urls && urls.length > 0)) { // No url supplied we are just checking which transports support this function on no url.
-            return this._transports.filter((t) => (t.validFor(undefined, func)))
+            return this._transports.filter((t) => (t.validFor(undefined, func, opts)))
                 .map((t) => [undefined, t]);
         } else {
             return [].concat(
                 ...urls.map((url) => typeof url === 'string' ? Url.parse(url) : url) // parse URLs once
                     .map((url) =>
-                        this._transports.filter((t) => (t.validFor(url, func))) // [ t1, t2 ]
+                        this._transports.filter((t) => (t.validFor(url, func, opts))) // [ t1, t2 ]
                             .map((t) => [url, t]))); // [[ u, t1], [u, t2]]
         }
     }
-    static async p_urlsValidFor(urls, func, options) {
+    static async p_urlsValidFor(urls, func, opts) {
         // Need a async version of this for serviceworker and TransportsProxy
-        return this.validFor(urls, func, options).map((ut) => ut[0]);
+        return this.validFor(urls, func, opts).map((ut) => ut[0]);
     }
 
     // SEE-OTHER-ADDTRANSPORT
@@ -181,6 +181,7 @@ class Transports {
             start,  integer - first byte wanted
             end     integer - last byte wanted (note this is inclusive start=0,end=1023 is 1024 bytes
             timeoutMS   integer - max time to wait on transports (IPFS) that support it
+            noCache bool - Skip caching (passed to Transports)
             }
         returns:	string - arbitrary bytes retrieved.
         throws:     TransportError with concatenated error messages if none succeed.
@@ -189,7 +190,7 @@ class Transports {
         if (!urls.length)  throw new errors.TransportError("Transports.p_rawfetch given an empty list of urls");
         let resolvedurls = await this.p_resolveNames(urls); // If naming is loaded then convert name to [urls]
         if (!resolvedurls.length)  throw new errors.TransportError("Transports.p_rawfetch none of the urls resolved: " + urls);
-        let tt = this.validFor(resolvedurls, "fetch"); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
+        let tt = this.validFor(resolvedurls, "fetch", {noCache: opts.noCache}); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
         if (!tt.length) {
             throw new errors.TransportError("Transports.p_rawfetch cant find any transport for urls: " + resolvedurls);
         }

httptools.js

@@ -105,18 +105,20 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
             start, end,     // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
             wantstream,     // Return a stream rather than data
             retries=12,        // How many times to retry
+            noCache         // Add Cache-Control: no-cache header
             }
         returns result via promise or cb(err, result)
     */
     if (typeof opts  === "function") { cb = opts; opts = {}; }
     let headers = new Headers();
     if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`);
+    //if (opts.noCache) headers.append("Cache-Control", "no-cache"); It complains about preflight with no-cache
     const retries = typeof opts.retries === "undefined" ? 12 : opts.retries;
     let init = {    //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
         method: 'GET',
         headers: headers,
         mode: 'cors',
-        cache: 'default',
+        cache: opts.noCache ? 'no-cache' : 'default', // In Chrome, This will set Cache-Control: max-age=0
         redirect: 'follow',  // Chrome defaults to manual
         keepalive: true    // Keep alive - mostly we'll be going back to same places a lot
     };