Support for noCache (reload) partial

This commit is contained in:
Mitra Ardron 2019-05-30 15:57:37 +10:00
parent 38ca1c92af
commit 82890eab97
11 changed files with 22970 additions and 544 deletions

14
API.md
View File

@ -317,7 +317,7 @@ Finds an array or Transports that are STARTED and can support this URL.
urls: Array of urls
func: Function to check support for: fetch, store, add, list, listmonitor, reverse
- see supportFunctions on each Transport class
options For future use
options checks supportFeatures
Returns: Array of pairs of url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
```
@ -497,7 +497,7 @@ Option|Default|Meaning
urlbase|https://dweb.me:443|Connect to dweb.me for contenthash urls
supportURLS|http:*, https:*, contenthash:*}| (TODO: may in the future support `dweb:/contenthash/*`)
supportFunctions|fetch, store, add, list, reverse, newlisturls, get, set, keys, getall, delete, newtable, newdatabase|
supportFeatures|fetch.range|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
supportFeatures|fetch.range, noCache|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
## TransportIPFS
@ -525,7 +525,8 @@ supportURLS = `ipfs:*` (TODO: may in the future support `dweb:/ipfs/*`)
SupportFunctions (note YJS uses IPFS and supports some other functions):
`fetch, store`
SupportFeatures:
fetch.range Not supported (currently April 2018))
fetch.range Not supported (currently May 2019))
noCache: Not actually supported, but immutable
Currently there is code for p_f_createReadStream. It works but because IPFS cannot return an error even if it
cannot open the stream, IPFS is usually set as the last choice transport for streams.
@ -538,7 +539,8 @@ supportURLS = `yjs:*` (TODO: may in the future support `dweb:/yjs/*`)
supportFunctions (note YJS uses IPFS and supports some other functions):
`fetch, add, list, listmonitor, newlisturls, connection, get, set, getall, keys, newdatabase, newtable, monitor`
supportFeatures:
fetch.range Not supported (currently April 2018)
fetch.range Not supported (currently May 2019)
noCache: Not supported (currently May 2019)
## TransportWEBTORRENT
A subclass of Transport for handling WEBTORRENT connections (similar to, with interworking with BitTorrent)
@ -553,7 +555,8 @@ supportFunctions:
`fetch`, `createReadStream`
supportFeatures:
fetch.range Not supported (currently April 2018)
fetch.range Not supported (currently May 2019)
noCache: Not actually supported, but immutable
## TransportGUN
A subclass of Transport for handling GUN connections (decentralized database)
@ -562,6 +565,7 @@ supportURLS = `gun:*` (TODO: may in the future support `dweb:/gun/*`)
supportFunctions = `add`, `list`, `listmonitor`, `newlisturls`, `connection`, `get`, `set`, `getall`, `keys`, `newdatabase`, `newtable`, `monitor`
supportFeatures:
noCache: Not supported (and cache flushing is hard)
## TransportWOLK
A subclass of Transport for handling the WOLK transport layer (decentralized, block chain based, incentivised storage)

View File

@ -96,11 +96,12 @@ class Transport {
// True if connected (status==STATUS_CONNECTED==0) should not need subclassing
return ! this.status;
}
supports(url, func) { //TODO-API
supports(url, func, {noCache=undefined}={}) { //TODO-API
/*
Determine if this transport supports a certain set of URLs and a func
:param url: String or parsed URL
:param opts: { noCache } check against supportFeatures
:return: true if this protocol supports these URLs and this func
:throw: TransportError if invalid URL
*/
@ -112,13 +113,15 @@ class Transport {
} //Should be TransportError but out of scope here
// noinspection Annotator supportURLs is defined in subclasses
return ( (!url || this.supportURLs.includes(url.protocol.slice(0, -1)))
&& (!func || this.supportFunctions.includes(func)))
&& (!func || this.supportFunctions.includes(func))
&& (!noCache || this.supportFeatures.includes("noCache"))
)
}
validFor(url, func) { //TODO-API
validFor(url, func, opts) { //TODO-API
// By default a transport can handle a url and a func if its connected and supports that url/func
// This shouldnt need subclassing, an exception is HTTP which only applies "connected" against urls heading for the gateway
return this.connected() && this.supports(url, func);
return this.connected() && this.supports(url, func, opts);
}

View File

@ -73,6 +73,7 @@ class TransportGUN extends Transport {
this.supportFunctions = [ 'fetch', //'store'
'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor',
'add', 'list', 'listmonitor', 'newlisturls'];
this.supportFeatures = []; // Doesnt support noCache and is mutable
this.status = Transport.STATUS_LOADED;
}

View File

@ -32,12 +32,13 @@ class TransportHTTP extends Transport {
this.urlbase = options.urlbase; // e.g. https://dweb.me
this.supportURLs = ['contenthash', 'http','https'];
this.supportFunctions = ['fetch', 'store', 'add', 'list', 'reverse', 'newlisturls', "get", "set", "keys", "getall", "delete", "newtable", "newdatabase"]; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller
this.supportFeatures = ['noCache'];
if (typeof window === "undefined") {
// running in node, can support createReadStream, (browser can't - see createReadStream below)
this.supportFunctions.push("createReadStream");
}
// noinspection JSUnusedGlobalSymbols
this.supportFeatures = ['fetch.range'];
this.supportFeatures = ['fetch.range', 'noCache'];
this.name = "HTTP"; // For console log etc
this.status = Transport.STATUS_LOADED;
}
@ -87,10 +88,10 @@ class TransportHTTP extends Transport {
return url;
}
validFor(url, func) {
validFor(url, func, opts) {
// Overrides Transport.prototype.validFor because HTTP's connection test is only really for dweb.me
// in particular this allows urls like https://be-api.us.archive.org
return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func);
return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func, opts);
}
// noinspection JSCheckFunctionSignatures
async p_rawfetch(url, opts={}) {
@ -98,7 +99,7 @@ class TransportHTTP extends Transport {
Fetch from underlying transport,
Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have
url: Of resource - which is turned into the HTTP url in p_httpfetch
opts: {start, end, retries} see p_GET for documentation
opts: {start, end, retries, noCache} see p_GET for documentation
throws: TransportError if fails
*/
//if (!(url && url.includes(':') ))

View File

@ -69,6 +69,7 @@ class TransportIPFS extends Transport {
this.name = "IPFS"; // For console log etc
this.supportURLs = ['ipfs'];
this.supportFunctions = ['fetch', 'store', 'seed', 'createReadStream']; // Does not support reverse
this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
this.status = Transport.STATUS_LOADED;
}

View File

@ -35,6 +35,7 @@ class TransportWEBTORRENT extends Transport {
this.name = "WEBTORRENT"; // For console log etc
this.supportURLs = ['magnet'];
this.supportFunctions = ['fetch', 'createReadStream', "seed"];
this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
this.status = Transport.STATUS_LOADED;
}

View File

@ -30,6 +30,8 @@ class TransportWOLK extends Transport {
this.name = "WOLK"; // For console log etc
this.supportURLs = ['wolk'];
this.supportFunctions = [ 'fetch', 'connection', 'get', 'set', ]; // 'store' - requires chunkdata; 'createReadStream' not implemented
this.supportFeatures = []; // Doesnt support noCache and is mutable
this.status = Transport.STATUS_LOADED;
}

View File

@ -50,6 +50,7 @@ class TransportYJS extends Transport {
this.supportURLs = ['yjs'];
this.supportFunctions = ['fetch', 'add', 'list', 'listmonitor', 'newlisturls',
'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor']; // Only does list functions, Does not support reverse,
this.supportFeatures = []; // Doesnt support noCache and is mutable
this.status = Transport.STATUS_LOADED;
}

View File

@ -44,7 +44,7 @@ class Transports {
const res = this._transports.map((t) => { return {"name": t.name, "status": t.status}})
if (cb) { cb(null, res)} else { return new Promise((resolve, reject) => resolve(res))}
}
static validFor(urls, func, options) {
static validFor(urls, func, opts) { //TODO-RELOAD check for noCache support
/*
Finds an array or Transports that can support this URL.
@ -52,7 +52,7 @@ class Transports {
urls: Array of urls
func: Function to check support for: fetch, store, add, list, listmonitor, reverse - see supportFunctions on each Transport class
options: For future use
opts: Passed to each Transport, esp for supportFeatures
returns: Array of pairs of Url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
throws: CodingError if urls empty or [undefined...]
*/
@ -62,19 +62,19 @@ class Transports {
return [];
}
if (!(urls && urls.length > 0)) { // No url supplied we are just checking which transports support this function on no url.
return this._transports.filter((t) => (t.validFor(undefined, func)))
return this._transports.filter((t) => (t.validFor(undefined, func, opts)))
.map((t) => [undefined, t]);
} else {
return [].concat(
...urls.map((url) => typeof url === 'string' ? Url.parse(url) : url) // parse URLs once
.map((url) =>
this._transports.filter((t) => (t.validFor(url, func))) // [ t1, t2 ]
this._transports.filter((t) => (t.validFor(url, func, opts))) // [ t1, t2 ]
.map((t) => [url, t]))); // [[ u, t1], [u, t2]]
}
}
static async p_urlsValidFor(urls, func, options) {
static async p_urlsValidFor(urls, func, opts) {
// Need a async version of this for serviceworker and TransportsProxy
return this.validFor(urls, func, options).map((ut) => ut[0]);
return this.validFor(urls, func, opts).map((ut) => ut[0]);
}
// SEE-OTHER-ADDTRANSPORT
@ -181,6 +181,7 @@ class Transports {
start, integer - first byte wanted
end integer - last byte wanted (note this is inclusive start=0,end=1023 is 1024 bytes
timeoutMS integer - max time to wait on transports (IPFS) that support it
noCache bool - Skip caching (passed to Transports)
}
returns: string - arbitrary bytes retrieved.
throws: TransportError with concatenated error messages if none succeed.
@ -189,7 +190,7 @@ class Transports {
if (!urls.length) throw new errors.TransportError("Transports.p_rawfetch given an empty list of urls");
let resolvedurls = await this.p_resolveNames(urls); // If naming is loaded then convert name to [urls]
if (!resolvedurls.length) throw new errors.TransportError("Transports.p_rawfetch none of the urls resolved: " + urls);
let tt = this.validFor(resolvedurls, "fetch"); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
let tt = this.validFor(resolvedurls, "fetch", {noCache: opts.noCache}); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
if (!tt.length) {
throw new errors.TransportError("Transports.p_rawfetch cant find any transport for urls: " + resolvedurls);
}

File diff suppressed because one or more lines are too long

View File

@ -105,18 +105,20 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
wantstream, // Return a stream rather than data
retries=12, // How many times to retry
noCache // Add Cache-Control: no-cache header
}
returns result via promise or cb(err, result)
*/
if (typeof opts === "function") { cb = opts; opts = {}; }
let headers = new Headers();
if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`);
//if (opts.noCache) headers.append("Cache-Control", "no-cache"); It complains about preflight with no-cache
const retries = typeof opts.retries === "undefined" ? 12 : opts.retries;
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method: 'GET',
headers: headers,
mode: 'cors',
cache: 'default',
cache: opts.noCache ? 'no-cache' : 'default', // In Chrome, This will set Cache-Control: max-age=0
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};