mirror of
https://github.com/fluencelabs/dweb-transports
synced 2025-03-15 18:30:49 +00:00
Support for noCache (reload) partial
This commit is contained in:
parent
38ca1c92af
commit
82890eab97
16
API.md
16
API.md
@ -317,7 +317,7 @@ Finds an array or Transports that are STARTED and can support this URL.
|
||||
urls: Array of urls
|
||||
func: Function to check support for: fetch, store, add, list, listmonitor, reverse
|
||||
- see supportFunctions on each Transport class
|
||||
options For future use
|
||||
options checks supportFeatures
|
||||
Returns: Array of pairs of url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
|
||||
```
|
||||
|
||||
@ -497,7 +497,7 @@ Option|Default|Meaning
|
||||
urlbase|https://dweb.me:443|Connect to dweb.me for contenthash urls
|
||||
supportURLS|http:*, https:*, contenthash:*}| (TODO: may in the future support `dweb:/contenthash/*`)
|
||||
supportFunctions|fetch, store, add, list, reverse, newlisturls, get, set, keys, getall, delete, newtable, newdatabase|
|
||||
supportFeatures|fetch.range|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
|
||||
supportFeatures|fetch.range, noCache|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
|
||||
|
||||
|
||||
## TransportIPFS
|
||||
@ -525,7 +525,8 @@ supportURLS = `ipfs:*` (TODO: may in the future support `dweb:/ipfs/*`)
|
||||
SupportFunctions (note YJS uses IPFS and supports some other functions):
|
||||
`fetch, store`
|
||||
SupportFeatures:
|
||||
fetch.range Not supported (currently April 2018))
|
||||
fetch.range Not supported (currently May 2019))
|
||||
noCache: Not actually supported, but immutable
|
||||
|
||||
Currently there is code for p_f_createReadStream. It works but because IPFS cannot return an error even if it
|
||||
cannot open the stream, IPFS is usually set as the last choice transport for streams.
|
||||
@ -538,8 +539,9 @@ supportURLS = `yjs:*` (TODO: may in the future support `dweb:/yjs/*`)
|
||||
supportFunctions (note YJS uses IPFS and supports some other functions):
|
||||
`fetch, add, list, listmonitor, newlisturls, connection, get, set, getall, keys, newdatabase, newtable, monitor`
|
||||
supportFeatures:
|
||||
fetch.range Not supported (currently April 2018)
|
||||
|
||||
fetch.range Not supported (currently May 2019)
|
||||
noCache: Not supported (currently May 2019)
|
||||
|
||||
## TransportWEBTORRENT
|
||||
A subclass of Transport for handling WEBTORRENT connections (similar to, with interworking with BitTorrent)
|
||||
|
||||
@ -553,7 +555,8 @@ supportFunctions:
|
||||
`fetch`, `createReadStream`
|
||||
|
||||
supportFeatures:
|
||||
fetch.range Not supported (currently April 2018)
|
||||
fetch.range Not supported (currently May 2019)
|
||||
noCache: Not actually supported, but immutable
|
||||
|
||||
## TransportGUN
|
||||
A subclass of Transport for handling GUN connections (decentralized database)
|
||||
@ -562,6 +565,7 @@ supportURLS = `gun:*` (TODO: may in the future support `dweb:/gun/*`)
|
||||
|
||||
supportFunctions = `add`, `list`, `listmonitor`, `newlisturls`, `connection`, `get`, `set`, `getall`, `keys`, `newdatabase`, `newtable`, `monitor`
|
||||
supportFeatures:
|
||||
noCache: Not supported (and cache flushing is hard)
|
||||
|
||||
## TransportWOLK
|
||||
A subclass of Transport for handling the WOLK transport layer (decentralized, block chain based, incentivised storage)
|
||||
|
11
Transport.js
11
Transport.js
@ -96,11 +96,12 @@ class Transport {
|
||||
// True if connected (status==STATUS_CONNECTED==0) should not need subclassing
|
||||
return ! this.status;
|
||||
}
|
||||
supports(url, func) { //TODO-API
|
||||
supports(url, func, {noCache=undefined}={}) { //TODO-API
|
||||
/*
|
||||
Determine if this transport supports a certain set of URLs and a func
|
||||
|
||||
:param url: String or parsed URL
|
||||
:param opts: { noCache } check against supportFeatures
|
||||
:return: true if this protocol supports these URLs and this func
|
||||
:throw: TransportError if invalid URL
|
||||
*/
|
||||
@ -112,13 +113,15 @@ class Transport {
|
||||
} //Should be TransportError but out of scope here
|
||||
// noinspection Annotator supportURLs is defined in subclasses
|
||||
return ( (!url || this.supportURLs.includes(url.protocol.slice(0, -1)))
|
||||
&& (!func || this.supportFunctions.includes(func)))
|
||||
&& (!func || this.supportFunctions.includes(func))
|
||||
&& (!noCache || this.supportFeatures.includes("noCache"))
|
||||
)
|
||||
}
|
||||
|
||||
validFor(url, func) { //TODO-API
|
||||
validFor(url, func, opts) { //TODO-API
|
||||
// By default a transport can handle a url and a func if its connected and supports that url/func
|
||||
// This shouldnt need subclassing, an exception is HTTP which only applies "connected" against urls heading for the gateway
|
||||
return this.connected() && this.supports(url, func);
|
||||
return this.connected() && this.supports(url, func, opts);
|
||||
}
|
||||
|
||||
|
||||
|
@ -73,6 +73,7 @@ class TransportGUN extends Transport {
|
||||
this.supportFunctions = [ 'fetch', //'store'
|
||||
'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor',
|
||||
'add', 'list', 'listmonitor', 'newlisturls'];
|
||||
this.supportFeatures = []; // Doesnt support noCache and is mutable
|
||||
this.status = Transport.STATUS_LOADED;
|
||||
}
|
||||
|
||||
|
@ -32,12 +32,13 @@ class TransportHTTP extends Transport {
|
||||
this.urlbase = options.urlbase; // e.g. https://dweb.me
|
||||
this.supportURLs = ['contenthash', 'http','https'];
|
||||
this.supportFunctions = ['fetch', 'store', 'add', 'list', 'reverse', 'newlisturls', "get", "set", "keys", "getall", "delete", "newtable", "newdatabase"]; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller
|
||||
this.supportFeatures = ['noCache'];
|
||||
if (typeof window === "undefined") {
|
||||
// running in node, can support createReadStream, (browser can't - see createReadStream below)
|
||||
this.supportFunctions.push("createReadStream");
|
||||
}
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
this.supportFeatures = ['fetch.range'];
|
||||
this.supportFeatures = ['fetch.range', 'noCache'];
|
||||
this.name = "HTTP"; // For console log etc
|
||||
this.status = Transport.STATUS_LOADED;
|
||||
}
|
||||
@ -87,10 +88,10 @@ class TransportHTTP extends Transport {
|
||||
return url;
|
||||
}
|
||||
|
||||
validFor(url, func) {
|
||||
validFor(url, func, opts) {
|
||||
// Overrides Transport.prototype.validFor because HTTP's connection test is only really for dweb.me
|
||||
// in particular this allows urls like https://be-api.us.archive.org
|
||||
return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func);
|
||||
return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func, opts);
|
||||
}
|
||||
// noinspection JSCheckFunctionSignatures
|
||||
async p_rawfetch(url, opts={}) {
|
||||
@ -98,7 +99,7 @@ class TransportHTTP extends Transport {
|
||||
Fetch from underlying transport,
|
||||
Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have
|
||||
url: Of resource - which is turned into the HTTP url in p_httpfetch
|
||||
opts: {start, end, retries} see p_GET for documentation
|
||||
opts: {start, end, retries, noCache} see p_GET for documentation
|
||||
throws: TransportError if fails
|
||||
*/
|
||||
//if (!(url && url.includes(':') ))
|
||||
|
@ -69,6 +69,7 @@ class TransportIPFS extends Transport {
|
||||
this.name = "IPFS"; // For console log etc
|
||||
this.supportURLs = ['ipfs'];
|
||||
this.supportFunctions = ['fetch', 'store', 'seed', 'createReadStream']; // Does not support reverse
|
||||
this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
|
||||
this.status = Transport.STATUS_LOADED;
|
||||
}
|
||||
|
||||
|
@ -35,6 +35,7 @@ class TransportWEBTORRENT extends Transport {
|
||||
this.name = "WEBTORRENT"; // For console log etc
|
||||
this.supportURLs = ['magnet'];
|
||||
this.supportFunctions = ['fetch', 'createReadStream', "seed"];
|
||||
this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
|
||||
this.status = Transport.STATUS_LOADED;
|
||||
}
|
||||
|
||||
|
@ -30,6 +30,8 @@ class TransportWOLK extends Transport {
|
||||
this.name = "WOLK"; // For console log etc
|
||||
this.supportURLs = ['wolk'];
|
||||
this.supportFunctions = [ 'fetch', 'connection', 'get', 'set', ]; // 'store' - requires chunkdata; 'createReadStream' not implemented
|
||||
this.supportFeatures = []; // Doesnt support noCache and is mutable
|
||||
|
||||
this.status = Transport.STATUS_LOADED;
|
||||
}
|
||||
|
||||
|
@ -50,6 +50,7 @@ class TransportYJS extends Transport {
|
||||
this.supportURLs = ['yjs'];
|
||||
this.supportFunctions = ['fetch', 'add', 'list', 'listmonitor', 'newlisturls',
|
||||
'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor']; // Only does list functions, Does not support reverse,
|
||||
this.supportFeatures = []; // Doesnt support noCache and is mutable
|
||||
this.status = Transport.STATUS_LOADED;
|
||||
}
|
||||
|
||||
|
@ -44,7 +44,7 @@ class Transports {
|
||||
const res = this._transports.map((t) => { return {"name": t.name, "status": t.status}})
|
||||
if (cb) { cb(null, res)} else { return new Promise((resolve, reject) => resolve(res))}
|
||||
}
|
||||
static validFor(urls, func, options) {
|
||||
static validFor(urls, func, opts) { //TODO-RELOAD check for noCache support
|
||||
/*
|
||||
Finds an array or Transports that can support this URL.
|
||||
|
||||
@ -52,7 +52,7 @@ class Transports {
|
||||
|
||||
urls: Array of urls
|
||||
func: Function to check support for: fetch, store, add, list, listmonitor, reverse - see supportFunctions on each Transport class
|
||||
options: For future use
|
||||
opts: Passed to each Transport, esp for supportFeatures
|
||||
returns: Array of pairs of Url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
|
||||
throws: CodingError if urls empty or [undefined...]
|
||||
*/
|
||||
@ -62,19 +62,19 @@ class Transports {
|
||||
return [];
|
||||
}
|
||||
if (!(urls && urls.length > 0)) { // No url supplied we are just checking which transports support this function on no url.
|
||||
return this._transports.filter((t) => (t.validFor(undefined, func)))
|
||||
return this._transports.filter((t) => (t.validFor(undefined, func, opts)))
|
||||
.map((t) => [undefined, t]);
|
||||
} else {
|
||||
return [].concat(
|
||||
...urls.map((url) => typeof url === 'string' ? Url.parse(url) : url) // parse URLs once
|
||||
.map((url) =>
|
||||
this._transports.filter((t) => (t.validFor(url, func))) // [ t1, t2 ]
|
||||
this._transports.filter((t) => (t.validFor(url, func, opts))) // [ t1, t2 ]
|
||||
.map((t) => [url, t]))); // [[ u, t1], [u, t2]]
|
||||
}
|
||||
}
|
||||
static async p_urlsValidFor(urls, func, options) {
|
||||
static async p_urlsValidFor(urls, func, opts) {
|
||||
// Need a async version of this for serviceworker and TransportsProxy
|
||||
return this.validFor(urls, func, options).map((ut) => ut[0]);
|
||||
return this.validFor(urls, func, opts).map((ut) => ut[0]);
|
||||
}
|
||||
|
||||
// SEE-OTHER-ADDTRANSPORT
|
||||
@ -181,6 +181,7 @@ class Transports {
|
||||
start, integer - first byte wanted
|
||||
end integer - last byte wanted (note this is inclusive start=0,end=1023 is 1024 bytes
|
||||
timeoutMS integer - max time to wait on transports (IPFS) that support it
|
||||
noCache bool - Skip caching (passed to Transports)
|
||||
}
|
||||
returns: string - arbitrary bytes retrieved.
|
||||
throws: TransportError with concatenated error messages if none succeed.
|
||||
@ -189,7 +190,7 @@ class Transports {
|
||||
if (!urls.length) throw new errors.TransportError("Transports.p_rawfetch given an empty list of urls");
|
||||
let resolvedurls = await this.p_resolveNames(urls); // If naming is loaded then convert name to [urls]
|
||||
if (!resolvedurls.length) throw new errors.TransportError("Transports.p_rawfetch none of the urls resolved: " + urls);
|
||||
let tt = this.validFor(resolvedurls, "fetch"); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
|
||||
let tt = this.validFor(resolvedurls, "fetch", {noCache: opts.noCache}); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
|
||||
if (!tt.length) {
|
||||
throw new errors.TransportError("Transports.p_rawfetch cant find any transport for urls: " + resolvedurls);
|
||||
}
|
||||
|
23453
dist/dweb-transports-bundle.js
vendored
23453
dist/dweb-transports-bundle.js
vendored
File diff suppressed because one or more lines are too long
@ -105,18 +105,20 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
|
||||
start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
|
||||
wantstream, // Return a stream rather than data
|
||||
retries=12, // How many times to retry
|
||||
noCache // Add Cache-Control: no-cache header
|
||||
}
|
||||
returns result via promise or cb(err, result)
|
||||
*/
|
||||
if (typeof opts === "function") { cb = opts; opts = {}; }
|
||||
let headers = new Headers();
|
||||
if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`);
|
||||
//if (opts.noCache) headers.append("Cache-Control", "no-cache"); It complains about preflight with no-cache
|
||||
const retries = typeof opts.retries === "undefined" ? 12 : opts.retries;
|
||||
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
|
||||
method: 'GET',
|
||||
headers: headers,
|
||||
mode: 'cors',
|
||||
cache: 'default',
|
||||
cache: opts.noCache ? 'no-cache' : 'default', // In Chrome, This will set Cache-Control: max-age=0
|
||||
redirect: 'follow', // Chrome defaults to manual
|
||||
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
|
||||
};
|
||||
|
Loading…
x
Reference in New Issue
Block a user