fluencelabs/dweb-transports
1
0
Fork 0
mirror of https://github.com/fluencelabs/dweb-transports synced 2025-03-15 10:30:48 +00:00
Code Issues Projects Releases Wiki Activity

Support for noCache (reload) partial

Browse Source
This commit is contained in:
Mitra Ardron 2019-05-30 15:57:37 +10:00
parent 38ca1c92af
commit 82890eab97
11 changed files with 22970 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
API.md
View File

@ -317,7 +317,7 @@ Finds an array or Transports that are STARTED and can support this URL.
urls: Array of urls
func: Function to check support for: fetch, store, add, list, listmonitor, reverse
- see supportFunctions on each Transport class
options For future use
options checks supportFeatures
Returns: Array of pairs of url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
```
@ -497,7 +497,7 @@ Option|Default|Meaning
urlbase|https://dweb.me:443|Connect to dweb.me for contenthash urls
supportURLS|http:*, https:*, contenthash:*}| (TODO: may in the future support `dweb:/contenthash/*`)
supportFunctions|fetch, store, add, list, reverse, newlisturls, get, set, keys, getall, delete, newtable, newdatabase|
supportFeatures|fetch.range|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
supportFeatures|fetch.range, noCache|it will fetch a range of bytes if specified in {start, end} to p_rawfetch()
## TransportIPFS
@ -525,7 +525,8 @@ supportURLS = `ipfs:*` (TODO: may in the future support `dweb:/ipfs/*`)
SupportFunctions (note YJS uses IPFS and supports some other functions):
`fetch, store`
SupportFeatures:
fetch.range Not supported (currently April 2018))
fetch.range Not supported (currently May 2019))
noCache: Not actually supported, but immutable
Currently there is code for p_f_createReadStream. It works but because IPFS cannot return an error even if it
cannot open the stream, IPFS is usually set as the last choice transport for streams.
@ -538,8 +539,9 @@ supportURLS = `yjs:*` (TODO: may in the future support `dweb:/yjs/*`)
supportFunctions (note YJS uses IPFS and supports some other functions):
`fetch, add, list, listmonitor, newlisturls, connection, get, set, getall, keys, newdatabase, newtable, monitor`
supportFeatures:
fetch.range Not supported (currently April 2018)
fetch.range Not supported (currently May 2019)
noCache: Not supported (currently May 2019)
## TransportWEBTORRENT
A subclass of Transport for handling WEBTORRENT connections (similar to, with interworking with BitTorrent)
@ -553,7 +555,8 @@ supportFunctions:
`fetch`, `createReadStream`
supportFeatures:
fetch.range Not supported (currently April 2018)
fetch.range Not supported (currently May 2019)
noCache: Not actually supported, but immutable
## TransportGUN
A subclass of Transport for handling GUN connections (decentralized database)
@ -562,6 +565,7 @@ supportURLS = `gun:*` (TODO: may in the future support `dweb:/gun/*`)
supportFunctions = `add`, `list`, `listmonitor`, `newlisturls`, `connection`, `get`, `set`, `getall`, `keys`, `newdatabase`, `newtable`, `monitor`
supportFeatures:
noCache: Not supported (and cache flushing is hard)
## TransportWOLK
A subclass of Transport for handling the WOLK transport layer (decentralized, block chain based, incentivised storage)

11
Transport.js
View File

@ -96,11 +96,12 @@ class Transport {
// True if connected (status==STATUS_CONNECTED==0) should not need subclassing
return ! this.status;
}
supports(url, func) { //TODO-API
supports(url, func, {noCache=undefined}={}) { //TODO-API
/*
Determine if this transport supports a certain set of URLs and a func
:param url: String or parsed URL
:param opts: { noCache } check against supportFeatures
:return: true if this protocol supports these URLs and this func
:throw: TransportError if invalid URL
*/
@ -112,13 +113,15 @@ class Transport {
} //Should be TransportError but out of scope here
// noinspection Annotator supportURLs is defined in subclasses
return ( (!url || this.supportURLs.includes(url.protocol.slice(0, -1)))
&& (!func || this.supportFunctions.includes(func)))
&& (!func || this.supportFunctions.includes(func))
&& (!noCache || this.supportFeatures.includes("noCache"))
)
}
validFor(url, func) { //TODO-API
validFor(url, func, opts) { //TODO-API
// By default a transport can handle a url and a func if its connected and supports that url/func
// This shouldnt need subclassing, an exception is HTTP which only applies "connected" against urls heading for the gateway
return this.connected() && this.supports(url, func);
return this.connected() && this.supports(url, func, opts);
}

1
TransportGUN.js
View File

@ -73,6 +73,7 @@ class TransportGUN extends Transport {
this.supportFunctions = [ 'fetch', //'store'
'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor',
'add', 'list', 'listmonitor', 'newlisturls'];
this.supportFeatures = []; // Doesnt support noCache and is mutable
this.status = Transport.STATUS_LOADED;
}

9
TransportHTTP.js
View File

@ -32,12 +32,13 @@ class TransportHTTP extends Transport {
this.urlbase = options.urlbase; // e.g. https://dweb.me
this.supportURLs = ['contenthash', 'http','https'];
this.supportFunctions = ['fetch', 'store', 'add', 'list', 'reverse', 'newlisturls', "get", "set", "keys", "getall", "delete", "newtable", "newdatabase"]; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller
this.supportFeatures = ['noCache'];
if (typeof window === "undefined") {
// running in node, can support createReadStream, (browser can't - see createReadStream below)
this.supportFunctions.push("createReadStream");
}
// noinspection JSUnusedGlobalSymbols
this.supportFeatures = ['fetch.range'];
this.supportFeatures = ['fetch.range', 'noCache'];
this.name = "HTTP"; // For console log etc
this.status = Transport.STATUS_LOADED;
}
@ -87,10 +88,10 @@ class TransportHTTP extends Transport {
return url;
}
validFor(url, func) {
validFor(url, func, opts) {
// Overrides Transport.prototype.validFor because HTTP's connection test is only really for dweb.me
// in particular this allows urls like https://be-api.us.archive.org
return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func);
return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func, opts);
}
// noinspection JSCheckFunctionSignatures
async p_rawfetch(url, opts={}) {
@ -98,7 +99,7 @@ class TransportHTTP extends Transport {
Fetch from underlying transport,
Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have
url: Of resource - which is turned into the HTTP url in p_httpfetch
opts: {start, end, retries} see p_GET for documentation
opts: {start, end, retries, noCache} see p_GET for documentation
throws: TransportError if fails
*/
//if (!(url && url.includes(':') ))

1
TransportIPFS.js
View File

@ -69,6 +69,7 @@ class TransportIPFS extends Transport {
this.name = "IPFS"; // For console log etc
this.supportURLs = ['ipfs'];
this.supportFunctions = ['fetch', 'store', 'seed', 'createReadStream']; // Does not support reverse
this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
this.status = Transport.STATUS_LOADED;
}

1
TransportWEBTORRENT.js
View File

@ -35,6 +35,7 @@ class TransportWEBTORRENT extends Transport {
this.name = "WEBTORRENT"; // For console log etc
this.supportURLs = ['magnet'];
this.supportFunctions = ['fetch', 'createReadStream', "seed"];
this.supportFeatures = ['noCache']; // Note doesnt actually support noCache, but immutable is same
this.status = Transport.STATUS_LOADED;
}

2
TransportWOLK.js
View File

@ -30,6 +30,8 @@ class TransportWOLK extends Transport {
this.name = "WOLK"; // For console log etc
this.supportURLs = ['wolk'];
this.supportFunctions = [ 'fetch', 'connection', 'get', 'set', ]; // 'store' - requires chunkdata; 'createReadStream' not implemented
this.supportFeatures = []; // Doesnt support noCache and is mutable
this.status = Transport.STATUS_LOADED;
}

1
TransportYJS.js
View File

@ -50,6 +50,7 @@ class TransportYJS extends Transport {
this.supportURLs = ['yjs'];
this.supportFunctions = ['fetch', 'add', 'list', 'listmonitor', 'newlisturls',
'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor']; // Only does list functions, Does not support reverse,
this.supportFeatures = []; // Doesnt support noCache and is mutable
this.status = Transport.STATUS_LOADED;
}

15
Transports.js
View File

@ -44,7 +44,7 @@ class Transports {
const res = this._transports.map((t) => { return {"name": t.name, "status": t.status}})
if (cb) { cb(null, res)} else { return new Promise((resolve, reject) => resolve(res))}
}
static validFor(urls, func, options) {
static validFor(urls, func, opts) { //TODO-RELOAD check for noCache support
/*
Finds an array or Transports that can support this URL.
@ -52,7 +52,7 @@ class Transports {
urls: Array of urls
func: Function to check support for: fetch, store, add, list, listmonitor, reverse - see supportFunctions on each Transport class
options: For future use
opts: Passed to each Transport, esp for supportFeatures
returns: Array of pairs of Url & transport instance [ [ u1, t1], [u1, t2], [u2, t1]]
throws: CodingError if urls empty or [undefined...]
*/
@ -62,19 +62,19 @@ class Transports {
return [];
}
if (!(urls && urls.length > 0)) { // No url supplied we are just checking which transports support this function on no url.
return this._transports.filter((t) => (t.validFor(undefined, func)))
return this._transports.filter((t) => (t.validFor(undefined, func, opts)))
.map((t) => [undefined, t]);
} else {
return [].concat(
...urls.map((url) => typeof url === 'string' ? Url.parse(url) : url) // parse URLs once
.map((url) =>
this._transports.filter((t) => (t.validFor(url, func))) // [ t1, t2 ]
this._transports.filter((t) => (t.validFor(url, func, opts))) // [ t1, t2 ]
.map((t) => [url, t]))); // [[ u, t1], [u, t2]]
}
}
static async p_urlsValidFor(urls, func, options) {
static async p_urlsValidFor(urls, func, opts) {
// Need a async version of this for serviceworker and TransportsProxy
return this.validFor(urls, func, options).map((ut) => ut[0]);
return this.validFor(urls, func, opts).map((ut) => ut[0]);
}
// SEE-OTHER-ADDTRANSPORT
@ -181,6 +181,7 @@ class Transports {
start, integer - first byte wanted
end integer - last byte wanted (note this is inclusive start=0,end=1023 is 1024 bytes
timeoutMS integer - max time to wait on transports (IPFS) that support it
noCache bool - Skip caching (passed to Transports)
}
returns: string - arbitrary bytes retrieved.
throws: TransportError with concatenated error messages if none succeed.
@ -189,7 +190,7 @@ class Transports {
if (!urls.length) throw new errors.TransportError("Transports.p_rawfetch given an empty list of urls");
let resolvedurls = await this.p_resolveNames(urls); // If naming is loaded then convert name to [urls]
if (!resolvedurls.length) throw new errors.TransportError("Transports.p_rawfetch none of the urls resolved: " + urls);
let tt = this.validFor(resolvedurls, "fetch"); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
let tt = this.validFor(resolvedurls, "fetch", {noCache: opts.noCache}); //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
if (!tt.length) {
throw new errors.TransportError("Transports.p_rawfetch cant find any transport for urls: " + resolvedurls);
}

23453
dist/dweb-transports-bundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
httptools.js
View File

@ -105,18 +105,20 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
wantstream, // Return a stream rather than data
retries=12, // How many times to retry
noCache // Add Cache-Control: no-cache header
}
returns result via promise or cb(err, result)
*/
if (typeof opts === "function") { cb = opts; opts = {}; }
let headers = new Headers();
if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`);
//if (opts.noCache) headers.append("Cache-Control", "no-cache"); It complains about preflight with no-cache
const retries = typeof opts.retries === "undefined" ? 12 : opts.retries;
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method: 'GET',
headers: headers,
mode: 'cors',
cache: 'default',
cache: opts.noCache ? 'no-cache' : 'default', // In Chrome, This will set Cache-Control: max-age=0
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};