merged (version 0.1.39)

This commit is contained in:
Rodney Witcher 2019-04-12 12:36:52 -07:00
commit bd846af057
7 changed files with 428 additions and 46 deletions

13
API.md
View File

@ -441,20 +441,23 @@ returns: Depends on mime type;
If text/* returns text If text/* returns text
Oherwise Buffer Oherwise Buffer
##### p_GET(url, {start, end}) ##### p_GET(url, {start, end, retries})
Shortcut to do a HTTP/POST get, sets `mode: cors, redirect: follow, keepalive: true, cache: default` Shortcut to do a HTTP/POST get, sets `mode: cors, redirect: follow, keepalive: true, cache: default`
start: First byte to retrieve start: First byte to retrieve
end: Last byte to retrieve (undefined means end of file) end: Last byte to retrieve (undefined means end of file)
wantstream: Return a stream rather than data
retries: How may times to retry if fails at the network layer (i.e. 404 is a success)
Note that it passes start and end as the Range header, most servers support it, Note that it passes start and end as the Range header, most servers support it,
but it does not (yet) explicitly check the result. but it does not (yet) explicitly check the result.
##### p_POST(url, type, data) ##### p_POST(url, type, data, {retries})
Shortcut to do a HTTP/HTTPS POST. sets same options as p_GET Shortcut to do a HTTP/HTTPS POST. sets same options as p_GET
data: Data to send to fetch, typically the body, data: Data to send to fetch, typically the body,
type: Currently not passed as header{Content-type} because fetch appears to ignore it. contenttype: Currently not passed as header{Content-type} because fetch appears to ignore it.
retries: How may times to retry if fails at the network layer (i.e. 404 is a success)
## TransportHTTP ## TransportHTTP

View File

@ -132,3 +132,7 @@ See [Dweb document index](./DOCUMENTINDEX.md) for a list of the repos that make
* 0.1.37: IPFS - dont stop it if we didnt start it (were stopping via API) * 0.1.37: IPFS - dont stop it if we didnt start it (were stopping via API)
* 0.1.37: Start move to unpromisify pattern v5 * 0.1.37: Start move to unpromisify pattern v5
* 0.1.37: IPFS - updated to (significant) v0.34.0 API changes * 0.1.37: IPFS - updated to (significant) v0.34.0 API changes
* 0.1.38: httptools - adds retries
* 0.1.38: WOLK - added to the library
* 0.1.39: WOLK - updated wolk.js module to fix bugs

View File

@ -88,7 +88,7 @@ class TransportHTTP extends Transport {
Fetch from underlying transport, Fetch from underlying transport,
Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have
url: Of resource - which is turned into the HTTP url in p_httpfetch url: Of resource - which is turned into the HTTP url in p_httpfetch
opts: {start, end} see p_GET for documentation opts: {start, end, retries} see p_GET for documentation
throws: TransportError if fails throws: TransportError if fails
*/ */
//if (!(url && url.includes(':') )) //if (!(url && url.includes(':') ))

View File

@ -58,10 +58,10 @@ class TransportWOLK extends Transport {
.then( async () => { //TODO-WOLK check - I'm just not familiar with this construct - an async function inside a .then .then( async () => { //TODO-WOLK check - I'm just not familiar with this construct - an async function inside a .then
if( this.wolk.ecdsaKey == undefined || this.wolk.ecdsaKey == null ) { if( this.wolk.ecdsaKey == undefined || this.wolk.ecdsaKey == null ) {
var wolkName = "user" + Math.floor((Math.random() * 1000) + 1); var wolkName = "user" + Math.floor((Math.random() * 1000) + 1);
debug("WOLK: createAccount because ecdsaKey null") debug("createAccount because ecdsaKey null")
return await this.wolk.createAccount(wolkName) return await this.wolk.createAccount(wolkName)
.then( hash => { .then( hash => {
debug("[WOLK] Account Created: [" + wolkName + "] hash: " + hash + " KEY: " + this.wolk.ecdsaKey) debug("Account Created: [" + wolkName + "] hash: " + hash + " KEY: " + this.wolk.ecdsaKey)
}) })
.catch( err => { .catch( err => {
throw new Error("Error Creating Account: " + err); throw new Error("Error Creating Account: " + err);

File diff suppressed because one or more lines are too long

View File

@ -53,7 +53,7 @@ async function loopfetch(req, ms, count, what) {
} }
} }
httptools.p_httpfetch = async function(httpurl, init, {wantstream=false}={}) { // Embrace and extend "fetch" to check result etc. httptools.p_httpfetch = async function(httpurl, init, {wantstream=false, retries=undefined}={}) { // Embrace and extend "fetch" to check result etc.
/* /*
Fetch a url Fetch a url
@ -70,7 +70,7 @@ httptools.p_httpfetch = async function(httpurl, init, {wantstream=false}={}) { /
// Using window.fetch, because it doesn't appear to be in scope otherwise in the browser. // Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.
let req = new Request(httpurl, init); let req = new Request(httpurl, init);
//let response = await fetch(req); //let response = await fetch(req);
let response = await loopfetch(req, 500, (init.method === "GET") ? ( init.count || 12) : 1, "fetching "+httpurl); let response = await loopfetch(req, 500, retries, "fetching "+httpurl);
// fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch) // fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)
// Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer. // Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
if (response.ok) { if (response.ok) {
@ -104,12 +104,14 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
opts { opts {
start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
wantstream, // Return a stream rather than data wantstream, // Return a stream rather than data
retries=12, // How many times to retry
} }
returns result via promise or cb(err, result) returns result via promise or cb(err, result)
*/ */
if (typeof opts === "function") { cb = opts; opts = {}; } if (typeof opts === "function") { cb = opts; opts = {}; }
let headers = new Headers(); let headers = new Headers();
if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`); if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`);
const retries = typeof opts.retries === "undefined" ? 12 : opts.retries;
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method: 'GET', method: 'GET',
headers: headers, headers: headers,
@ -118,20 +120,21 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
redirect: 'follow', // Chrome defaults to manual redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot keepalive: true // Keep alive - mostly we'll be going back to same places a lot
}; };
const prom = httptools.p_httpfetch(httpurl, init, {wantstream: opts.wantstream}); // This s a real http url const prom = httptools.p_httpfetch(httpurl, init, {retries, wantstream: opts.wantstream}); // This s a real http url
//if (cb) { prom.then((res)=>cb(null,res)).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v3 //if (cb) { prom.then((res)=>cb(null,res)).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v3
//if (cb) { prom.catch((err) => cb(err)).then((res)=>cb(null,res)).catch((err) => debug("Uncaught error %O",err)); } else { return prom; } // Unpromisify pattern v4 //if (cb) { prom.catch((err) => cb(err)).then((res)=>cb(null,res)).catch((err) => debug("Uncaught error %O",err)); } else { return prom; } // Unpromisify pattern v4
if (cb) { prom.then((res)=>{ try { cb(null,res)} catch(err) { debug("Uncaught error %O",err)}}).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v5 if (cb) { prom.then((res)=>{ try { cb(null,res)} catch(err) { debug("Uncaught error %O",err)}}).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v5
} }
httptools.p_POST = function(httpurl, opts={}, cb) { //TODO-API rearranged and addded cb httptools.p_POST = function(httpurl, opts={}, cb) { //TODO-API rearranged and addded cb
/* Locate and return a block, based on its url /* Locate and return a block, based on its url
opts = { data, contenttype } opts = { data, contenttype, retries }
returns result via promise or cb(err, result) returns result via promise or cb(err, result)
*/ */
// Throws TransportError if fails // Throws TransportError if fails
//let headers = new window.Headers(); //let headers = new window.Headers();
//headers.set('content-type',type); Doesn't work, it ignores it //headers.set('content-type',type); Doesn't work, it ignores it
if (typeof opts === "function") { cb = opts; opts = {}; } if (typeof opts === "function") { cb = opts; opts = {}; }
const retries = typeof opts.retries === "undefined" ? 0 : opts.retries;
let init = { let init = {
//https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
//https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set //https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set
@ -145,7 +148,7 @@ httptools.p_POST = function(httpurl, opts={}, cb) { //TODO-API rearranged and ad
keepalive: false // Keep alive - mostly we'll be going back to same places a lot keepalive: false // Keep alive - mostly we'll be going back to same places a lot
}; };
if (opts.contenttype) init.headers["Content-Type"] = opts.contenttype; if (opts.contenttype) init.headers["Content-Type"] = opts.contenttype;
const prom = httptools.p_httpfetch(httpurl, init); const prom = httptools.p_httpfetch(httpurl, init, {retries});
if (cb) { prom.then((res)=>cb(null,res)).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v3 if (cb) { prom.then((res)=>cb(null,res)).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v3
} }

View File

@ -11,7 +11,6 @@
"url": "https://github.com/internetarchive/dweb-transports/issues" "url": "https://github.com/internetarchive/dweb-transports/issues"
}, },
"dependencies": { "dependencies": {
"wolkjs": "git://github.com/wolkdb/wolkjs.git#master",
"canonical-json": "latest", "canonical-json": "latest",
"cids": "^0.5.7", "cids": "^0.5.7",
"gun": "^0.9.9999991", "gun": "^0.9.9999991",
@ -22,6 +21,7 @@
"readable-stream": "^3.1.1", "readable-stream": "^3.1.1",
"webpack": "^4.29.3", "webpack": "^4.29.3",
"webtorrent": "^0.103.0", "webtorrent": "^0.103.0",
"wolkjs": "git://github.com/wolkdb/wolkjs.git#master",
"y-array": "^10.1.4", "y-array": "^10.1.4",
"y-indexeddb": "^8.1.9", "y-indexeddb": "^8.1.9",
"y-ipfs-connector": "^2.3.0", "y-ipfs-connector": "^2.3.0",
@ -52,5 +52,5 @@
"test": "cd src; node ./test.js", "test": "cd src; node ./test.js",
"help": "echo 'test (test it)'; echo 'build (creates dweb-transports-bundle)'" "help": "echo 'test (test it)'; echo 'build (creates dweb-transports-bundle)'"
}, },
"version": "0.1.37" "version": "0.1.39"
} }