merged (version 0.1.39)

API.md

@@ -441,20 +441,23 @@ returns:    Depends on mime type;
     If text/* returns text
     Oherwise    Buffer
 
-##### p_GET(url, {start, end})
+##### p_GET(url, {start, end, retries})
 Shortcut to do a HTTP/POST get, sets `mode: cors, redirect: follow, keepalive: true, cache: default`
 
-start:  First byte to retrieve
-end:    Last byte to retrieve (undefined means end of file)
+start:      First byte to retrieve
+end:        Last byte to retrieve (undefined means end of file)
+wantstream: Return a stream rather than data
+retries:    How may times to retry if fails at the network layer (i.e. 404 is a success)
 
 Note that it passes start and end as the Range header, most servers support it, 
 but it does not (yet) explicitly check the result. 
 
-##### p_POST(url, type, data)
+##### p_POST(url, type, data, {retries})
 Shortcut to do a HTTP/HTTPS POST. sets same options as p_GET
 
 data:   Data to send to fetch, typically the body, 
-type:   Currently not passed as header{Content-type} because fetch appears to ignore it.
+contenttype:   Currently not passed as header{Content-type} because fetch appears to ignore it.
+retries: How may times to retry if fails at the network layer (i.e. 404 is a success)
 
 
 ## TransportHTTP

README.md

@@ -132,3 +132,7 @@ See [Dweb document index](./DOCUMENTINDEX.md) for a list of the repos that make
 * 0.1.37: IPFS - dont stop it if we didnt start it (were stopping via API)
 * 0.1.37: Start move to unpromisify pattern v5
 * 0.1.37: IPFS - updated to (significant) v0.34.0 API changes
+* 0.1.38: httptools - adds retries
+* 0.1.38: WOLK - added to the library
+* 0.1.39: WOLK - updated wolk.js module to fix bugs
+

TransportHTTP.js

@@ -88,7 +88,7 @@ class TransportHTTP extends Transport {
         Fetch from underlying transport,
         Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have
         url: Of resource - which is turned into the HTTP url in p_httpfetch
-        opts: {start, end} see p_GET for documentation
+        opts: {start, end, retries} see p_GET for documentation
         throws: TransportError if fails
          */
         //if (!(url && url.includes(':') ))

TransportWOLK.js

@@ -58,10 +58,10 @@ class TransportWOLK extends Transport {
       .then( async () => { //TODO-WOLK check - I'm just not familiar with this construct - an async function inside a .then
         if( this.wolk.ecdsaKey == undefined || this.wolk.ecdsaKey == null ) {
           var wolkName = "user" + Math.floor((Math.random() * 1000) + 1);
-          debug("WOLK: createAccount because ecdsaKey null")
+          debug("createAccount because ecdsaKey null")
           return await this.wolk.createAccount(wolkName)
                   .then( hash => {
-                    debug("[WOLK] Account Created: [" + wolkName + "] hash: " + hash + " KEY: " + this.wolk.ecdsaKey)
+                    debug("Account Created: [" + wolkName + "] hash: " + hash + " KEY: " + this.wolk.ecdsaKey)
                   })
                   .catch( err => {
                     throw new Error("Error Creating Account: " + err);

httptools.js

@@ -53,7 +53,7 @@ async function loopfetch(req, ms, count, what) {
     }
 }
 
-httptools.p_httpfetch = async function(httpurl, init, {wantstream=false}={}) { // Embrace and extend "fetch" to check result etc.
+httptools.p_httpfetch = async function(httpurl, init, {wantstream=false, retries=undefined}={}) { // Embrace and extend "fetch" to check result etc.
     /*
     Fetch a url
 
@@ -70,7 +70,7 @@ httptools.p_httpfetch = async function(httpurl, init, {wantstream=false}={}) { /
         // Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.
         let req = new Request(httpurl, init);
         //let response = await fetch(req);
-        let response = await loopfetch(req, 500, (init.method === "GET") ? ( init.count || 12) : 1, "fetching "+httpurl);
+        let response = await loopfetch(req, 500, retries, "fetching "+httpurl);
         // fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)
         // Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
         if (response.ok) {
@@ -104,12 +104,14 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
         opts {
             start, end,     // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
             wantstream,     // Return a stream rather than data
+            retries=12,        // How many times to retry
             }
         returns result via promise or cb(err, result)
     */
     if (typeof opts  === "function") { cb = opts; opts = {}; }
     let headers = new Headers();
     if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`);
+    const retries = typeof opts.retries === "undefined" ? 12 : opts.retries;
     let init = {    //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
         method: 'GET',
         headers: headers,
@@ -118,20 +120,21 @@ httptools.p_GET = function(httpurl, opts={}, cb) { //TODO-API rearranged and add
         redirect: 'follow',  // Chrome defaults to manual
         keepalive: true    // Keep alive - mostly we'll be going back to same places a lot
     };
-    const prom = httptools.p_httpfetch(httpurl, init, {wantstream: opts.wantstream}); // This s a real http url
+    const prom = httptools.p_httpfetch(httpurl, init, {retries, wantstream: opts.wantstream}); // This s a real http url
     //if (cb) { prom.then((res)=>cb(null,res)).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v3
     //if (cb) { prom.catch((err) => cb(err)).then((res)=>cb(null,res)).catch((err) => debug("Uncaught error %O",err)); } else { return prom; } // Unpromisify pattern v4
     if (cb) { prom.then((res)=>{ try { cb(null,res)} catch(err) { debug("Uncaught error %O",err)}}).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v5
 }
 httptools.p_POST = function(httpurl, opts={}, cb) { //TODO-API rearranged and addded cb
     /* Locate and return a block, based on its url
-    opts = { data, contenttype }
+    opts = { data, contenttype, retries }
     returns result via promise or cb(err, result)
      */
     // Throws TransportError if fails
     //let headers = new window.Headers();
     //headers.set('content-type',type); Doesn't work, it ignores it
     if (typeof opts  === "function") { cb = opts; opts = {}; }
+    const retries = typeof opts.retries === "undefined" ? 0 : opts.retries;
     let init = {
         //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
         //https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set
@@ -145,7 +148,7 @@ httptools.p_POST = function(httpurl, opts={}, cb) { //TODO-API rearranged and ad
         keepalive: false    // Keep alive - mostly we'll be going back to same places a lot
     };
     if (opts.contenttype) init.headers["Content-Type"] = opts.contenttype;
-    const prom = httptools.p_httpfetch(httpurl, init);
+    const prom = httptools.p_httpfetch(httpurl, init, {retries});
     if (cb) { prom.then((res)=>cb(null,res)).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v3
 }
 

package.json

@@ -11,7 +11,6 @@
     "url": "https://github.com/internetarchive/dweb-transports/issues"
   },
   "dependencies": {
-    "wolkjs": "git://github.com/wolkdb/wolkjs.git#master",
     "canonical-json": "latest",
     "cids": "^0.5.7",
     "gun": "^0.9.9999991",
@@ -22,6 +21,7 @@
     "readable-stream": "^3.1.1",
     "webpack": "^4.29.3",
     "webtorrent": "^0.103.0",
+    "wolkjs": "git://github.com/wolkdb/wolkjs.git#master",
     "y-array": "^10.1.4",
     "y-indexeddb": "^8.1.9",
     "y-ipfs-connector": "^2.3.0",
@@ -52,5 +52,5 @@
     "test": "cd src; node ./test.js",
     "help": "echo 'test (test it)'; echo 'build (creates dweb-transports-bundle)'"
   },
-  "version": "0.1.37"
+  "version": "0.1.39"
 }