dweb-transports/httptools.js

142 lines
6.9 KiB
JavaScript
Raw Normal View History

const nodefetch = require('node-fetch'); // Note, were using node-fetch-npm which had a warning in webpack see https://github.com/bitinn/node-fetch/issues/421 and is intended for clients
const errors = require('./Errors'); // Standard Dweb Errors
const debught = require('debug')('dweb-transports:httptools');
//var fetch,Headers,Request;
//if (typeof(Window) === "undefined") {
if (typeof(fetch) === "undefined") {
//var fetch = require('whatwg-fetch').fetch; //Not as good as node-fetch-npm, but might be the polyfill needed for browser.safari
//XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest; // Note this doesnt work if set to a var or const, needed by whatwg-fetch
fetch = nodefetch;
Headers = fetch.Headers; // A class
Request = fetch.Request; // A class
} /* else {
// If on a browser, need to find fetch,Headers,Request in window
console.log("Loading browser version of fetch,Headers,Request");
fetch = window.fetch;
Headers = window.Headers;
Request = window.Request;
} */
//TODO-HTTP to work on Safari or mobile will require a polyfill, see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch for comment
httptools = {};
async function loopfetch(req, ms, count, what) {
/*
A workaround for a nasty Chrome issue which fails if there is a (cross-origin?) fetch of more than 6 files. See other WORKAROUND-CHROME-CROSSORIGINFETCH
Loops at longer and longer intervals trying
req: Request
ms: Initial wait between polls
count: Max number of times to try (0 means just once)
what: Name of what retrieving for log (usually file name or URL)
returns Response:
*/
let lasterr;
2018-06-26 18:46:28 -07:00
let loopguard = (typeof window != "undefined") && window.loopguard; // Optional global parameter, will cancel any loops if changes
while (count-- && (loopguard === ((typeof window != "undefined") && window.loopguard)) ) {
try {
return await fetch(req);
} catch(err) {
lasterr = err;
debught("Delaying %s by %d ms because %s", what, ms, err.message);
await new Promise(resolve => {setTimeout(() => { resolve(); },ms)})
ms = ms*(1+Math.random()); // Spread out delays incase all requesting same time
}
}
console.warn("loopfetch of",what,"failed");
if (loopguard !== ((typeof window != "undefined") && window.loopguard)) {
debught("Looping exited because of page change %s", what);
throw new Error("Looping exited because of page change "+ what)
} else {
throw(lasterr);
}
}
httptools.p_httpfetch = async function(httpurl, init, {wantstream=false}={}) { // Embrace and extend "fetch" to check result etc.
/*
Fetch a url
httpurl: optional (depends on command)
init: {headers}
resolves to: data as text or json depending on Content-Type header
throws: TransportError if fails to fetch
//TODO explicitly parameterise if want it to loop
*/
try {
2018-10-13 21:29:10 +11:00
debught("p_httpfetch: %s %o", httpurl, init.headers.get("range"));
//console.log('CTX=',init["headers"].get('Content-Type'))
// Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.
let req = new Request(httpurl, init);
//let response = await fetch(req);
let response = await loopfetch(req, 500, 12, "fetching "+httpurl);
// fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)
// Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
if (response.ok) {
let contenttype = response.headers.get('Content-Type');
if (wantstream) {
return response.body; // Note property while json() or text() are functions
} else if (contenttype === "application/json") {
return response.json(); // promise resolving to JSON
2018-10-22 17:47:11 -07:00
} else if ((contenttype !== "undefined") && contenttype.startsWith("text")) { // Note in particular this is used for responses to store
return response.text();
} else { // Typically application/octetStream when don't know what fetching
return new Buffer(await response.arrayBuffer()); // Convert arrayBuffer to Buffer which is much more usable currently
}
}
// noinspection ExceptionCaughtLocallyJS
throw new errors.TransportError(`Transport Error ${response.status}: ${response.statusText}`);
} catch (err) {
// Error here is particularly unhelpful - if rejected during the COrs process it throws a TypeError
debught("p_httpfetch failed: %s", err.message); // note TypeErrors are generated by CORS or the Chrome anti DDOS 'feature' should catch them here and comment
if (err instanceof errors.TransportError) {
throw err;
} else {
throw new errors.TransportError(`Transport error thrown by ${httpurl}: ${err.message}`);
}
}
}
httptools.p_GET = function(httpurl, opts={}) {
/* Locate and return a block, based on its url
Throws TransportError if fails
opts {
start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
wantstream, // Return a stream rather than data
}
resolves to: URL that can be used to fetch the resource, of form contenthash:/contenthash/Q123
*/
let headers = new Headers();
2018-10-13 21:29:10 +11:00
if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${(opts.end<Infinity) ? opts.end : ""}`);
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method: 'GET',
headers: headers,
mode: 'cors',
cache: 'default',
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};
return httptools.p_httpfetch(httpurl, init, {wantstream: opts.wantstream}); // This s a real http url
}
httptools.p_POST = function(httpurl, type, data) {
// Locate and return a block, based on its url
// Throws TransportError if fails
//let headers = new window.Headers();
//headers.set('content-type',type); Doesn't work, it ignores it
let init = {
//https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
//https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set
method: 'POST',
headers: {}, //headers,
//body: new Buffer(data),
body: data,
mode: 'cors',
cache: 'default',
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};
return httptools.p_httpfetch(httpurl, init);
}
exports = module.exports = httptools;