2018-05-08 16:46:57 -07:00
const nodefetch = require ( 'node-fetch' ) ; // Note, were using node-fetch-npm which had a warning in webpack see https://github.com/bitinn/node-fetch/issues/421 and is intended for clients
const errors = require ( './Errors' ) ; // Standard Dweb Errors
2019-02-22 16:31:17 +11:00
const debug = require ( 'debug' ) ( 'dweb-transports:httptools' ) ;
2018-05-08 16:46:57 -07:00
//var fetch,Headers,Request;
//if (typeof(Window) === "undefined") {
if ( typeof ( fetch ) === "undefined" ) {
//var fetch = require('whatwg-fetch').fetch; //Not as good as node-fetch-npm, but might be the polyfill needed for browser.safari
//XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest; // Note this doesnt work if set to a var or const, needed by whatwg-fetch
fetch = nodefetch ;
Headers = fetch . Headers ; // A class
Request = fetch . Request ; // A class
} / * else {
// If on a browser, need to find fetch,Headers,Request in window
console . log ( "Loading browser version of fetch,Headers,Request" ) ;
fetch = window . fetch ;
Headers = window . Headers ;
Request = window . Request ;
} * /
//TODO-HTTP to work on Safari or mobile will require a polyfill, see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch for comment
httptools = { } ;
2018-06-08 14:39:33 -07:00
async function loopfetch ( req , ms , count , what ) {
2018-07-21 11:08:38 -07:00
/ *
A workaround for a nasty Chrome issue which fails if there is a ( cross - origin ? ) fetch of more than 6 files . See other WORKAROUND - CHROME - CROSSORIGINFETCH
Loops at longer and longer intervals trying
req : Request
ms : Initial wait between polls
count : Max number of times to try ( 0 means just once )
what : Name of what retrieving for log ( usually file name or URL )
returns Response :
* /
2018-06-08 14:39:33 -07:00
let lasterr ;
2018-06-26 18:46:28 -07:00
let loopguard = ( typeof window != "undefined" ) && window . loopguard ; // Optional global parameter, will cancel any loops if changes
while ( count -- && ( loopguard === ( ( typeof window != "undefined" ) && window . loopguard ) ) ) {
2018-06-08 14:39:33 -07:00
try {
return await fetch ( req ) ;
} catch ( err ) {
lasterr = err ;
2019-02-22 16:31:17 +11:00
debug ( "Delaying %s by %d ms because %s" , what , ms , err . message ) ;
2018-06-08 14:39:33 -07:00
await new Promise ( resolve => { setTimeout ( ( ) => { resolve ( ) ; } , ms ) } )
ms = ms * ( 1 + Math . random ( ) ) ; // Spread out delays incase all requesting same time
}
}
2018-08-13 17:55:04 +10:00
console . warn ( "loopfetch of" , what , "failed" ) ;
2018-10-18 16:17:03 +13:00
if ( loopguard !== ( ( typeof window != "undefined" ) && window . loopguard ) ) {
2019-02-22 16:31:17 +11:00
debug ( "Looping exited because of page change %s" , what ) ;
2018-06-11 20:28:07 -07:00
throw new Error ( "Looping exited because of page change " + what )
} else {
throw ( lasterr ) ;
}
2018-06-08 14:39:33 -07:00
}
2019-03-20 21:20:26 -07:00
httptools . p _httpfetch = async function ( httpurl , init , { wantstream = false , retries = undefined } = { } ) { // Embrace and extend "fetch" to check result etc.
2018-05-08 16:46:57 -07:00
/ *
Fetch a url
2018-07-21 11:08:38 -07:00
httpurl : optional ( depends on command )
init : { headers }
2018-05-08 16:46:57 -07:00
resolves to : data as text or json depending on Content - Type header
throws : TransportError if fails to fetch
2018-10-11 15:45:32 +11:00
//TODO explicitly parameterise if want it to loop
2018-05-08 16:46:57 -07:00
* /
try {
2018-11-11 16:37:03 -08:00
// THis was get("range") but that works when init.headers is a Headers, but not when its an object
2019-02-22 16:31:17 +11:00
debug ( "p_httpfetch: %s %o" , httpurl , init . headers . range || "" ) ;
2018-05-08 16:46:57 -07:00
//console.log('CTX=',init["headers"].get('Content-Type'))
// Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.
2018-06-08 14:39:33 -07:00
let req = new Request ( httpurl , init ) ;
2018-10-12 15:17:58 +11:00
//let response = await fetch(req);
2019-03-20 21:20:26 -07:00
let response = await loopfetch ( req , 500 , retries , "fetching " + httpurl ) ;
2018-05-08 16:46:57 -07:00
// fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)
// Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
if ( response . ok ) {
let contenttype = response . headers . get ( 'Content-Type' ) ;
2018-07-21 11:08:38 -07:00
if ( wantstream ) {
return response . body ; // Note property while json() or text() are functions
2018-11-29 11:32:56 +11:00
} else if ( ( typeof contenttype !== "undefined" ) && contenttype . startsWith ( "application/json" ) ) {
2018-05-08 16:46:57 -07:00
return response . json ( ) ; // promise resolving to JSON
2018-10-28 20:59:11 -07:00
} else if ( ( typeof contenttype !== "undefined" ) && contenttype . startsWith ( "text" ) ) { // Note in particular this is used for responses to store
2018-05-08 16:46:57 -07:00
return response . text ( ) ;
} else { // Typically application/octetStream when don't know what fetching
return new Buffer ( await response . arrayBuffer ( ) ) ; // Convert arrayBuffer to Buffer which is much more usable currently
}
}
// noinspection ExceptionCaughtLocallyJS
throw new errors . TransportError ( ` Transport Error ${ response . status } : ${ response . statusText } ` ) ;
} catch ( err ) {
// Error here is particularly unhelpful - if rejected during the COrs process it throws a TypeError
2019-02-22 16:31:17 +11:00
debug ( "p_httpfetch failed: %s" , err . message ) ; // note TypeErrors are generated by CORS or the Chrome anti DDOS 'feature' should catch them here and comment
2018-05-08 16:46:57 -07:00
if ( err instanceof errors . TransportError ) {
throw err ;
} else {
throw new errors . TransportError ( ` Transport error thrown by ${ httpurl } : ${ err . message } ` ) ;
}
}
}
2019-02-12 16:22:34 +11:00
httptools . p _GET = function ( httpurl , opts = { } , cb ) { //TODO-API rearranged and addded cb
2018-05-08 16:46:57 -07:00
/ * L o c a t e a n d r e t u r n a b l o c k , b a s e d o n i t s u r l
Throws TransportError if fails
opts {
start , end , // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
2018-07-21 11:08:38 -07:00
wantstream , // Return a stream rather than data
2019-03-20 21:20:26 -07:00
retries = 12 , // How many times to retry
2018-08-13 17:55:04 +10:00
}
2019-02-12 16:22:34 +11:00
returns result via promise or cb ( err , result )
2018-05-08 16:46:57 -07:00
* /
2019-02-12 16:22:34 +11:00
if ( typeof opts === "function" ) { cb = opts ; opts = { } ; }
2018-05-08 16:46:57 -07:00
let headers = new Headers ( ) ;
2018-10-13 21:29:10 +11:00
if ( opts . start || opts . end ) headers . append ( "range" , ` bytes= ${ opts . start || 0 } - ${ ( opts . end < Infinity ) ? opts . end : "" } ` ) ;
2019-03-20 21:20:26 -07:00
const retries = typeof opts . retries === "undefined" ? 12 : opts . retries ;
2018-05-08 16:46:57 -07:00
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method : 'GET' ,
headers : headers ,
mode : 'cors' ,
cache : 'default' ,
redirect : 'follow' , // Chrome defaults to manual
keepalive : true // Keep alive - mostly we'll be going back to same places a lot
} ;
2019-03-20 21:20:26 -07:00
const prom = httptools . p _httpfetch ( httpurl , init , { retries , wantstream : opts . wantstream } ) ; // This s a real http url
2019-02-22 16:31:17 +11:00
//if (cb) { prom.then((res)=>cb(null,res)).catch((err) => cb(err)); } else { return prom; } // Unpromisify pattern v3
2019-02-22 18:44:52 +11:00
//if (cb) { prom.catch((err) => cb(err)).then((res)=>cb(null,res)).catch((err) => debug("Uncaught error %O",err)); } else { return prom; } // Unpromisify pattern v4
if ( cb ) { prom . then ( ( res ) => { try { cb ( null , res ) } catch ( err ) { debug ( "Uncaught error %O" , err ) } } ) . catch ( ( err ) => cb ( err ) ) ; } else { return prom ; } // Unpromisify pattern v5
2018-05-08 16:46:57 -07:00
}
2019-02-12 16:22:34 +11:00
httptools . p _POST = function ( httpurl , opts = { } , cb ) { //TODO-API rearranged and addded cb
/ * L o c a t e a n d r e t u r n a b l o c k , b a s e d o n i t s u r l
2019-03-20 21:20:26 -07:00
opts = { data , contenttype , retries }
2019-02-12 16:22:34 +11:00
returns result via promise or cb ( err , result )
* /
2018-05-08 16:46:57 -07:00
// Throws TransportError if fails
//let headers = new window.Headers();
//headers.set('content-type',type); Doesn't work, it ignores it
2019-02-12 16:22:34 +11:00
if ( typeof opts === "function" ) { cb = opts ; opts = { } ; }
2019-03-20 21:20:26 -07:00
const retries = typeof opts . retries === "undefined" ? 0 : opts . retries ;
2018-05-08 16:46:57 -07:00
let init = {
//https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
//https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set
method : 'POST' ,
headers : { } , //headers,
//body: new Buffer(data),
2019-02-22 16:31:17 +11:00
body : opts . data ,
2018-05-08 16:46:57 -07:00
mode : 'cors' ,
cache : 'default' ,
redirect : 'follow' , // Chrome defaults to manual
2019-02-22 16:31:17 +11:00
keepalive : false // Keep alive - mostly we'll be going back to same places a lot
2018-05-08 16:46:57 -07:00
} ;
2019-02-22 16:31:17 +11:00
if ( opts . contenttype ) init . headers [ "Content-Type" ] = opts . contenttype ;
2019-03-20 21:20:26 -07:00
const prom = httptools . p _httpfetch ( httpurl , init , { retries } ) ;
2019-02-12 16:22:34 +11:00
if ( cb ) { prom . then ( ( res ) => cb ( null , res ) ) . catch ( ( err ) => cb ( err ) ) ; } else { return prom ; } // Unpromisify pattern v3
2018-05-08 16:46:57 -07:00
}
2019-02-12 16:22:34 +11:00
2019-02-22 16:31:17 +11:00
exports = module . exports = httptools ;