2018-04-06 19:28:04 +10:00
const errors = require ( './Errors' ) ; // Standard Dweb Errors
const Transport = require ( './Transport' ) ; // Base class for TransportXyz
const Transports = require ( './Transports' ) ; // Manage all Transports that are loaded
const nodefetch = require ( 'node-fetch' ) ; // Note, were using node-fetch-npm which had a warning in webpack see https://github.com/bitinn/node-fetch/issues/421 and is intended for clients
const Url = require ( 'url' ) ;
2018-04-13 17:26:44 +10:00
//var fetch,Headers,Request;
//if (typeof(Window) === "undefined") {
if ( typeof ( fetch ) === "undefined" ) {
2018-04-06 19:28:04 +10:00
//var fetch = require('whatwg-fetch').fetch; //Not as good as node-fetch-npm, but might be the polyfill needed for browser.safari
//XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest; // Note this doesnt work if set to a var or const, needed by whatwg-fetch
console . log ( "Node loaded" ) ;
fetch = nodefetch ;
Headers = fetch . Headers ; // A class
Request = fetch . Request ; // A class
2018-04-13 17:26:44 +10:00
} / * else {
2018-04-06 19:28:04 +10:00
// If on a browser, need to find fetch,Headers,Request in window
console . log ( "Loading browser version of fetch,Headers,Request" ) ;
fetch = window . fetch ;
Headers = window . Headers ;
Request = window . Request ;
2018-04-13 17:26:44 +10:00
} * /
2018-04-06 19:28:04 +10:00
//TODO-HTTP to work on Safari or mobile will require a polyfill, see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch for comment
defaulthttpoptions = {
urlbase : 'https://gateway.dweb.me:443'
} ;
servercommands = { // What the server wants to see to return each of these
rawfetch : "content/rawfetch" ,
rawstore : "contenturl/rawstore" ,
rawadd : "void/rawadd" ,
rawlist : "metadata/rawlist" ,
get : "get/table" ,
set : "set/table" ,
delete : "delete/table" ,
keys : "keys/table" ,
getall : "getall/table"
} ;
class TransportHTTP extends Transport {
constructor ( options , verbose ) {
super ( options , verbose ) ;
this . options = options ;
this . urlbase = options . http . urlbase ;
this . supportURLs = [ 'contenthash' , 'http' , 'https' ] ;
this . supportFunctions = [ 'fetch' , 'store' , 'add' , 'list' , 'reverse' , 'newlisturls' , "get" , "set" , "keys" , "getall" , "delete" , "newtable" , "newdatabase" ] ; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller
this . supportFeatures = [ 'fetch.range' ]
this . name = "HTTP" ; // For console log etc
this . status = Transport . STATUS _LOADED ;
}
static setup0 ( options , verbose ) {
let combinedoptions = Transport . mergeoptions ( { http : defaulthttpoptions } , options ) ;
try {
let t = new TransportHTTP ( combinedoptions , verbose ) ;
Transports . addtransport ( t ) ;
return t ;
} catch ( err ) {
console . log ( "Exception thrown in TransportHTTP.p_setup" , err . message ) ;
throw err ;
}
}
2018-04-08 14:53:19 +10:00
async p _setup1 ( verbose , cb ) {
this . status = Transport . STATUS _STARTING ;
if ( cb ) cb ( this ) ;
await this . p _status ( verbose ) ;
if ( cb ) cb ( this ) ;
2018-04-06 19:28:04 +10:00
return this ;
}
2018-04-09 12:12:20 +10:00
async p _status ( verbose ) {
2018-04-06 19:28:04 +10:00
/ *
2018-04-09 12:12:20 +10:00
Return a numeric code for the status of a transport .
2018-04-06 19:28:04 +10:00
* /
try {
this . info = await this . p _info ( verbose ) ;
this . status = Transport . STATUS _CONNECTED ;
} catch ( err ) {
console . log ( this . name , ": Error in p_status.info" , err . message ) ;
this . status = Transport . STATUS _FAILED ;
}
2018-04-23 10:49:53 +10:00
return super . p _status ( verbose ) ;
2018-04-06 19:28:04 +10:00
}
async p _httpfetch ( httpurl , init , verbose ) { // Embrace and extend "fetch" to check result etc.
/ *
2018-04-23 10:49:53 +10:00
Fetch a url
2018-04-06 19:28:04 +10:00
url : optional ( depends on command )
resolves to : data as text or json depending on Content - Type header
throws : TransportError if fails to fetch
* /
try {
if ( verbose ) console . log ( "httpurl=%s init=%o" , httpurl , init ) ;
//console.log('CTX=',init["headers"].get('Content-Type'))
// Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.
let response = await fetch ( new Request ( httpurl , init ) ) ;
// fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)
// Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
if ( response . ok ) {
let contenttype = response . headers . get ( 'Content-Type' ) ;
if ( contenttype === "application/json" ) {
return response . json ( ) ; // promise resolving to JSON
} else if ( contenttype . startsWith ( "text" ) ) { // Note in particular this is used for responses to store
2018-04-23 10:49:53 +10:00
return response . text ( ) ;
2018-04-06 19:28:04 +10:00
} else { // Typically application/octetStream when don't know what fetching
return new Buffer ( await response . arrayBuffer ( ) ) ; // Convert arrayBuffer to Buffer which is much more usable currently
}
}
// noinspection ExceptionCaughtLocallyJS
throw new errors . TransportError ( ` Transport Error ${ response . status } : ${ response . statusText } ` ) ;
} catch ( err ) {
// Error here is particularly unhelpful - if rejected during the COrs process it throws a TypeError
console . log ( "Note error from fetch might be misleading especially TypeError can be Cors issue:" , httpurl ) ;
if ( err instanceof errors . TransportError ) {
throw err ;
} else {
throw new errors . TransportError ( ` Transport error thrown by ${ httpurl } : ${ err . message } ` ) ;
}
}
}
async p _GET ( httpurl , opts = { } ) {
/ * L o c a t e a n d r e t u r n a b l o c k , b a s e d o n i t s u r l
Throws TransportError if fails
opts {
start , end , // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
verbose }
resolves to : URL that can be used to fetch the resource , of form contenthash : / c o n t e n t h a s h / Q 1 2 3
* /
let headers = new Headers ( ) ;
if ( opts . start || opts . end ) headers . append ( "range" , ` bytes= ${ opts . start || 0 } - ${ opts . end || "" } ` ) ;
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method : 'GET' ,
headers : headers ,
mode : 'cors' ,
cache : 'default' ,
redirect : 'follow' , // Chrome defaults to manual
keepalive : true // Keep alive - mostly we'll be going back to same places a lot
} ;
return await this . p _httpfetch ( httpurl , init , opts . verbose ) ; // This s a real http url
}
async p _POST ( httpurl , type , data , verbose ) {
// Locate and return a block, based on its url
// Throws TransportError if fails
//let headers = new window.Headers();
//headers.set('content-type',type); Doesn't work, it ignores it
let init = {
//https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
//https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set
method : 'POST' ,
headers : { } , //headers,
//body: new Buffer(data),
body : data ,
mode : 'cors' ,
cache : 'default' ,
redirect : 'follow' , // Chrome defaults to manual
keepalive : true // Keep alive - mostly we'll be going back to same places a lot
} ;
return await this . p _httpfetch ( httpurl , init , verbose ) ;
}
_cmdurl ( command ) {
return ` ${ this . urlbase } / ${ command } `
}
_url ( url , command , parmstr ) {
if ( ! url ) throw new errors . CodingError ( ` ${ command } : requires url ` ) ;
if ( typeof url !== "string" ) { url = url . href }
url = url . replace ( 'contenthash:/contenthash' , this . _cmdurl ( command ) ) ; // Note leaves http: and https: urls unchanged
url = url . replace ( 'getall/table' , command ) ;
url = url + ( parmstr ? "?" + parmstr : "" ) ;
return url ;
}
async p _rawfetch ( url , opts = { } ) {
/ *
Fetch from underlying transport ,
Fetch is used both for contenthash requests and table as when passed to SmartDict . p _fetch may not know what we have
url : Of resource - which is turned into the HTTP url in p _httpfetch
opts : { start , end , verbose } see p _GET for documentation
throws : TransportError if fails
* /
//if (!(url && url.includes(':') ))
// throw new errors.CodingError("TransportHTTP.p_rawfetch bad url: "+url);
if ( ( ( typeof url === "string" ) ? url : url . href ) . includes ( '/getall/table' ) ) {
console . log ( "XXX@176 - probably dont want to be calling p_rawfetch on a KeyValueTable, especially since dont know if its keyvaluetable or subclass" ) ; //TODO-NAMING
return {
table : "keyvaluetable" ,
}
} else {
return await this . p _GET ( this . _url ( url , servercommands . rawfetch ) , opts ) ;
}
}
2018-04-09 12:12:20 +10:00
p _rawlist ( url , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
// obj being loaded
// Locate and return a block, based on its url
if ( ! url ) throw new errors . CodingError ( "TransportHTTP.p_rawlist: requires url" ) ;
return this . p _GET ( this . _url ( url , servercommands . rawlist ) , { verbose } ) ;
}
rawreverse ( ) { throw new errors . ToBeImplementedError ( "Undefined function TransportHTTP.rawreverse" ) ; }
2018-04-09 12:12:20 +10:00
async p _rawstore ( data , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
/ *
Store data on http server ,
data : string
resolves to : { string } : url
throws : TransportError on failure in p _POST > p _httpfetch
* /
//PY: res = self._sendGetPost(True, "rawstore", headers={"Content-Type": "application/octet-stream"}, urlargs=[], data=data, verbose=verbose)
console . assert ( data , "TransportHttp.p_rawstore: requires data" ) ;
let res = await this . p _POST ( this . _cmdurl ( servercommands . rawstore ) , "application/octet-stream" , data , verbose ) ; // resolves to URL
let parsedurl = Url . parse ( res ) ;
let pathparts = parsedurl . pathname . split ( '/' ) ;
return ` contenthash:/contenthash/ ${ pathparts . slice ( - 1 ) } `
}
2018-04-09 12:12:20 +10:00
p _rawadd ( url , sig , { verbose = false } = { } ) { //TODO-BACKPORT turn date into ISO before adding
2018-04-06 19:28:04 +10:00
//verbose=true;
if ( ! url || ! sig ) throw new errors . CodingError ( "TransportHTTP.p_rawadd: invalid parms" , url , sig ) ;
if ( verbose ) console . log ( "rawadd" , url , sig ) ;
let value = JSON . stringify ( sig . preflight ( Object . assign ( { } , sig ) ) ) + "\n" ;
return this . p _POST ( this . _url ( url , servercommands . rawadd ) , "application/json" , value , verbose ) ; // Returns immediately
}
2018-04-09 12:12:20 +10:00
p _newlisturls ( cl , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
let u = cl . _publicurls . map ( urlstr => Url . parse ( urlstr ) )
. find ( parsedurl =>
( parsedurl . protocol === "https" && parsedurl . host === "gateway.dweb.me" && parsedurl . pathname . includes ( '/content/rawfetch' ) )
|| ( parsedurl . protocol === "contenthash:" && ( parsedurl . pathname . split ( '/' ) [ 1 ] === "contenthash" ) ) ) ;
if ( ! u ) {
u = ` contenthash:/contenthash/ ${ cl . keypair . verifyexportmultihashsha256 _58 ( ) } ` ; // Pretty random, but means same test will generate same list and server is expecting base58 of a hash
}
return [ u , u ] ;
}
// ============================== Key Value support
// Support for Key-Value pairs as per
// https://docs.google.com/document/d/1yfmLRqKPxKwB939wIy9sSaa7GKOzM5PrCZ4W1jRGW6M/edit#
2018-04-09 12:12:20 +10:00
async p _newdatabase ( pubkey , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
//if (pubkey instanceof Dweb.PublicPrivate)
if ( pubkey . hasOwnProperty ( "keypair" ) )
pubkey = pubkey . keypair . signingexport ( )
// By this point pubkey should be an export of a public key of form xyz:abc where xyz
// specifies the type of public key (NACL VERIFY being the only kind we expect currently)
let u = ` ${ this . urlbase } /getall/table/ ${ encodeURIComponent ( pubkey ) } ` ;
return { "publicurl" : u , "privateurl" : u } ;
}
2018-04-09 12:12:20 +10:00
async p _newtable ( pubkey , table , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
if ( ! pubkey ) throw new errors . CodingError ( "p_newtable currently requires a pubkey" ) ;
2018-04-09 12:12:20 +10:00
let database = await this . p _newdatabase ( pubkey , { verbose } ) ;
2018-04-06 19:28:04 +10:00
// If have use cases without a database, then call p_newdatabase first
return { privateurl : ` ${ database . privateurl } / ${ table } ` , publicurl : ` ${ database . publicurl } / ${ table } ` } // No action required to create it
}
//TODO-KEYVALUE needs signing with private key of list
2018-04-09 12:12:20 +10:00
async p _set ( url , keyvalues , value , { verbose = false } = { } ) { // url = yjs:/yjs/database/table/key //TODO-KEYVALUE-API
2018-04-06 19:28:04 +10:00
if ( ! url || ! keyvalues ) throw new errors . CodingError ( "TransportHTTP.p_set: invalid parms" , url , keyvalyes ) ;
if ( verbose ) console . log ( "p_set" , url , keyvalues , value ) ;
if ( typeof keyvalues === "string" ) {
let kv = JSON . stringify ( [ { key : keyvalues , value : value } ] ) ;
await this . p _POST ( this . _url ( url , servercommands . set ) , "application/json" , kv , verbose ) ; // Returns immediately
} else {
let kv = JSON . stringify ( Object . keys ( keyvalues ) . map ( ( k ) => ( { "key" : k , "value" : keyvalues [ k ] } ) ) ) ;
await this . p _POST ( this . _url ( url , servercommands . set ) , "application/json" , kv , verbose ) ; // Returns immediately
}
}
_keyparm ( key ) {
return ` key= ${ encodeURIComponent ( key ) } `
}
2018-04-09 12:12:20 +10:00
async p _get ( url , keys , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
let parmstr = Array . isArray ( keys ) ? keys . map ( k => this . _keyparm ( k ) ) . join ( '&' ) : this . _keyparm ( keys )
let res = await this . p _GET ( this . _url ( url , servercommands . get , parmstr ) , { verbose } ) ;
return Array . isArray ( keys ) ? res : res [ keys ]
}
2018-04-09 12:12:20 +10:00
async p _delete ( url , keys , { verbose = false } = { } ) { //TODO-KEYVALUE-API need to think this one through
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
let parmstr = keys . map ( k => this . _keyparm ( k ) ) . join ( '&' ) ;
await this . p _GET ( this . _url ( url , servercommands . delete , parmstr ) , { verbose } ) ;
}
2018-04-09 12:12:20 +10:00
async p _keys ( url , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
return await this . p _GET ( this . _url ( url , servercommands . keys ) , { verbose } ) ;
}
2018-04-09 12:12:20 +10:00
async p _getall ( url , { verbose = false } = { } ) {
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
return await this . p _GET ( this . _url ( url , servercommands . getall ) , { verbose } ) ;
}
/ * M a k e s u r e d o e s n t s h a d o w r e g u l a r p _ r a w f e t c h
async p _rawfetch ( url , verbose ) {
return {
table : "keyvaluetable" ,
2018-04-09 12:12:20 +10:00
_map : await this . p _getall ( url , { verbose } )
2018-04-06 19:28:04 +10:00
} ; // Data struc is ok as SmartDict.p_fetch will pass to KVT constructor
}
* /
2018-04-09 12:12:20 +10:00
p _info ( verbose ) { return this . p _GET ( ` ${ this . urlbase } /info ` , { verbose } ) ; }
2018-04-06 19:28:04 +10:00
static async p _test ( opts = { } , verbose = false ) {
if ( verbose ) { console . log ( "TransportHTTP.test" ) }
try {
let transport = await this . p _setup ( opts , verbose ) ;
if ( verbose ) console . log ( "HTTP connected" ) ;
let res = await transport . p _info ( verbose ) ;
if ( verbose ) console . log ( "TransportHTTP info=" , res ) ;
res = await transport . p _status ( verbose ) ;
console . assert ( res === Transport . STATUS _CONNECTED ) ;
await transport . p _test _kvt ( "NACL%20VERIFY" , verbose ) ;
} catch ( err ) {
console . log ( "Exception thrown in TransportHTTP.test:" , err . message ) ;
throw err ;
}
}
static async test ( ) {
return this ;
}
}
Transports . _transportclasses [ "HTTP" ] = TransportHTTP ;
exports = module . exports = TransportHTTP ;