2018-04-06 19:28:04 +10:00
const Transport = require ( './Transport' ) ; // Base class for TransportXyz
const Transports = require ( './Transports' ) ; // Manage all Transports that are loaded
2018-05-08 16:46:57 -07:00
const httptools = require ( './httptools' ) ; // Expose some of the httptools so that IPFS can use it as a backup
2018-04-06 19:28:04 +10:00
const Url = require ( 'url' ) ;
2018-07-25 21:21:35 -07:00
const stream = require ( 'readable-stream' ) ;
2018-08-13 17:55:04 +10:00
const debughttp = require ( 'debug' ) ( 'dweb-transports:http' ) ;
2018-09-15 17:28:18 +10:00
const stringify = require ( 'canonical-json' ) ;
2018-04-06 19:28:04 +10:00
defaulthttpoptions = {
2018-06-12 11:36:10 -07:00
urlbase : 'https://dweb.me:443'
2018-04-06 19:28:04 +10:00
} ;
servercommands = { // What the server wants to see to return each of these
2018-06-11 14:56:38 -07:00
rawfetch : "contenthash" , // was content/rawfetch which should still work.
2018-04-06 19:28:04 +10:00
rawstore : "contenturl/rawstore" ,
rawadd : "void/rawadd" ,
rawlist : "metadata/rawlist" ,
get : "get/table" ,
set : "set/table" ,
delete : "delete/table" ,
keys : "keys/table" ,
getall : "getall/table"
} ;
class TransportHTTP extends Transport {
2018-08-13 17:55:04 +10:00
constructor ( options ) {
2018-10-05 12:50:32 +10:00
super ( options ) ; // These are now options.http
2018-04-06 19:28:04 +10:00
this . options = options ;
2018-10-05 12:50:32 +10:00
this . urlbase = options . urlbase ;
2018-04-06 19:28:04 +10:00
this . supportURLs = [ 'contenthash' , 'http' , 'https' ] ;
this . supportFunctions = [ 'fetch' , 'store' , 'add' , 'list' , 'reverse' , 'newlisturls' , "get" , "set" , "keys" , "getall" , "delete" , "newtable" , "newdatabase" ] ; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller
2018-10-05 12:50:32 +10:00
// noinspection JSUnusedGlobalSymbols
this . supportFeatures = [ 'fetch.range' ] ;
2018-04-06 19:28:04 +10:00
this . name = "HTTP" ; // For console log etc
this . status = Transport . STATUS _LOADED ;
}
2018-08-13 17:55:04 +10:00
static setup0 ( options ) {
2018-10-05 12:50:32 +10:00
let combinedoptions = Transport . mergeoptions ( defaulthttpoptions , options . http ) ;
2018-04-06 19:28:04 +10:00
try {
2018-08-13 17:55:04 +10:00
let t = new TransportHTTP ( combinedoptions ) ;
2018-04-06 19:28:04 +10:00
Transports . addtransport ( t ) ;
return t ;
} catch ( err ) {
2018-08-13 17:55:04 +10:00
console . error ( "HTTP unable to setup0" , err . message ) ;
2018-04-06 19:28:04 +10:00
throw err ;
}
}
2018-08-13 17:55:04 +10:00
async p _setup1 ( cb ) {
2018-04-08 14:53:19 +10:00
this . status = Transport . STATUS _STARTING ;
if ( cb ) cb ( this ) ;
2018-08-13 17:55:04 +10:00
await this . p _status ( ) ;
2018-04-08 14:53:19 +10:00
if ( cb ) cb ( this ) ;
2018-04-06 19:28:04 +10:00
return this ;
}
2018-08-13 17:55:04 +10:00
async p _status ( ) {
2018-04-06 19:28:04 +10:00
/ *
2018-04-09 12:12:20 +10:00
Return a numeric code for the status of a transport .
2018-04-06 19:28:04 +10:00
* /
try {
2018-08-13 17:55:04 +10:00
this . info = await this . p _info ( ) ;
2018-04-06 19:28:04 +10:00
this . status = Transport . STATUS _CONNECTED ;
} catch ( err ) {
2018-08-13 17:55:04 +10:00
console . error ( this . name , ": Error in p_status.info" , err . message ) ;
2018-04-06 19:28:04 +10:00
this . status = Transport . STATUS _FAILED ;
}
2018-08-13 17:55:04 +10:00
return super . p _status ( ) ;
2018-04-06 19:28:04 +10:00
}
_cmdurl ( command ) {
return ` ${ this . urlbase } / ${ command } `
}
_url ( url , command , parmstr ) {
if ( ! url ) throw new errors . CodingError ( ` ${ command } : requires url ` ) ;
if ( typeof url !== "string" ) { url = url . href }
url = url . replace ( 'contenthash:/contenthash' , this . _cmdurl ( command ) ) ; // Note leaves http: and https: urls unchanged
url = url . replace ( 'getall/table' , command ) ;
url = url + ( parmstr ? "?" + parmstr : "" ) ;
return url ;
}
2018-10-05 12:50:32 +10:00
// noinspection JSCheckFunctionSignatures
2018-04-06 19:28:04 +10:00
async p _rawfetch ( url , opts = { } ) {
/ *
Fetch from underlying transport ,
Fetch is used both for contenthash requests and table as when passed to SmartDict . p _fetch may not know what we have
url : Of resource - which is turned into the HTTP url in p _httpfetch
2019-03-20 21:20:26 -07:00
opts : { start , end , retries } see p _GET for documentation
2018-04-06 19:28:04 +10:00
throws : TransportError if fails
* /
//if (!(url && url.includes(':') ))
// throw new errors.CodingError("TransportHTTP.p_rawfetch bad url: "+url);
2018-08-13 17:55:04 +10:00
//if (url.href.includes('contenthash//'))
// console.error("XXX@91", url)
2018-04-06 19:28:04 +10:00
if ( ( ( typeof url === "string" ) ? url : url . href ) . includes ( '/getall/table' ) ) {
2018-05-30 20:11:33 -07:00
throw new Error ( "Probably dont want to be calling p_rawfetch on a KeyValueTable, especially since dont know if its keyvaluetable or subclass" ) ; //TODO-NAMING
2018-04-06 19:28:04 +10:00
} else {
2018-05-08 16:46:57 -07:00
return await httptools . p _GET ( this . _url ( url , servercommands . rawfetch ) , opts ) ;
2018-04-06 19:28:04 +10:00
}
}
2018-08-13 17:55:04 +10:00
p _rawlist ( url ) {
2018-04-06 19:28:04 +10:00
// obj being loaded
// Locate and return a block, based on its url
if ( ! url ) throw new errors . CodingError ( "TransportHTTP.p_rawlist: requires url" ) ;
2018-08-13 17:55:04 +10:00
return httptools . p _GET ( this . _url ( url , servercommands . rawlist ) ) ;
2018-04-06 19:28:04 +10:00
}
rawreverse ( ) { throw new errors . ToBeImplementedError ( "Undefined function TransportHTTP.rawreverse" ) ; }
2018-08-13 17:55:04 +10:00
async p _rawstore ( data ) {
2018-04-06 19:28:04 +10:00
/ *
Store data on http server ,
data : string
resolves to : { string } : url
throws : TransportError on failure in p _POST > p _httpfetch
* /
2018-08-13 17:55:04 +10:00
//PY: res = self._sendGetPost(True, "rawstore", headers={"Content-Type": "application/octet-stream"}, urlargs=[], data=data)
2018-04-06 19:28:04 +10:00
console . assert ( data , "TransportHttp.p_rawstore: requires data" ) ;
2019-02-12 16:22:34 +11:00
const res = await httptools . p _POST ( this . _cmdurl ( servercommands . rawstore ) , { data , contenttype : "application/octet-stream" } ) ; // resolves to URL
2018-04-06 19:28:04 +10:00
let parsedurl = Url . parse ( res ) ;
let pathparts = parsedurl . pathname . split ( '/' ) ;
return ` contenthash:/contenthash/ ${ pathparts . slice ( - 1 ) } `
}
2018-08-13 17:55:04 +10:00
p _rawadd ( url , sig ) {
// Logged by Transports
if ( ! url || ! sig ) throw new errors . CodingError ( "TransportHTTP.p_rawadd: invalid parms" , url , sig ) ;
2019-02-12 16:22:34 +11:00
const data = stringify ( sig . preflight ( Object . assign ( { } , sig ) ) ) + "\n" ;
return httptools . p _POST ( this . _url ( url , servercommands . rawadd ) , { data , contenttype : "application/json" } ) ; // Returns immediately
2018-04-06 19:28:04 +10:00
}
2018-08-13 17:55:04 +10:00
p _newlisturls ( cl ) {
2018-10-11 15:47:46 +11:00
let u = cl . _publicurls . map ( urlstr => Url . parse ( urlstr ) )
2018-04-06 19:28:04 +10:00
. find ( parsedurl =>
2018-06-12 11:36:10 -07:00
( ( parsedurl . protocol === "https:" && [ "gateway.dweb.me" , "dweb.me" ] . includes ( parsedurl . host )
2018-05-21 17:23:13 -07:00
&& ( parsedurl . pathname . includes ( '/content/rawfetch' ) || parsedurl . pathname . includes ( '/contenthash/' ) ) )
2018-10-11 15:47:46 +11:00
|| ( parsedurl . protocol === "contenthash:" ) && ( parsedurl . pathname . split ( '/' ) [ 1 ] === "contenthash" ) ) ) ;
2018-04-06 19:28:04 +10:00
if ( ! u ) {
2018-10-05 12:50:32 +10:00
// noinspection JSUnresolvedVariable
2018-04-06 19:28:04 +10:00
u = ` contenthash:/contenthash/ ${ cl . keypair . verifyexportmultihashsha256 _58 ( ) } ` ; // Pretty random, but means same test will generate same list and server is expecting base58 of a hash
}
return [ u , u ] ;
}
2018-07-21 11:08:38 -07:00
// ============================== Stream support
/ *
Code disabled until have a chance to test it with < VIDEO > tag etc , problem is that it returns p _createReadStream whch is async
if need sync , look at WebTorrent and how it buffers through a stream which can be returned immediately
* /
2018-08-13 17:55:04 +10:00
async p _f _createReadStream ( url , { wanturl = false } = { } ) {
2018-07-21 11:08:38 -07:00
/ *
Fetch bytes progressively , using a node . js readable stream , based on a url of the form :
No assumption is made about the data in terms of size or structure .
This is the initialisation step , which returns a function suitable for < VIDEO >
Returns a new Promise that resolves to function for a node . js readable stream .
Node . js readable stream docs : https : //nodejs.org/api/stream.html#stream_readable_streams
: param string url : URL of object being retrieved of form magnet : xyzabc / path / to / file ( Where xyzabc is the typical magnet uri contents )
: resolves to : f ( { start , end } ) => stream ( The readable stream . )
: throws : TransportError if url invalid - note this happens immediately , not as a catch in the promise
* /
2018-08-13 17:55:04 +10:00
//Logged by Transports
//debughttp("p_f_createreadstream %s", Url.parse(url).href);
2018-07-21 11:08:38 -07:00
try {
let self = this ;
if ( wanturl ) {
return url ;
} else {
2018-09-16 20:51:28 +10:00
return function ( opts ) { return self . createReadStream ( url , opts ) ; } ;
2018-07-21 11:08:38 -07:00
}
} catch ( err ) {
2018-08-13 17:55:04 +10:00
//Logged by Transports
//console.warn(`p_f_createReadStream failed on ${Url.parse(url).href} ${err.message}`);
2018-07-21 11:08:38 -07:00
throw ( err ) ;
}
}
2018-09-16 20:51:28 +10:00
createReadStream ( url , opts ) {
2018-07-25 21:21:35 -07:00
/ *
The function , encapsulated and inside another function by p _f _createReadStream ( see docs )
NOTE THIS DOESNT WONT WORK FOR < VIDEO > tags , but shouldnt be using it there anyway - reports stream . on an filestream . pipe aren ' t functions
: param file : Webtorrent "file" as returned by webtorrentfindfile
: param opts : { start : byte to start from ; end : optional end byte }
2018-10-11 15:47:46 +11:00
: returns stream : The readable stream - it is returned immediately , though won ' t be sending data until the http completes
2018-07-25 21:21:35 -07:00
* /
2018-08-13 17:55:04 +10:00
debughttp ( "createreadstream %s %o" , Url . parse ( url ) . href , opts ) ;
2018-07-25 21:21:35 -07:00
let through ;
2018-10-11 15:47:46 +11:00
through = new stream . PassThrough ( ) ;
httptools . p _GET ( this . _url ( url , servercommands . rawfetch ) , Object . assign ( { wantstream : true } , opts ) )
. then ( s => s . pipe ( through ) )
. catch ( err => {
console . warn ( this . name , "createReadStream caught error" , err . message ) ;
if ( typeof through . destroy === 'function' ) {
through . destroy ( err ) ; // Will emit error & close and free up resources
} else {
through . emit ( 'error' , err ) ;
}
} ) ;
return through ; // Returns "through" synchronously, before the pipe is setup
2018-07-25 21:21:35 -07:00
}
2018-07-21 11:08:38 -07:00
2018-08-13 17:55:04 +10:00
async p _createReadStream ( url , opts ) {
2018-07-21 11:08:38 -07:00
/ *
The function , encapsulated and inside another function by p _f _createReadStream ( see docs )
NOTE THIS PROBABLY WONT WORK FOR < VIDEO > tags , but shouldnt be using it there anyway
: param file : Webtorrent "file" as returned by webtorrentfindfile
: param opts : { start : byte to start from ; end : optional end byte }
: resolves to stream : The readable stream .
* /
2018-08-13 17:55:04 +10:00
debughttp ( "createreadstream %s %o" , Url . parse ( url ) . href , opts ) ;
2018-07-21 11:08:38 -07:00
try {
return await httptools . p _GET ( this . _url ( url , servercommands . rawfetch ) , Object . assign ( { wantstream : true } , opts ) ) ;
} catch ( err ) {
console . warn ( this . name , "caught error" , err ) ;
throw err ;
}
}
2018-04-06 19:28:04 +10:00
// ============================== Key Value support
// Support for Key-Value pairs as per
// https://docs.google.com/document/d/1yfmLRqKPxKwB939wIy9sSaa7GKOzM5PrCZ4W1jRGW6M/edit#
2018-08-13 17:55:04 +10:00
async p _newdatabase ( pubkey ) {
2018-04-06 19:28:04 +10:00
//if (pubkey instanceof Dweb.PublicPrivate)
if ( pubkey . hasOwnProperty ( "keypair" ) )
2018-10-05 12:50:32 +10:00
pubkey = pubkey . keypair . signingexport ( ) ;
2018-04-06 19:28:04 +10:00
// By this point pubkey should be an export of a public key of form xyz:abc where xyz
// specifies the type of public key (NACL VERIFY being the only kind we expect currently)
let u = ` ${ this . urlbase } /getall/table/ ${ encodeURIComponent ( pubkey ) } ` ;
return { "publicurl" : u , "privateurl" : u } ;
}
2018-08-13 17:55:04 +10:00
async p _newtable ( pubkey , table ) {
2018-04-06 19:28:04 +10:00
if ( ! pubkey ) throw new errors . CodingError ( "p_newtable currently requires a pubkey" ) ;
2018-08-13 17:55:04 +10:00
let database = await this . p _newdatabase ( pubkey ) ;
2018-04-06 19:28:04 +10:00
// If have use cases without a database, then call p_newdatabase first
return { privateurl : ` ${ database . privateurl } / ${ table } ` , publicurl : ` ${ database . publicurl } / ${ table } ` } // No action required to create it
}
//TODO-KEYVALUE needs signing with private key of list
2018-08-13 17:55:04 +10:00
async p _set ( url , keyvalues , value ) { // url = yjs:/yjs/database/table/key
if ( ! url || ! keyvalues ) throw new errors . CodingError ( "TransportHTTP.p_set: invalid parms" , url , keyvalyes ) ;
// Logged by Transports
//debughttp("p_set %o %o %o", url, keyvalues, value);
2018-04-06 19:28:04 +10:00
if ( typeof keyvalues === "string" ) {
2019-02-12 16:22:34 +11:00
let data = stringify ( [ { key : keyvalues , value : value } ] ) ;
await httptools . p _POST ( this . _url ( url , servercommands . set ) , { data , contenttype : "application/json" } ) ; // Returns immediately
2018-04-06 19:28:04 +10:00
} else {
2019-02-12 16:22:34 +11:00
let data = stringify ( Object . keys ( keyvalues ) . map ( ( k ) => ( { "key" : k , "value" : keyvalues [ k ] } ) ) ) ;
await httptools . p _POST ( this . _url ( url , servercommands . set ) , { data , contenttype : "application/json" } ) ; // Returns immediately
2018-04-06 19:28:04 +10:00
}
}
_keyparm ( key ) {
return ` key= ${ encodeURIComponent ( key ) } `
}
2018-08-13 17:55:04 +10:00
async p _get ( url , keys ) {
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
2018-10-05 12:50:32 +10:00
let parmstr = Array . isArray ( keys ) ? keys . map ( k => this . _keyparm ( k ) ) . join ( '&' ) : this . _keyparm ( keys ) ;
2019-02-08 13:18:38 +11:00
const res = await httptools . p _GET ( this . _url ( url , servercommands . get , parmstr ) ) ;
2018-04-06 19:28:04 +10:00
return Array . isArray ( keys ) ? res : res [ keys ]
}
2018-08-13 17:55:04 +10:00
async p _delete ( url , keys ) {
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
let parmstr = keys . map ( k => this . _keyparm ( k ) ) . join ( '&' ) ;
2018-08-13 17:55:04 +10:00
await httptools . p _GET ( this . _url ( url , servercommands . delete , parmstr ) ) ;
2018-04-06 19:28:04 +10:00
}
2018-08-13 17:55:04 +10:00
async p _keys ( url ) {
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
2018-08-13 17:55:04 +10:00
return await httptools . p _GET ( this . _url ( url , servercommands . keys ) ) ;
2018-04-06 19:28:04 +10:00
}
2018-08-13 17:55:04 +10:00
async p _getall ( url ) {
2018-04-06 19:28:04 +10:00
if ( ! url && keys ) throw new errors . CodingError ( "TransportHTTP.p_get: requires url and at least one key" ) ;
2018-08-13 17:55:04 +10:00
return await httptools . p _GET ( this . _url ( url , servercommands . getall ) ) ;
2018-04-06 19:28:04 +10:00
}
/ * M a k e s u r e d o e s n t s h a d o w r e g u l a r p _ r a w f e t c h
2018-08-13 17:55:04 +10:00
async p _rawfetch ( url ) {
2018-04-06 19:28:04 +10:00
return {
table : "keyvaluetable" ,
2018-08-13 17:55:04 +10:00
_map : await this . p _getall ( url )
2018-04-06 19:28:04 +10:00
} ; // Data struc is ok as SmartDict.p_fetch will pass to KVT constructor
}
* /
2018-08-13 17:55:04 +10:00
p _info ( ) { return httptools . p _GET ( ` ${ this . urlbase } /info ` ) ; }
2018-04-06 19:28:04 +10:00
2018-08-13 17:55:04 +10:00
static async p _test ( opts = { } ) {
{ console . log ( "TransportHTTP.test" ) }
2018-04-06 19:28:04 +10:00
try {
2018-08-13 17:55:04 +10:00
let transport = await this . p _setup ( opts ) ;
console . log ( "HTTP connected" ) ;
let res = await transport . p _info ( ) ;
console . log ( "TransportHTTP info=" , res ) ;
res = await transport . p _status ( ) ;
2018-04-06 19:28:04 +10:00
console . assert ( res === Transport . STATUS _CONNECTED ) ;
2018-08-13 17:55:04 +10:00
await transport . p _test _kvt ( "NACL%20VERIFY" ) ;
2018-04-06 19:28:04 +10:00
} catch ( err ) {
console . log ( "Exception thrown in TransportHTTP.test:" , err . message ) ;
throw err ;
}
}
static async test ( ) {
return this ;
}
}
Transports . _transportclasses [ "HTTP" ] = TransportHTTP ;
exports = module . exports = TransportHTTP ;