2018-04-06 19:28:04 +10:00
const Url = require ( 'url' ) ;
const errors = require ( './Errors' ) ;
2018-04-09 12:12:20 +10:00
const utils = require ( './utils' ) ;
2019-04-16 12:22:33 +10:00
const debug = require ( 'debug' ) ( 'dweb-transports' ) ;
2019-02-22 16:31:17 +11:00
const httptools = require ( './httptools' ) ;
2019-04-14 16:36:33 +10:00
const each = require ( 'async/each' ) ;
2019-06-26 20:42:37 +10:00
const map = require ( 'async/map' ) ;
2019-09-08 12:20:40 +10:00
const { p _namingcb , naming } = require ( './Naming.js' )
2018-04-06 19:28:04 +10:00
class Transports {
2018-05-05 21:04:15 -07:00
/ *
Handles multiple transports , API should be ( almost ) the same as for an individual transport )
Fields :
_transports List of transports loaded ( internal )
namingcb If set will be called cb ( urls ) => urls to convert to urls from names .
_transportclasses All classes whose code is loaded e . g . { HTTP : TransportHTTP , IPFS : TransportIPFS }
_optionspaused Saves paused option for setup
* /
2019-07-17 08:15:23 -07:00
//TODO a few of these things could be much better as events that are listened for, especially p_statuses
2018-08-13 17:55:04 +10:00
constructor ( options ) {
// THIS IS UNUSED - ALL METHODS ARE STATIC, THERE IS NO Transports INSTANCE
2018-04-06 19:28:04 +10:00
}
static _connected ( ) {
/ *
Get an array of transports that are connected , i . e . currently usable
* /
return this . _transports . filter ( ( t ) => ( ! t . status ) ) ;
}
2019-06-09 09:24:58 +10:00
static p _connectedNames ( cb ) { //TODO rename to connectedNames
2018-04-09 12:12:20 +10:00
/ *
2018-04-23 10:49:53 +10:00
resolves to : an array of the names of connected transports
2018-12-29 10:06:06 +11:00
Note this is async only because the TransportsProxy version of this has to be - that isn ' t currently used , so this could be made sync
2018-04-09 12:12:20 +10:00
* /
2018-12-29 10:06:06 +11:00
const res = this . _connected ( ) . map ( t => t . name ) ;
if ( cb ) { cb ( null , res ) } else { return new Promise ( ( resolve , reject ) => resolve ( res ) ) }
2018-04-06 19:28:04 +10:00
}
2018-04-13 17:26:44 +10:00
static async p _connectedNamesParm ( ) { // Doesnt strictly need to be async, but for consistency with Proxy it has to be.
return ( await this . p _connectedNames ( ) ) . map ( n => "transport=" + n ) . join ( '&' )
2018-04-06 19:28:04 +10:00
}
2019-08-15 15:12:35 +10:00
static statuses ( { connected = undefined } ) { //TODO-API (especially add info:)
2019-07-17 08:15:23 -07:00
/ *
Return array of statuses ,
connected : If true then only connected transports
* /
2019-08-15 15:12:35 +10:00
const ss = Transports . _transports . map ( ( t ) => { return { name : t . name , status : t . status , info : t . info } } ) ;
2019-07-17 08:15:23 -07:00
return connected ? ss . filter ( s => ! s . status ) : ss ;
}
2019-05-14 16:57:42 +10:00
static p _statuses ( cb ) {
2019-07-17 08:15:23 -07:00
/ *
resolves to : a dictionary of statuses of transports , e . g . { TransportHTTP : STATUS _CONNECTED }
* /
const res = this . statuses ( { connected : false } ) ; // No errors possible
if ( cb ) { cb ( null , res ) } else { return new Promise ( ( resolve , reject ) => resolve ( res ) ) }
2018-04-15 17:34:20 +10:00
}
2019-05-30 15:57:37 +10:00
static validFor ( urls , func , opts ) { //TODO-RELOAD check for noCache support
2018-04-06 19:28:04 +10:00
/ *
Finds an array or Transports that can support this URL .
Excludes any transports whose status != 0 as they aren ' t connected
urls : Array of urls
func : Function to check support for : fetch , store , add , list , listmonitor , reverse - see supportFunctions on each Transport class
2019-05-30 15:57:37 +10:00
opts : Passed to each Transport , esp for supportFeatures
2018-08-13 17:55:04 +10:00
returns : Array of pairs of Url & transport instance [ [ u1 , t1 ] , [ u1 , t2 ] , [ u2 , t1 ] ]
2018-06-11 18:07:58 -07:00
throws : CodingError if urls empty or [ undefined ... ]
2018-04-06 19:28:04 +10:00
* /
2018-10-06 21:35:17 +10:00
if ( typeof urls === "string" ) urls = [ urls ] ;
2019-04-14 16:36:33 +10:00
if ( ! ( ( urls && urls [ 0 ] ) || [ "store" , "newlisturls" , "newdatabase" , "newtable" , "seed" ] . includes ( func ) ) ) {
2018-08-13 17:55:04 +10:00
console . error ( "Transports.validFor called with invalid arguments: urls=" , urls , "func=" , func ) ; // FOr debugging old calling patterns with [ undefined ]
2018-07-06 14:09:44 -07:00
return [ ] ;
}
2019-04-29 16:36:15 +10:00
if ( ! ( urls && urls . length > 0 ) ) { // No url supplied we are just checking which transports support this function on no url.
2019-05-30 15:57:37 +10:00
return this . _transports . filter ( ( t ) => ( t . validFor ( undefined , func , opts ) ) )
2018-04-06 19:28:04 +10:00
. map ( ( t ) => [ undefined , t ] ) ;
} else {
return [ ] . concat (
... urls . map ( ( url ) => typeof url === 'string' ? Url . parse ( url ) : url ) // parse URLs once
. map ( ( url ) =>
2019-05-30 15:57:37 +10:00
this . _transports . filter ( ( t ) => ( t . validFor ( url , func , opts ) ) ) // [ t1, t2 ]
2018-04-06 19:28:04 +10:00
. map ( ( t ) => [ url , t ] ) ) ) ; // [[ u, t1], [u, t2]]
}
}
2019-05-30 15:57:37 +10:00
static async p _urlsValidFor ( urls , func , opts ) {
2018-04-23 10:49:53 +10:00
// Need a async version of this for serviceworker and TransportsProxy
2019-05-30 15:57:37 +10:00
return this . validFor ( urls , func , opts ) . map ( ( ut ) => ut [ 0 ] ) ;
2018-04-23 10:49:53 +10:00
}
2018-07-06 20:41:45 -07:00
// SEE-OTHER-ADDTRANSPORT
2018-08-13 17:55:04 +10:00
static http ( ) {
2019-03-20 17:55:07 -07:00
// Find an http transport if it exists.
2018-04-06 19:28:04 +10:00
return Transports . _connected ( ) . find ( ( t ) => t . name === "HTTP" )
}
2019-03-20 11:35:26 -07:00
static wolk ( ) {
2019-03-20 17:55:07 -07:00
// Find a Wolk transport if it exists.
2019-03-20 11:35:26 -07:00
return Transports . _connected ( ) . find ( ( t ) => t . name === "WOLK" )
}
2018-08-13 17:55:04 +10:00
static ipfs ( ) {
2019-03-20 17:55:07 -07:00
// Find an ipfs transport if it exists, in particular, so YJS can use it.
2018-04-06 19:28:04 +10:00
return Transports . _connected ( ) . find ( ( t ) => t . name === "IPFS" )
}
2018-08-13 17:55:04 +10:00
static webtorrent ( ) {
2018-04-16 08:56:13 +10:00
// Find an ipfs transport if it exists, so for example ServiceWorker.p_respondWebTorrent can use it.
return Transports . _connected ( ) . find ( ( t ) => t . name === "WEBTORRENT" )
}
2018-08-13 17:55:04 +10:00
static gun ( ) {
2018-07-06 20:41:45 -07:00
// Find a GUN transport if it exists
return Transports . _connected ( ) . find ( ( t ) => t . name === "GUN" )
}
2018-04-06 19:28:04 +10:00
static async p _resolveNames ( urls ) {
/ * I f a n d o n l y i f T r a n s p o r t N A M E w a s l o a d e d ( i t m i g h t n o t b e a s i t d e p e n d s o n h i g h e r l e v e l c l a s s e s l i k e D o m a i n a n d S m a r t D i c t )
then resolve urls that might be names , returning a modified array .
* /
2018-09-24 19:14:20 +10:00
if ( this . mirror ) {
2019-02-08 13:18:38 +11:00
return Array . isArray ( urls ) ? this . gatewayUrls ( urls ) : this . gatewayUrl ( url ) ;
2018-09-24 19:14:20 +10:00
} else if ( this . namingcb ) {
2018-04-06 19:28:04 +10:00
return await this . namingcb ( urls ) ; // Array of resolved urls
} else {
return urls ;
}
}
2018-09-24 19:14:20 +10:00
2018-05-05 21:10:15 -07:00
static resolveNamesWith ( cb ) {
2018-04-06 19:28:04 +10:00
// Set a callback for p_resolveNames
this . namingcb = cb ;
}
2019-06-09 09:24:58 +10:00
static togglePaused ( name , cb ) {
2019-05-14 16:57:42 +10:00
/ *
Toggle a transport by name ,
name e . g . "HTTP"
cb ( err , status )
* /
const transport = this . _transports . find ( ( t ) => t . name === name ) ;
if ( ! transport ) {
cb ( undefined ) ;
} else {
transport . togglePaused ( t => cb ( null , t . status ) ) ;
}
}
// Storage of data
2018-08-13 17:55:04 +10:00
static async _p _rawstore ( tt , data ) {
2018-04-06 19:28:04 +10:00
// Internal method to store at known transports
let errs = [ ] ;
let rr = await Promise . all ( tt . map ( async function ( t ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Storing %d bytes to %s" , data . length , t . name ) ;
2018-08-13 17:55:04 +10:00
let url = await t . p _rawstore ( data ) ;
2019-04-16 12:22:33 +10:00
debug ( "Storing %d bytes to %s succeeded: %s" , data . length , t . name , url ) ;
2018-08-13 17:55:04 +10:00
return url ; //url
2018-04-06 19:28:04 +10:00
} catch ( err ) {
2019-04-16 12:22:33 +10:00
debug ( "Storing %d bytes to %s failed: %s" , data . length , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
errs . push ( err ) ;
return undefined ;
}
} ) ) ;
rr = rr . filter ( ( r ) => ! ! r ) ; // Trim any that had errors
if ( ! rr . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Storing %d bytes failed on all transports" , data . length ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; // New error with concatenated messages
}
return rr ;
}
2018-08-13 17:55:04 +10:00
static async p _rawstore ( data ) {
2018-04-06 19:28:04 +10:00
/ *
data : Raw data to store - typically a string , but its passed on unmodified here
returns : Array of urls of where stored
throws : TransportError with message being concatenated messages of transports if NONE of them succeed .
* /
let tt = this . validFor ( undefined , "store" ) . map ( ( [ u , t ] ) => t ) ; // Valid connected transports that support "store"
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Storing %d bytes failed: no transports available" , data . length ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( 'Transports.p_rawstore: Cant find transport for store' ) ;
}
2018-08-13 17:55:04 +10:00
return this . _p _rawstore ( tt , data ) ;
2018-04-06 19:28:04 +10:00
}
2018-08-29 14:16:09 +10:00
static async p _rawfetch ( urls , opts = { } ) {
2018-04-06 19:28:04 +10:00
/ *
Fetch the data for a url , transports act on the data , typically storing it .
urls : array of urls to retrieve ( any are valid )
opts {
start , integer - first byte wanted
end integer - last byte wanted ( note this is inclusive start = 0 , end = 1023 is 1024 bytes
timeoutMS integer - max time to wait on transports ( IPFS ) that support it
2019-05-30 15:57:37 +10:00
noCache bool - Skip caching ( passed to Transports )
2018-04-06 19:28:04 +10:00
}
returns : string - arbitrary bytes retrieved .
throws : TransportError with concatenated error messages if none succeed .
2018-06-11 18:07:58 -07:00
throws : CodingError if urls empty or [ undefined ... ]
2018-04-06 19:28:04 +10:00
* /
2018-07-06 14:09:44 -07:00
if ( ! urls . length ) throw new errors . TransportError ( "Transports.p_rawfetch given an empty list of urls" ) ;
2018-09-16 20:51:28 +10:00
let resolvedurls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert name to [urls]
2018-07-06 14:09:44 -07:00
if ( ! resolvedurls . length ) throw new errors . TransportError ( "Transports.p_rawfetch none of the urls resolved: " + urls ) ;
2019-05-30 15:57:37 +10:00
let tt = this . validFor ( resolvedurls , "fetch" , { noCache : opts . noCache } ) ; //[ [Url,t],[Url,t]] throws CodingError on empty /undefined urls
2018-04-06 19:28:04 +10:00
if ( ! tt . length ) {
2018-07-06 14:09:44 -07:00
throw new errors . TransportError ( "Transports.p_rawfetch cant find any transport for urls: " + resolvedurls ) ;
2018-04-06 19:28:04 +10:00
}
//With multiple transports, it should return when the first one returns something.
let errs = [ ] ;
let failedtransports = [ ] ; // Will accumulate any transports fail on before the success
for ( const [ url , t ] of tt ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Fetching %s via %s" , url . href , t . name ) ;
2018-04-06 19:28:04 +10:00
let data = await t . p _rawfetch ( url , opts ) ; // throws errors if fails or timesout
2019-04-16 12:22:33 +10:00
debug ( "Fetching %s via %s succeeded %d bytes" , url . href , t . name , data . length ) ;
2018-04-06 19:28:04 +10:00
if ( opts . relay && failedtransports . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Fetching attempting relay of %d bytes from %s to %o" , data . length , url . href , failedtransports . map ( t => t . name ) ) ;
2018-08-13 17:55:04 +10:00
this . _p _rawstore ( failedtransports , data )
2019-04-16 12:22:33 +10:00
. then ( uu => debug ( ` Fetching relayed %d bytes to %o ` , data . length , uu ) ) ; // Happening async, not waiting and dont care if fails
2018-04-06 19:28:04 +10:00
}
//END TODO-MULTI-GATEWAY
return data ;
} catch ( err ) {
failedtransports . push ( t ) ;
errs . push ( err ) ;
2019-04-16 12:22:33 +10:00
debug ( "Fetching %s via %s failed: %s" , url . href , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
// Don't throw anything here, loop round for next, only throw if drop out bottom
//TODO-MULTI-GATEWAY potentially copy from success to failed URLs.
}
}
2019-04-16 12:22:33 +10:00
debug ( "Fetching %o failed on all transports" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; //Throw err with combined messages if none succeed
}
2019-06-09 09:24:58 +10:00
static fetch ( urls , opts = { } , cb ) {
2019-04-16 12:22:52 +10:00
if ( typeof opts === "function" ) { cb = opts ; opts = { } ; }
const prom = this . p _rawfetch ( urls , opts ) ;
if ( cb ) { prom . then ( ( res ) => { try { cb ( null , res ) } catch ( err ) { debug ( "Uncaught error in fetch %O" , err ) } } ) . catch ( ( err ) => cb ( err ) ) ; } else { return prom ; } // Unpromisify pattern v5
}
2018-04-06 19:28:04 +10:00
2019-04-16 12:22:33 +10:00
// Seeding =====
2019-04-14 16:36:33 +10:00
// Similar to storing.
2019-05-14 16:57:42 +10:00
static seed ( { directoryPath = undefined , fileRelativePath = undefined , ipfsHash = undefined , urlToFile = undefined , torrentRelativePath = undefined } , cb ) {
2019-04-16 12:32:27 +10:00
/ *
ipfsHash : When passed as a parameter , its checked against whatever IPFS calculates .
Its reported , but not an error if it doesn ' t match . ( the cases are complex , for example the file might have been updated ) .
2019-06-09 09:24:58 +10:00
urlToFile : The URL where that file is available , this is to enable transports ( e . g . IPFS ) that just map an internal id to a URL .
directoryPath : Absolute path to the directory , for transports that think in terms of directories ( e . g . WebTorrent )
2019-04-16 12:32:27 +10:00
this is the unit corresponding to a torrent , and should be where the torrent file will be found or should be built
fileRelativePath : Path ( relative to directoryPath ) to the file to be seeded .
2019-06-09 09:24:58 +10:00
torrentRelativePath : Path within directory to torrent file if present .
2019-04-16 12:32:27 +10:00
* /
2019-04-14 16:36:33 +10:00
if ( cb ) { try { f . call ( this , cb ) } catch ( err ) { cb ( err ) } } else { return new Promise ( ( resolve , reject ) => { try { f . call ( this , ( err , res ) => { if ( err ) { reject ( err ) } else { resolve ( res ) } } ) } catch ( err ) { reject ( err ) } } ) } // Promisify pattern v2
function f ( cb1 ) {
let tt = this . validFor ( undefined , "seed" ) . map ( ( [ u , t ] ) => t ) ; // Valid connected transports that support "seed"
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Seeding: no transports available" ) ;
2019-04-14 16:36:33 +10:00
cb1 ( null ) ; // Its not (currently) an error to be unable to seed
} else {
const res = { } ;
each ( tt , // [ Transport]
2019-05-14 16:57:42 +10:00
( t , cb2 ) => t . seed ( { directoryPath , torrentRelativePath , fileRelativePath , ipfsHash , urlToFile } ,
2019-04-14 16:36:33 +10:00
( err , oneres ) => { res [ t . name ] = err ? { err : err . message } : oneres ; cb2 ( null ) } ) , // Its not an error for t.seed to fail - errors should have been logged by transports
( unusederr ) => cb1 ( null , res ) ) ; // Return result of any seeds that succeeded as e.g. { HTTP: {}, IPFS: {ipfsHash:} }
}
}
}
// List handling ===========================================
static async p _rawlist ( urls ) {
urls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert to a name
let tt = this . validFor ( urls , "list" ) ; // Valid connected transports that support "store"
if ( ! tt . length ) {
throw new errors . TransportError ( 'Transports.p_rawlist: Cant find transport to "list" urls:' + urls . join ( ',' ) ) ;
}
let errs = [ ] ;
let ttlines = await Promise . all ( tt . map ( async function ( [ url , t ] ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Listing %s via %s" , url , t . name ) ;
2019-04-14 16:36:33 +10:00
let res = await t . p _rawlist ( url ) ; // [sig]
2019-04-16 12:22:33 +10:00
debug ( "Listing %s via %s retrieved %d items" , url , t . name , res . length ) ;
2019-04-14 16:36:33 +10:00
return res ;
} catch ( err ) {
2019-04-16 12:22:33 +10:00
debug ( "Listing %s via %s failed: %s" , url , t . name , err . message ) ;
2019-04-14 16:36:33 +10:00
errs . push ( err ) ;
return [ ] ;
}
} ) ) ; // [[sig,sig],[sig,sig]]
if ( errs . length >= tt . length ) {
// All Transports failed (maybe only 1)
2019-04-16 12:22:33 +10:00
debug ( "Listing %o failed on all transports" , urls ) ;
2019-04-14 16:36:33 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; // New error with concatenated messages
}
let uniques = { } ; // Used to filter duplicates
return [ ] . concat ( ... ttlines )
. filter ( ( x ) => ( ! uniques [ x . signature ] && ( uniques [ x . signature ] = true ) ) ) ;
}
2018-08-13 17:55:04 +10:00
static async p _rawadd ( urls , sig ) {
2018-04-06 19:28:04 +10:00
/ *
urls : of lists to add to
sig : Sig to add
returns : undefined
throws : TransportError with message being concatenated messages of transports if NONE of them succeed .
* /
//TODO-MULTI-GATEWAY might be smarter about not waiting but Promise.race is inappropriate as returns after a failure as well.
urls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert to a name
let tt = this . validFor ( urls , "add" ) ; // Valid connected transports that support "store"
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Adding to %o failed: no transports available" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( 'Transports.p_rawstore: Cant find transport for urls:' + urls . join ( ',' ) ) ;
}
let errs = [ ] ;
await Promise . all ( tt . map ( async function ( [ u , t ] ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Adding to %s via %s" , u , t . name ) ;
2018-08-13 17:55:04 +10:00
await t . p _rawadd ( u , sig ) ; //undefined
2019-04-16 12:22:33 +10:00
debug ( "Adding to %s via %s succeeded" , u , t . name ) ;
2018-04-06 19:28:04 +10:00
return undefined ;
} catch ( err ) {
2019-04-16 12:22:33 +10:00
debug ( "Adding to %s via %s failed: %s" , u , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
errs . push ( err ) ;
return undefined ;
}
} ) ) ;
if ( errs . length >= tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Adding to %o failed on all transports" , urls ) ;
2018-04-06 19:28:04 +10:00
// All Transports failed (maybe only 1)
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; // New error with concatenated messages
}
return undefined ;
}
2018-04-23 10:49:53 +10:00
static listmonitor ( urls , cb , opts = { } ) {
2018-04-06 19:28:04 +10:00
/ *
Add a listmonitor for each transport - note this means if multiple transports support it , then will get duplicate events back if everyone else is notifying all of them .
* /
// Note cant do p_resolveNames since sync but should know real urls of resource by here.
this . validFor ( urls , "listmonitor" )
2018-08-13 17:55:04 +10:00
. map ( ( [ u , t ] ) => {
t . listmonitor ( u , cb , opts ) ;
2019-04-16 12:22:33 +10:00
debug ( "Monitoring list %s via %s" , u , t . name ) ;
2018-08-13 17:55:04 +10:00
} ) ;
2018-04-06 19:28:04 +10:00
}
2018-08-13 17:55:04 +10:00
static async p _newlisturls ( cl ) {
2018-04-06 19:28:04 +10:00
// Create a new list in any transport layer that supports lists.
// cl is a CommonList or subclass and can be used by the Transport to get info for choosing the list URL (normally it won't use it)
// Note that normally the CL will not have been stored yet, so you can't use its urls.
let uuu = await Promise . all ( this . validFor ( undefined , "newlisturls" )
2018-08-13 17:55:04 +10:00
. map ( ( [ u , t ] ) => t . p _newlisturls ( cl ) ) ) ; // [ [ priv, pub] [ priv, pub] [priv pub] ]
2018-04-06 19:28:04 +10:00
return [ uuu . map ( uu => uu [ 0 ] ) , uuu . map ( uu => uu [ 1 ] ) ] ; // [[ priv priv priv ] [ pub pub pub ] ]
}
// Stream handling ===========================================
2018-07-25 20:50:27 -07:00
//myArray[Math.floor(Math.random() * myArray.length)];
2018-10-02 15:18:40 +10:00
static async p _f _createReadStream ( urls , { wanturl = false , preferredTransports = [ ] } = { } ) { // Note options is options for selecting a stream, not the start/end in a createReadStream call
2018-04-06 19:28:04 +10:00
/ *
2018-10-15 10:31:04 +11:00
urls : Url or [ urls ] of the stream
2019-04-14 09:06:03 +10:00
wanturl True if want the URL of the stream ( for service workers )
2018-04-06 19:28:04 +10:00
returns : f ( opts ) => stream returning bytes from opts . start || start of file to opts . end - 1 || end of file
* /
2019-04-29 16:36:15 +10:00
// Find all the transports that CAN support this request
2018-04-16 08:56:13 +10:00
let tt = this . validFor ( urls , "createReadStream" , { } ) ; //[ [Url,t],[Url,t]] // Can pass options TODO-STREAM support options in validFor
2018-04-06 19:28:04 +10:00
if ( ! tt . length ) {
2019-04-29 16:36:15 +10:00
debug ( "Opening stream from %o failed: no transports available" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( "Transports.p_createReadStream cant find any transport for urls: " + urls ) ;
}
//With multiple transports, it should return when the first one returns something.
let errs = [ ] ;
2018-10-02 15:18:40 +10:00
2019-04-29 16:36:15 +10:00
// Select first from preferredTransports in the order presented, then the rest at random
2018-10-02 15:18:40 +10:00
tt . sort ( ( a , b ) =>
( ( preferredTransports . indexOf ( a [ 1 ] . name ) + 1 ) || 999 + Math . random ( ) ) - ( ( preferredTransports . indexOf ( b [ 1 ] . name ) + 1 ) || 999 + Math . random ( ) )
) ;
for ( const [ url , t ] of tt ) {
2018-04-06 19:28:04 +10:00
try {
2019-04-29 16:36:15 +10:00
debug ( "Opening stream from %s via %s" , url . href , t . name ) ;
2018-08-13 17:55:04 +10:00
let res = await t . p _f _createReadStream ( url , { wanturl } ) ;
2019-04-29 16:36:15 +10:00
debug ( "Opening stream from %s via %s succeeded" , url . href , t . name ) ;
2018-08-13 17:55:04 +10:00
return res ;
2018-04-06 19:28:04 +10:00
} catch ( err ) {
errs . push ( err ) ;
2019-04-29 16:36:15 +10:00
debug ( "Opening stream from %s via %s failed: %s" , url . href , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
// Don't throw anything here, loop round for next, only throw if drop out bottom
//TODO-MULTI-GATEWAY potentially copy from success to failed URLs.
}
}
2019-04-29 16:36:15 +10:00
debug ( "Opening stream from %o failed on all transports" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; //Throw err with combined messages if none succeed
2018-10-10 09:45:44 +11:00
}
2019-04-14 09:06:03 +10:00
static createReadStream ( urls , opts , cb ) {
2018-10-10 09:45:44 +11:00
/ *
Different interface , more suitable when just want a stream , now .
2018-10-15 10:31:04 +11:00
urls : Url or [ urls ] of the stream
2019-04-14 09:06:03 +10:00
opts {
start , end : First and last byte wanted ( default to 0. . . last )
preferredTransports : preferred order to select stream transports ( usually determined by application )
}
2018-10-10 09:45:44 +11:00
cb ( err , stream ) : Called with open readable stream from the net .
Returns promise if no cb
* /
if ( typeof opts === "function" ) { cb = opts ; opts = { start : 0 } ; } // Allow skipping opts
2019-04-14 09:06:03 +10:00
DwebTransports . p _f _createReadStream ( urls , { preferredTransports : ( opts . preferredTransports || [ ] ) } )
2018-10-10 09:45:44 +11:00
. then ( f => {
let s = f ( opts ) ;
if ( cb ) { cb ( null , s ) ; } else { return ( s ) ; } ; // Callback or resolve stream
} )
. catch ( err => {
2018-10-18 16:17:03 +13:00
if ( err instanceof errors . TransportError ) {
2018-10-15 10:31:04 +11:00
console . warn ( "Transports.createReadStream caught" , err . message ) ;
2018-10-10 09:45:44 +11:00
} else {
2018-10-15 10:31:04 +11:00
console . error ( "Transports.createReadStream caught" , err ) ;
2018-10-10 09:45:44 +11:00
}
if ( cb ) { cb ( err ) ; } else { reject ( err ) }
} ) ;
} ;
2018-04-06 19:28:04 +10:00
// KeyValue support ===========================================
2018-08-13 17:55:04 +10:00
static async p _get ( urls , keys ) {
2018-04-06 19:28:04 +10:00
/ *
Fetch the values for a url and one or more keys , transports act on the data , typically storing it .
urls : array of urls to retrieve ( any are valid )
keys : array of keys wanted or single key
returns : string - arbitrary bytes retrieved or dict of key : value
throws : TransportError with concatenated error messages if none succeed .
* /
let tt = this . validFor ( urls , "get" ) ; //[ [Url,t],[Url,t]]
2018-08-13 17:55:04 +10:00
let debug1 = Array . isArray ( keys ) ? ` ${ keys . length } keys ` : keys ; // "1 keys" or "foo"
2018-04-06 19:28:04 +10:00
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Getting %s from %o failed: no transports available" , debug1 , urls ) ;
2018-08-13 17:55:04 +10:00
throw new errors . TransportError ( "Transports.p_get cant find any transport to get keys from urls: " + urls ) ;
2018-04-06 19:28:04 +10:00
}
//With multiple transports, it should return when the first one returns something.
let errs = [ ] ;
for ( const [ url , t ] of tt ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Getting %s from %s via %s" , debug1 , url . href , t . name ) ;
2018-08-13 17:55:04 +10:00
let res = await t . p _get ( url , keys ) ; //TODO-MULTI-GATEWAY potentially copy from success to failed URLs.
2019-04-16 12:22:33 +10:00
debug ( "Getting %s from %s via %s succeeded length=%d" , debug1 , url . href , t . name , res . length ) ;
2018-08-13 17:55:04 +10:00
return res ;
2018-04-06 19:28:04 +10:00
} catch ( err ) {
errs . push ( err ) ;
2019-04-16 12:22:33 +10:00
debug ( "Getting %s from %s via %s failed: %s" , debug1 , url . href , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
// Don't throw anything here, loop round for next, only throw if drop out bottom
}
}
2019-04-16 12:22:33 +10:00
debug ( "Getting %s from %o failed on all transports" , debug1 , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; //Throw err with combined messages if none succeed
}
2018-08-13 17:55:04 +10:00
static async p _set ( urls , keyvalues , value ) {
2018-04-06 19:28:04 +10:00
/ * S e t a s e r i e s o f k e y / v a l u e s o r a s i n g l e v a l u e
keyvalues : Either dict or a string
value : if kv is a string , this is the value to set
throws : TransportError with message being concatenated messages of transports if NONE of them succeed .
* /
urls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert to a name
2018-08-13 17:55:04 +10:00
let debug1 = typeof keyvalues === "object" ? ` ${ keyvalues . length } keys ` : keyvalues ; // "1 keys" or "foo"
2018-04-06 19:28:04 +10:00
let tt = this . validFor ( urls , "set" ) ; //[ [Url,t],[Url,t]]
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Setting %s on %o failed: no transports available" , debug1 , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( "Transports.p_set cant find any transport for urls: " + urls ) ;
}
let errs = [ ] ;
let success = false ;
await Promise . all ( tt . map ( async function ( [ url , t ] ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Setting %s on %s via %s" , debug1 , url . href , t . name ) ;
2018-08-13 17:55:04 +10:00
await t . p _set ( url , keyvalues , value ) ;
2019-04-16 12:22:33 +10:00
debug ( "Setting %s on %s via %s succeeded" , debug1 , url . href , t . name ) ;
2018-04-06 19:28:04 +10:00
success = true ; // Any one success will return true
} catch ( err ) {
2019-04-16 12:22:33 +10:00
debug ( "Setting %s on %s via %s failed: %s" , debug1 , url . href , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
errs . push ( err ) ;
}
} ) ) ;
if ( ! success ) {
2019-04-16 12:22:33 +10:00
debug ( "Setting %s on %o failed on all transports" , debug1 , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; // New error with concatenated messages
}
}
2018-08-13 17:55:04 +10:00
static async p _delete ( urls , keys ) {
2018-04-06 19:28:04 +10:00
/ * D e l e t e a k e y o r a l i s t o f k e y s
kv : Either dict or a string
value : if kv is a string , this is the value to set
throws : TransportError with message being concatenated messages of transports if NONE of them succeed .
* /
urls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert to a name
2018-08-13 17:55:04 +10:00
let debug1 = Array . isArray ( keys ) ? ` ${ keys . length } keys ` : keys ; // "1 keys" or "foo"
2018-04-06 19:28:04 +10:00
let tt = this . validFor ( urls , "set" ) ; //[ [Url,t],[Url,t]]
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Deleting %s on %o failed: no transports available" , debug1 , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( "Transports.p_set cant find any transport for urls: " + urls ) ;
}
let errs = [ ] ;
let success = false ;
await Promise . all ( tt . map ( async function ( [ url , t ] ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Deleting %s on %s via %s" , debug1 , url . href , t . name ) ;
2018-08-13 17:55:04 +10:00
await t . p _delete ( url , keys ) ;
2019-04-16 12:22:33 +10:00
debug ( "Deleting %s on %s via %s succeeded" , debug1 , url . href , t . name ) ;
2018-04-06 19:28:04 +10:00
success = true ; // Any one success will return true
} catch ( err ) {
2019-04-16 12:22:33 +10:00
debug ( "Deleting %s on %s via %s failed: %s" , debug1 , url . href , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
errs . push ( err ) ;
}
} ) ) ;
if ( ! success ) {
2019-04-16 12:22:33 +10:00
debug ( "Deleting %s on %o failed on all transports" , debug1 , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; // New error with concatenated messages
}
}
2018-08-13 17:55:04 +10:00
static async p _keys ( urls ) {
2018-04-06 19:28:04 +10:00
/ *
Fetch the values for a url and one or more keys , transports act on the data , typically storing it .
urls : array of urls to retrieve ( any are valid )
keys : array of keys wanted
returns : string - arbitrary bytes retrieved or dict of key : value
throws : TransportError with concatenated error messages if none succeed .
* /
urls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert to a name
let tt = this . validFor ( urls , "keys" ) ; //[ [Url,t],[Url,t]]
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Getting all keys on %o failed: no transports available" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( "Transports.p_keys cant find any transport for urls: " + urls ) ;
}
2018-08-13 17:55:04 +10:00
//With multiple transports, it should return when the first one returns something. TODO make it return the aggregate
2018-04-06 19:28:04 +10:00
let errs = [ ] ;
for ( const [ url , t ] of tt ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Getting all keys on %s via %s" , url . href , t . name ) ;
2018-08-13 17:55:04 +10:00
let res = await t . p _keys ( url ) ; //TODO-MULTI-GATEWAY potentially copy from success to failed URLs.
2019-04-16 12:22:33 +10:00
debug ( "Getting all keys on %s via %s succeeded with %d keys" , url . href , t . name , res . length ) ;
2018-08-13 17:55:04 +10:00
return res ;
2018-04-06 19:28:04 +10:00
} catch ( err ) {
errs . push ( err ) ;
2019-04-16 12:22:33 +10:00
debug ( "Getting all keys on %s via %s failed: %s" , url . href , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
// Don't throw anything here, loop round for next, only throw if drop out bottom
}
}
2019-04-16 12:22:33 +10:00
debug ( "Getting all keys on %o failed on all transports" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; //Throw err with combined messages if none succeed
}
2018-08-13 17:55:04 +10:00
static async p _getall ( urls ) {
2018-04-06 19:28:04 +10:00
/ *
Fetch the values for a url and one or more keys , transports act on the data , typically storing it .
urls : array of urls to retrieve ( any are valid )
keys : array of keys wanted
returns : array of strings returned for the keys . //TODO consider issues around return a data type rather than array of strings
throws : TransportError with concatenated error messages if none succeed .
* /
urls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert to a name
let tt = this . validFor ( urls , "getall" ) ; //[ [Url,t],[Url,t]]
if ( ! tt . length ) {
2019-04-16 12:22:33 +10:00
debug ( "Getting all values on %o failed: no transports available" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( "Transports.p_getall cant find any transport for urls: " + urls ) ;
}
//With multiple transports, it should return when the first one returns something.
let errs = [ ] ;
for ( const [ url , t ] of tt ) {
try {
2019-04-16 12:22:33 +10:00
debug ( "Getting all values on %s via %s" , url . href , t . name ) ;
2018-08-13 17:55:04 +10:00
let res = await t . p _getall ( url ) ; //TODO-MULTI-GATEWAY potentially copy from success to failed URLs.
2019-04-16 12:22:33 +10:00
debug ( "Getting all values on %s via %s succeeded with %d values" , url . href , t . name , res . length ) ;
2018-08-13 17:55:04 +10:00
return res ;
2018-04-06 19:28:04 +10:00
} catch ( err ) {
errs . push ( err ) ;
2019-04-16 12:22:33 +10:00
debug ( "Getting all values on %s via %s failed: %s" , url . href , t . name , err . message ) ;
2018-04-06 19:28:04 +10:00
// Don't throw anything here, loop round for next, only throw if drop out bottom
}
}
2019-04-16 12:22:33 +10:00
debug ( "Getting all keys on %o failed on all transports" , urls ) ;
2018-04-06 19:28:04 +10:00
throw new errors . TransportError ( errs . map ( ( err ) => err . message ) . join ( ', ' ) ) ; //Throw err with combined messages if none succeed
}
2018-08-13 17:55:04 +10:00
static async p _newdatabase ( pubkey ) {
2018-04-06 19:28:04 +10:00
/ *
Create a new database in any transport layer that supports databases ( key value pairs ) .
pubkey : CommonList , KeyPair , or exported public key
resolves to : [ privateurl , publicurl ]
* /
let uuu = await Promise . all ( this . validFor ( undefined , "newdatabase" )
2018-08-13 17:55:04 +10:00
. map ( ( [ u , t ] ) => t . p _newdatabase ( pubkey ) ) ) ; // [ { privateurl, publicurl} { privateurl, publicurl} { privateurl, publicurl} ]
2018-04-06 19:28:04 +10:00
return { privateurls : uuu . map ( uu => uu . privateurl ) , publicurls : uuu . map ( uu => uu . publicurl ) } ; // { privateurls: [], publicurls: [] }
}
2018-08-13 17:55:04 +10:00
static async p _newtable ( pubkey , table ) {
2018-04-06 19:28:04 +10:00
/ *
Create a new table in any transport layer that supports the function ( key value pairs ) .
pubkey : CommonList , KeyPair , or exported public key
resolves to : [ privateurl , publicurl ]
* /
let uuu = await Promise . all ( this . validFor ( undefined , "newtable" )
2018-08-13 17:55:04 +10:00
. map ( ( [ u , t ] ) => t . p _newtable ( pubkey , table ) ) ) ; // [ [ priv, pub] [ priv, pub] [priv pub] ]
2018-04-06 19:28:04 +10:00
return { privateurls : uuu . map ( uu => uu . privateurl ) , publicurls : uuu . map ( uu => uu . publicurl ) } ; // {privateurls: [ priv priv priv ], publicurls: [ pub pub pub ] }
}
2018-08-13 17:55:04 +10:00
static async p _connection ( urls ) {
2018-04-06 19:28:04 +10:00
/ *
Do any asynchronous connection opening work prior to potentially synchronous methods ( like monitor )
* /
urls = await this . p _resolveNames ( urls ) ; // If naming is loaded then convert to a name
await Promise . all (
this . validFor ( urls , "connection" )
2018-10-15 10:31:04 +11:00
. map ( ( [ u , t ] ) => t . p _connection ( u ) ) ) ;
2018-04-06 19:28:04 +10:00
}
2018-08-13 17:55:04 +10:00
static monitor ( urls , cb , { current = false } = { } ) {
2018-04-06 19:28:04 +10:00
/ *
Add a listmonitor for each transport - note this means if multiple transports support it , then will get duplicate events back if everyone else is notifying all of them .
Stack : KVT ( ) | KVT . p _new => KVT . monitor => ( a : Transports . monitor => YJS . monitor ) ( b : dispatchEvent )
2018-07-09 19:08:56 -07:00
cb : function ( { type , key , value } )
current : If true then then send all current entries as well
2018-04-06 19:28:04 +10:00
* /
2018-08-13 17:55:04 +10:00
//Can't its async. urls = await this.p_resolveNames(urls); // If naming is loaded then convert to a name
2018-04-06 19:28:04 +10:00
this . validFor ( urls , "monitor" )
2018-08-13 17:55:04 +10:00
. map ( ( [ u , t ] ) => {
2019-04-16 12:22:33 +10:00
debug ( "Monitoring table %s via %s" , u , t . name ) ;
2018-08-13 17:55:04 +10:00
t . monitor ( u , cb , { current } )
}
) ;
2018-04-06 19:28:04 +10:00
}
2018-04-09 12:12:20 +10:00
// Setup and connection
2018-04-06 19:28:04 +10:00
static addtransport ( t ) {
/ *
Add a transport to _transports ,
* /
Transports . _transports . push ( t ) ;
}
// Setup Transports - setup0 is called once, and should return quickly, p_setup1 and p_setup2 are asynchronous and p_setup2 relies on p_setup1 having resolved.
2018-08-13 17:55:04 +10:00
static setup0 ( tabbrevs , options , cb ) {
2018-04-06 19:28:04 +10:00
/ *
Setup Transports for a range of classes
2018-04-08 14:53:19 +10:00
tabbrevs is abbreviation HTTP , IPFS , LOCAL or list of them e . g . "HTTP,IPFS"
2018-08-13 17:55:04 +10:00
cb is callback for when status changes , but there are no status changes here so its not called .
2018-04-06 19:28:04 +10:00
Handles "LOCAL" specially , turning into a HTTP to a local server ( for debugging )
returns array of transport instances
* /
// "IPFS" or "IPFS,LOCAL,HTTP"
2018-09-24 19:14:20 +10:00
let localoptions = { http : { urlbase : "http://localhost:4244" } } ; //TODO-MIRROR "localoptions" may not be needed any more
2018-04-08 14:53:19 +10:00
return tabbrevs . map ( ( tabbrev ) => {
2019-08-19 17:11:26 +10:00
//TODO-SPLIT-UPNEXT remove LOCAL - not used any more
2018-08-13 17:55:04 +10:00
let transportclass = this . _transportclasses [ ( tabbrev === "LOCAL" ) ? "HTTP" : tabbrev ] ;
2018-04-06 19:28:04 +10:00
if ( ! transportclass ) {
2019-04-16 12:22:33 +10:00
debug ( "Connection to %s unavailable" , tabbrev ) ;
2018-04-06 19:28:04 +10:00
return undefined ;
} else {
2019-04-16 12:22:33 +10:00
debug ( "Setting up connection to %s with options %o" , tabbrev , options ) ;
2018-08-13 17:55:04 +10:00
return transportclass . setup0 ( tabbrev === "LOCAL" ? localoptions : options ) ;
2018-04-06 19:28:04 +10:00
}
} ) . filter ( f => ! ! f ) ; // Trim out any undefined
}
2019-01-03 14:05:05 +11:00
static p _setup1 ( refreshstatus , cb ) {
2018-04-06 19:28:04 +10:00
/* Second stage of setup, connect if possible */
// Does all setup1a before setup1b since 1b can rely on ones with 1a, e.g. YJS relies on IPFS
2019-01-03 14:05:05 +11:00
const prom = Promise . all ( this . _transports
2018-05-05 21:04:15 -07:00
. filter ( ( t ) => ( ! this . _optionspaused . includes ( t . name ) ) )
2018-08-13 17:55:04 +10:00
. map ( ( t ) => {
2019-04-16 12:22:33 +10:00
debug ( "Connection stage 1 to %s" , t . name ) ;
2019-01-03 14:05:05 +11:00
return t . p _setup1 ( refreshstatus ) ;
2018-08-13 17:55:04 +10:00
} ) )
2019-01-07 09:03:55 +11:00
if ( cb ) { prom . catch ( ( err ) => cb ( err ) ) . then ( ( res ) => cb ( null , res ) ) ; } else { return prom ; } // This should be a standard unpromisify pattern
2018-04-06 19:28:04 +10:00
}
2019-01-03 14:05:05 +11:00
static p _setup2 ( refreshstatus , cb ) {
2018-04-06 19:28:04 +10:00
/* Second stage of setup, connect if possible */
// Does all setup1a before setup1b since 1b can rely on ones with 1a, e.g. YJS relies on IPFS
2019-01-03 14:05:05 +11:00
const prom = Promise . all ( this . _transports
2018-05-05 21:04:15 -07:00
. filter ( ( t ) => ( ! this . _optionspaused . includes ( t . name ) ) )
2018-08-13 17:55:04 +10:00
. map ( ( t ) => {
2019-04-16 12:22:33 +10:00
debug ( "Connection stage 2 to %s" , t . name ) ;
2019-01-03 14:05:05 +11:00
return t . p _setup2 ( refreshstatus ) ;
} ) ) ;
2019-01-07 09:03:55 +11:00
if ( cb ) { prom . catch ( ( err ) => cb ( err ) ) . then ( ( res ) => cb ( null , res ) ) ; } else { return prom ; } // This should be a standard unpromisify pattern
}
2019-06-26 20:42:37 +10:00
static p _stop ( refreshstatus , cb ) { //TODO-API cb
if ( cb ) { try { f . call ( this , cb ) } catch ( err ) { cb ( err ) } } else { return new Promise ( ( resolve , reject ) => { try { f . call ( this , ( err , res ) => { if ( err ) { reject ( err ) } else { resolve ( res ) } } ) } catch ( err ) { reject ( err ) } } ) } // Promisify pattern v2
2019-01-07 09:03:55 +11:00
/* Disconnect from all services, may not be able to reconnect */
2019-06-26 20:42:37 +10:00
//TODO rewrite with async/map
function f ( cb ) {
map ( this . _connected ( ) ,
( t , cb2 ) => {
debug ( "Stopping %s" , t . name ) ;
t . stop ( refreshstatus , cb2 ) ;
} ,
cb ) ;
}
2018-04-06 19:28:04 +10:00
}
2018-04-09 12:12:20 +10:00
static async refreshstatus ( t ) {
2018-04-15 17:34:20 +10:00
//Note "this' undefined as called as callback
2018-04-09 12:12:20 +10:00
let statusclasses = [ "transportstatus0" , "transportstatus1" , "transportstatus2" , "transportstatus3" , "transportstatus4" ] ;
let el = t . statuselement ;
if ( el ) {
el . classList . remove ( ... statusclasses ) ;
el . classList . add ( statusclasses [ t . status ] ) ;
}
2018-04-15 17:34:20 +10:00
if ( Transports . statuscb ) {
Transports . statuscb ( t ) ;
}
2018-04-09 12:12:20 +10:00
}
2019-01-03 14:05:05 +11:00
static connect ( options , cb ) {
2019-01-07 09:03:55 +11:00
const prom = this . p _connect ( options ) ;
if ( cb ) { prom . catch ( ( err ) => cb ( err ) ) . then ( ( res ) => cb ( null , res ) ) ; } else { return prom ; } // This should be a standard unpromisify pattern
2019-01-03 14:05:05 +11:00
}
2018-08-13 17:55:04 +10:00
static async p _connect ( options ) {
2018-04-09 12:12:20 +10:00
/ *
This is a standardish starting process , feel free to subclass or replace !
It will connect to a set of standard transports and is intended to work inside a browser .
2018-05-05 21:04:15 -07:00
options = { defaulttransports : [ "IPFS" ] , statuselement : el , http : { } , ipfs : { } ; paused : [ "IPFS" ] }
2018-04-09 12:12:20 +10:00
* /
try {
options = options || { } ;
let tabbrevs = options . transports ; // Array of transport abbreviations
2018-05-09 10:18:33 -07:00
this . _optionspaused = ( options . paused || [ ] ) . map ( n => n . toUpperCase ( ) ) ; // Array of transports paused - defaults to none, upper cased
2018-04-13 17:26:44 +10:00
if ( ! ( tabbrevs && tabbrevs . length ) ) { tabbrevs = options . defaulttransports || [ ] }
2019-06-14 15:27:07 +10:00
// WOLK is currently failing, and what is worse returning a data structure with a 404 instead of just failing
2019-06-19 14:54:12 +10:00
// GUN is turned off by default because it fills up localstorage on browser and stops working, https://github.com/internetarchive/dweb-archive/issues/106
2019-06-14 15:27:07 +10:00
//if (! tabbrevs.length) { tabbrevs = ["HTTP", "YJS", "IPFS", "WEBTORRENT", "GUN", "WOLK"]; } // SEE-OTHER-ADDTRANSPORT
2019-06-21 18:48:03 +10:00
if ( ! tabbrevs . length ) { tabbrevs = [ "HTTP" , "IPFS" , "WEBTORRENT" , "WOLK" ] ; } // SEE-OTHER-ADDTRANSPORT
2018-04-09 12:12:20 +10:00
tabbrevs = tabbrevs . map ( n => n . toUpperCase ( ) ) ;
2019-08-19 17:11:26 +10:00
let transports = this . setup0 ( tabbrevs , options ) ; // synchronous
2018-09-24 19:14:20 +10:00
[ "statuscb" , "mirror" ] . forEach ( k => { if ( options [ k ] ) this [ k ] = options [ k ] ; } )
2019-08-19 17:11:26 +10:00
//TODO move this to function and then call this from consumer
2018-04-09 12:12:20 +10:00
if ( ! ! options . statuselement ) {
2018-07-26 15:35:12 -07:00
let statuselement = options . statuselement ;
2018-04-10 09:56:48 +10:00
while ( statuselement . lastChild ) { statuselement . removeChild ( statuselement . lastChild ) ; } // Remove any exist status
2018-04-09 12:12:20 +10:00
statuselement . appendChild (
utils . createElement ( "UL" , { } , transports . map ( t => {
2019-03-20 11:35:26 -07:00
let el = utils . createElement ( "LI" ,
2019-04-16 12:32:27 +10:00
{ onclick : "this.source.togglePaused(DwebTransports.refreshstatus);" , source : t , name : t . name } , //TODO-SW figure out how t osend this back
t . name ) ;
2019-03-20 11:35:26 -07:00
t . statuselement = el ; // Save status element on transport
return el ;
} ) )
) ;
2018-04-09 12:12:20 +10:00
}
2019-08-19 17:11:26 +10:00
//TODO-SPLIT-UPNEXT invert this, use a waterfall here, and then wrap in promise for p_setup, then put load's here
2018-08-13 17:55:04 +10:00
await this . p _setup1 ( this . refreshstatus ) ;
await this . p _setup2 ( this . refreshstatus ) ;
2019-04-16 12:22:33 +10:00
debug ( "Connection completed to %o" , this . _connected ( ) . map ( t => t . name ) )
2018-04-09 12:12:20 +10:00
} catch ( err ) {
console . error ( "ERROR in p_connect:" , err . message ) ;
throw ( err ) ;
}
}
2018-05-30 20:11:33 -07:00
static async p _urlsFrom ( url ) {
2018-04-09 12:12:20 +10:00
/ * U t i l i t y t o c o n v e r t t o u r l s f o r m w a n t e d f o r T r a n s p o r t s f u n c t i o n s , e . g . f r o m u s e r i n p u t
url : Array of urls , or string representing url or representing array of urls
return : Array of strings representing url
* /
if ( typeof ( url ) === "string" ) {
if ( url [ 0 ] === '[' )
url = JSON . parse ( url ) ;
else if ( url . includes ( ',' ) )
url = url . split ( ',' ) ;
else
url = [ url ] ;
}
if ( ! Array . isArray ( url ) ) throw new Error ( ` Unparsable url: ${ url } ` ) ;
return url ;
}
2018-04-23 10:49:53 +10:00
static async p _httpfetchurl ( urls ) {
2018-04-13 17:26:44 +10:00
/ *
Utility to take a array of Transport urls , convert back to a single url that can be used for a fetch , typically
this is done when cant handle a stream , so want to give the url to the < VIDEO > tag .
* /
return Transports . http ( ) . _url ( urls . find ( u => ( u . startsWith ( "contenthash" ) || u . startsWith ( "http" ) ) ) , "content/rawfetch" ) ;
}
2018-09-24 19:14:20 +10:00
static canonicalName ( url , options = { } ) {
/ *
Utility function to convert a variety of missentered , or presumed names into a canonical result that can be resolved or passed to a transport
* /
if ( typeof url !== "string" ) url = Url . parse ( url ) . href ;
// In patterns below http or https; and :/ or :// are treated the same
2019-03-20 11:35:26 -07:00
const gateways = [ "dweb.me" , "ipfs.io" ] ; // Known gateways, may dynamically load this at some point
2019-03-20 17:55:07 -07:00
// SEE-OTHER-ADDTRANSPORT
2019-03-20 11:35:26 -07:00
const protocols = [ "ipfs" , "gun" , "magnet" , "yjs" , "wolk" , "arc" , "contenthash" , "http" , "https" ] ;
2018-09-24 19:14:20 +10:00
const protocolsWantingDomains = [ "arc" , "http" , "https" ] ;
const gatewaypatts = [ // Must be before patts because gateway names often start with a valid proto
/^http[s]?:[/]+([^/]+)[/](\w+)[/](.*)/i , // https://(gateway)/proto/(internal) + gateway in list (IPFS gateways. dweb.me)
]
const patts = [ // No overlap between patts & arcpatts, so order unimportant
/^dweb:[/]+(\w+)[/]+(.*)/i , // dweb://(proto)/(internal)
/^\w+:[/]+(\w+)[/](.*)/i , // proto1://proto2//(internal) - maybe only match if proto1=proto2 (must be before proto:/internal)
2018-10-22 17:46:25 -07:00
/^(\w+):[/]*(.*)/i , // (proto)://(internal) # must be after proto1://proto2
2018-09-24 19:14:20 +10:00
/^[/]*(\w+)[/](.*)/i , // /(proto)//(internal) - maybe only match if proto1=proto2
/^[/]*dweb[/]*(\w+)[/](.*)/i , // /dweb/(proto)//(internal)
]
const arcpatts = [ // No overlap between patts & arcpatts, so order unimportant
/^http[s]?:[/]+[^/]+[/](archive).(org)[/]*(.*)/i , // https://localhost;123/(archive.org)/(internal)
2019-05-22 17:04:33 +10:00
/^http[s]?:[/]+[^/]+[/]arc[/](archive).(org)[/]*(.*)/i , // https://localhost;123/arc/(archive.org)/(internal)
2018-09-24 19:14:20 +10:00
/^http[s]?:[/]+dweb.(\w+)[.]([^/]+)[/]*(.*)/i , // https://dweb.(proto).(dom.ain)/(internal) # Before dweb.dom.ain
// /^http[s]?:[/]+dweb.([^/]+[.][^/]+[/]*.*)/i, // https://dweb.(dom.ain)/internal) or https://dweb.(domain) Handled by coe on recognizing above
/^(http[s])?:[/]+([^/]+)[/]+(.*)/i , // https://dom.ain/pa/th
]
for ( let patt of gatewaypatts ) {
let rr = url . match ( patt ) ;
if ( rr && gateways . includes ( rr [ 1 ] ) && protocols . includes ( rr [ 2 ] ) )
return { proto : rr [ 2 ] , internal : rr [ 3 ] } ;
2018-04-06 19:28:04 +10:00
}
2018-09-24 19:14:20 +10:00
for ( let patt of arcpatts ) {
let rr = url . match ( patt ) ;
if ( rr ) {
if ( protocols . includes ( rr [ 1 ] ) ) {
// arc (and possibly others) want the domain as part of the internal
return { proto : rr [ 1 ] , internal : ( protocolsWantingDomains . includes ( rr [ 1 ] ) ? [ rr [ 2 ] , rr [ 3 ] ] . join ( '/' ) : rr [ 3 ] ) } ;
} else {
return { proto : "arc" , internal : [ [ rr [ 1 ] , rr [ 2 ] ] . join ( '.' ) , rr [ 3 ] ] . join ( '/' ) } ;
}
}
} ;
for ( let patt of patts ) {
let rr = url . match ( patt ) ;
if ( rr && protocols . includes ( rr [ 1 ] ) )
return { proto : rr [ 1 ] , internal : rr [ 2 ] } ;
} ;
return undefined ;
}
static canonicalUrl ( url , options = { } ) {
let o = this . canonicalName ( url , options ) ;
return o . protocol + ":/" + o . internal ;
}
2019-02-08 13:18:38 +11:00
static _o2url ( o ) {
return [ "http" , "https" ] . includes ( o . proto ) ? [ o . proto , o . internal ] . join ( '://' ) // Shouldnt be relative
: o . proto ? [ this . mirror , o . proto , o . internal ] . join ( '/' )
: o . internal ; // Uncanonicalizable
}
2018-09-24 19:14:20 +10:00
static gatewayUrl ( url ) {
2018-10-22 17:46:25 -07:00
// Convert url to gateway url, if not canonicalizable then just pass the url along
2018-09-24 19:14:20 +10:00
let o = Transports . canonicalName ( url ) ;
2019-02-08 13:18:38 +11:00
return ! o ? url : this . _o2url ( o )
}
static gatewayUrls ( urls ) { //TODO-API
// Convert urls to gateway urls,
// Easier to work on single form [ { proto, internal } ]
const oo = urls . map ( url => Transports . canonicalName ( url ) || { proto : undefined , internal : url } ) ; //if not canonicalizable then just pass the url along
const oArc = oo . filter ( o => [ "arc" ] . includes ( o . proto ) ) ; // Prefered
return ( oArc . length ? oArc : oo ) // Prefered if have them, else others
. map ( o => this . _o2url ( o ) )
2018-04-06 19:28:04 +10:00
}
}
Transports . _transports = [ ] ; // Array of transport instances connected
2019-09-08 12:20:40 +10:00
Transports . naming = naming ;
2019-09-02 06:27:22 +10:00
Transports . namingcb = p _namingcb ; // Will be defined by the naming component (turns URLs for names into URLs for transport)
2018-04-06 19:28:04 +10:00
Transports . _transportclasses = { } ; // Pointers to classes whose code is loaded.
2019-02-22 16:31:17 +11:00
Transports . httptools = httptools ; // Static http tools
2018-04-06 19:28:04 +10:00
exports = module . exports = Transports ;