mirror of
https://github.com/fluencelabs/dweb-transports
synced 2025-03-15 18:30:49 +00:00
Added support for seed() - tested in IPFS
This commit is contained in:
parent
765e4e7607
commit
b010189f0b
@ -60,11 +60,15 @@ class TransportIPFS extends Transport {
|
|||||||
|
|
||||||
constructor(options) {
|
constructor(options) {
|
||||||
super(options);
|
super(options);
|
||||||
|
if (options.urlUrlstore) {
|
||||||
|
this.urlUrlstore = options.urlUrlstore;
|
||||||
|
delete options.urlUrlstore;
|
||||||
|
}
|
||||||
this.ipfs = undefined; // Undefined till start IPFS
|
this.ipfs = undefined; // Undefined till start IPFS
|
||||||
this.options = options; // Dictionary of options
|
this.options = options; // Dictionary of options
|
||||||
this.name = "IPFS"; // For console log etc
|
this.name = "IPFS"; // For console log etc
|
||||||
this.supportURLs = ['ipfs'];
|
this.supportURLs = ['ipfs'];
|
||||||
this.supportFunctions = ['fetch', 'store', 'createReadStream']; // Does not support reverse
|
this.supportFunctions = ['fetch', 'store', 'seed', 'createReadStream']; // Does not support reverse
|
||||||
this.status = Transport.STATUS_LOADED;
|
this.status = Transport.STATUS_LOADED;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -348,6 +352,36 @@ class TransportIPFS extends Transport {
|
|||||||
return TransportIPFS.urlFrom(res);
|
return TransportIPFS.urlFrom(res);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
seed({directoryPath=undefined, fileRelativePath=undefined, ipfsHash=undefined, urlToFile=undefined}, cb) {
|
||||||
|
/* Note always passed a cb by Transports.seed - no need to support Promise here
|
||||||
|
ipfsHash: IPFS hash if known (usually not known)
|
||||||
|
urlToFile: Where the IPFS server can get the file - must be live before this called as will fetch and hash
|
||||||
|
TODO support directoryPath/fileRelativePath, but to working around IPFS limitation in https://github.com/ipfs/go-ipfs/issues/4224 will need to check relative to IPFS home, and if not symlink it and add symlink
|
||||||
|
TODO maybe support adding raw data (using add)
|
||||||
|
|
||||||
|
Note neither js-ipfs-http-client nor js-ipfs appear to support urlstore yet, see https://github.com/ipfs/js-ipfs-http-client/issues/969
|
||||||
|
*/
|
||||||
|
// This is the URL that the IPFS server uses to get the file from the local mirrorHttp
|
||||||
|
if (!(this.urlUrlstore && urlToFile)) { // Not doing IPFS
|
||||||
|
debug("IPFS.seed support requires urlUrlstore and urlToFile"); // Report, though Transports.seed currently ignores this
|
||||||
|
cb(new Error("IPFS.seed support requires urlUrlstore and urlToFile")); // Report, though Transports.seed currently ignores this
|
||||||
|
} else {
|
||||||
|
// Building by hand becase of lack of support in js-ipfs-http-client
|
||||||
|
const url = `${this.urlUrlstore}?arg=${encodeURIComponent(urlToFile)}`;
|
||||||
|
// Have to be careful to avoid loops, the call to addIPFS should only be after file is retrieved and cached, and then addIPFS shouldnt be called if already cached
|
||||||
|
httptools.p_GET(url, {retries:0}, (err, res) => {
|
||||||
|
if (err) {
|
||||||
|
debug("IPFS.seed for %s failed in http: %s", urlToFile, err.message);
|
||||||
|
cb(err); // Note error currently ignored in Transports
|
||||||
|
} else {
|
||||||
|
debug("Added %s to IPFS key=", urlToFile, res.Key);
|
||||||
|
// Check for mismatch - this isn't an error, for example it could be an updated file, old IPFS hash will now fail, but is out of date and shouldnt be shared
|
||||||
|
if (ipfsHash && ipfsHash !== res.Key) { debug("ipfs hash doesnt match expected metadata has %s daemon returned %s", ipfsHash, res.Key); }
|
||||||
|
cb(null, res)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
async p_f_createReadStream(url, {wanturl=false}={}) {
|
async p_f_createReadStream(url, {wanturl=false}={}) {
|
||||||
/*
|
/*
|
||||||
Fetch bytes progressively, using a node.js readable stream, based on a url of the form:
|
Fetch bytes progressively, using a node.js readable stream, based on a url of the form:
|
||||||
|
@ -4,6 +4,7 @@ const utils = require('./utils');
|
|||||||
//process.env.DEBUG = "dweb-transports"; //TODO-DEBUG set at top level
|
//process.env.DEBUG = "dweb-transports"; //TODO-DEBUG set at top level
|
||||||
const debugtransports = require('debug')('dweb-transports');
|
const debugtransports = require('debug')('dweb-transports');
|
||||||
const httptools = require('./httptools');
|
const httptools = require('./httptools');
|
||||||
|
const each = require('async/each');
|
||||||
|
|
||||||
class Transports {
|
class Transports {
|
||||||
/*
|
/*
|
||||||
@ -56,7 +57,7 @@ class Transports {
|
|||||||
throws: CodingError if urls empty or [undefined...]
|
throws: CodingError if urls empty or [undefined...]
|
||||||
*/
|
*/
|
||||||
if (typeof urls === "string") urls = [urls];
|
if (typeof urls === "string") urls = [urls];
|
||||||
if (!((urls && urls[0]) || ["store", "newlisturls", "newdatabase", "newtable"].includes(func))) {
|
if (!((urls && urls[0]) || ["store", "newlisturls", "newdatabase", "newtable", "seed"].includes(func))) {
|
||||||
console.error("Transports.validFor called with invalid arguments: urls=", urls, "func=", func); // FOr debugging old calling patterns with [ undefined ]
|
console.error("Transports.validFor called with invalid arguments: urls=", urls, "func=", func); // FOr debugging old calling patterns with [ undefined ]
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
@ -157,35 +158,6 @@ class Transports {
|
|||||||
}
|
}
|
||||||
return this._p_rawstore(tt, data);
|
return this._p_rawstore(tt, data);
|
||||||
}
|
}
|
||||||
static async p_rawlist(urls) {
|
|
||||||
urls = await this.p_resolveNames(urls); // If naming is loaded then convert to a name
|
|
||||||
let tt = this.validFor(urls, "list"); // Valid connected transports that support "store"
|
|
||||||
if (!tt.length) {
|
|
||||||
throw new errors.TransportError('Transports.p_rawlist: Cant find transport to "list" urls:'+urls.join(','));
|
|
||||||
}
|
|
||||||
let errs = [];
|
|
||||||
let ttlines = await Promise.all(tt.map(async function([url, t]) {
|
|
||||||
try {
|
|
||||||
debugtransports("Listing %s via %s", url, t.name);
|
|
||||||
let res = await t.p_rawlist(url); // [sig]
|
|
||||||
debugtransports("Listing %s via %s retrieved %d items", url, t.name, res.length);
|
|
||||||
return res;
|
|
||||||
} catch(err) {
|
|
||||||
debugtransports("Listing %s via %s failed: %s", url, t.name, err.message);
|
|
||||||
errs.push(err);
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
})); // [[sig,sig],[sig,sig]]
|
|
||||||
if (errs.length >= tt.length) {
|
|
||||||
// All Transports failed (maybe only 1)
|
|
||||||
debugtransports("Listing %o failed on all transports", urls);
|
|
||||||
throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); // New error with concatenated messages
|
|
||||||
}
|
|
||||||
let uniques = {}; // Used to filter duplicates
|
|
||||||
return [].concat(...ttlines)
|
|
||||||
.filter((x) => (!uniques[x.signature] && (uniques[x.signature] = true)));
|
|
||||||
}
|
|
||||||
|
|
||||||
static async p_rawfetch(urls, opts={}) {
|
static async p_rawfetch(urls, opts={}) {
|
||||||
/*
|
/*
|
||||||
Fetch the data for a url, transports act on the data, typically storing it.
|
Fetch the data for a url, transports act on the data, typically storing it.
|
||||||
@ -234,6 +206,56 @@ class Transports {
|
|||||||
throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); //Throw err with combined messages if none succeed
|
throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); //Throw err with combined messages if none succeed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Seeding =====
|
||||||
|
// Similar to storing.
|
||||||
|
static seed({directoryPath=undefined, fileRelativePath=undefined, ipfsHash=undefined, urlToFile=undefined}, cb) {
|
||||||
|
if (cb) { try { f.call(this, cb) } catch(err) { cb(err)}} else { return new Promise((resolve, reject) => { try { f.call(this, (err, res) => { if (err) {reject(err)} else {resolve(res)} })} catch(err) {reject(err)}})} // Promisify pattern v2
|
||||||
|
function f(cb1) {
|
||||||
|
let tt = this.validFor(undefined, "seed").map(([u, t]) => t); // Valid connected transports that support "seed"
|
||||||
|
if (!tt.length) {
|
||||||
|
debugtransports("Seeding: no transports available");
|
||||||
|
cb1(null); // Its not (currently) an error to be unable to seed
|
||||||
|
} else {
|
||||||
|
const res = {};
|
||||||
|
each(tt, // [ Transport]
|
||||||
|
(t, cb2) => t.seed({directoryPath, fileRelativePath, ipfsHash, urlToFile},
|
||||||
|
(err, oneres) => { res[t.name] = err ? { err: err.message } : oneres; cb2(null)}), // Its not an error for t.seed to fail - errors should have been logged by transports
|
||||||
|
(unusederr) => cb1(null, res)); // Return result of any seeds that succeeded as e.g. { HTTP: {}, IPFS: {ipfsHash:} }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// List handling ===========================================
|
||||||
|
|
||||||
|
static async p_rawlist(urls) {
|
||||||
|
urls = await this.p_resolveNames(urls); // If naming is loaded then convert to a name
|
||||||
|
let tt = this.validFor(urls, "list"); // Valid connected transports that support "store"
|
||||||
|
if (!tt.length) {
|
||||||
|
throw new errors.TransportError('Transports.p_rawlist: Cant find transport to "list" urls:'+urls.join(','));
|
||||||
|
}
|
||||||
|
let errs = [];
|
||||||
|
let ttlines = await Promise.all(tt.map(async function([url, t]) {
|
||||||
|
try {
|
||||||
|
debugtransports("Listing %s via %s", url, t.name);
|
||||||
|
let res = await t.p_rawlist(url); // [sig]
|
||||||
|
debugtransports("Listing %s via %s retrieved %d items", url, t.name, res.length);
|
||||||
|
return res;
|
||||||
|
} catch(err) {
|
||||||
|
debugtransports("Listing %s via %s failed: %s", url, t.name, err.message);
|
||||||
|
errs.push(err);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
})); // [[sig,sig],[sig,sig]]
|
||||||
|
if (errs.length >= tt.length) {
|
||||||
|
// All Transports failed (maybe only 1)
|
||||||
|
debugtransports("Listing %o failed on all transports", urls);
|
||||||
|
throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); // New error with concatenated messages
|
||||||
|
}
|
||||||
|
let uniques = {}; // Used to filter duplicates
|
||||||
|
return [].concat(...ttlines)
|
||||||
|
.filter((x) => (!uniques[x.signature] && (uniques[x.signature] = true)));
|
||||||
|
}
|
||||||
|
|
||||||
static async p_rawadd(urls, sig) {
|
static async p_rawadd(urls, sig) {
|
||||||
/*
|
/*
|
||||||
urls: of lists to add to
|
urls: of lists to add to
|
||||||
|
Loading…
x
Reference in New Issue
Block a user