diff --git a/TransportIPFS.js b/TransportIPFS.js index f7ff7c8..cef25de 100644 --- a/TransportIPFS.js +++ b/TransportIPFS.js @@ -60,11 +60,15 @@ class TransportIPFS extends Transport { constructor(options) { super(options); + if (options.urlUrlstore) { + this.urlUrlstore = options.urlUrlstore; + delete options.urlUrlstore; + } this.ipfs = undefined; // Undefined till start IPFS this.options = options; // Dictionary of options this.name = "IPFS"; // For console log etc this.supportURLs = ['ipfs']; - this.supportFunctions = ['fetch', 'store', 'createReadStream']; // Does not support reverse + this.supportFunctions = ['fetch', 'store', 'seed', 'createReadStream']; // Does not support reverse this.status = Transport.STATUS_LOADED; } @@ -348,6 +352,36 @@ class TransportIPFS extends Transport { return TransportIPFS.urlFrom(res); } + seed({directoryPath=undefined, fileRelativePath=undefined, ipfsHash=undefined, urlToFile=undefined}, cb) { + /* Note always passed a cb by Transports.seed - no need to support Promise here + ipfsHash: IPFS hash if known (usually not known) + urlToFile: Where the IPFS server can get the file - must be live before this called as will fetch and hash + TODO support directoryPath/fileRelativePath, but to working around IPFS limitation in https://github.com/ipfs/go-ipfs/issues/4224 will need to check relative to IPFS home, and if not symlink it and add symlink + TODO maybe support adding raw data (using add) + + Note neither js-ipfs-http-client nor js-ipfs appear to support urlstore yet, see https://github.com/ipfs/js-ipfs-http-client/issues/969 + */ + // This is the URL that the IPFS server uses to get the file from the local mirrorHttp + if (!(this.urlUrlstore && urlToFile)) { // Not doing IPFS + debug("IPFS.seed support requires urlUrlstore and urlToFile"); // Report, though Transports.seed currently ignores this + cb(new Error("IPFS.seed support requires urlUrlstore and urlToFile")); // Report, though Transports.seed currently ignores this + } else { + // Building by hand becase of lack of support in js-ipfs-http-client + const url = `${this.urlUrlstore}?arg=${encodeURIComponent(urlToFile)}`; + // Have to be careful to avoid loops, the call to addIPFS should only be after file is retrieved and cached, and then addIPFS shouldnt be called if already cached + httptools.p_GET(url, {retries:0}, (err, res) => { + if (err) { + debug("IPFS.seed for %s failed in http: %s", urlToFile, err.message); + cb(err); // Note error currently ignored in Transports + } else { + debug("Added %s to IPFS key=", urlToFile, res.Key); + // Check for mismatch - this isn't an error, for example it could be an updated file, old IPFS hash will now fail, but is out of date and shouldnt be shared + if (ipfsHash && ipfsHash !== res.Key) { debug("ipfs hash doesnt match expected metadata has %s daemon returned %s", ipfsHash, res.Key); } + cb(null, res) + } + }) + } + } async p_f_createReadStream(url, {wanturl=false}={}) { /* Fetch bytes progressively, using a node.js readable stream, based on a url of the form: diff --git a/Transports.js b/Transports.js index 2330670..e3893e7 100644 --- a/Transports.js +++ b/Transports.js @@ -4,6 +4,7 @@ const utils = require('./utils'); //process.env.DEBUG = "dweb-transports"; //TODO-DEBUG set at top level const debugtransports = require('debug')('dweb-transports'); const httptools = require('./httptools'); +const each = require('async/each'); class Transports { /* @@ -56,7 +57,7 @@ class Transports { throws: CodingError if urls empty or [undefined...] */ if (typeof urls === "string") urls = [urls]; - if (!((urls && urls[0]) || ["store", "newlisturls", "newdatabase", "newtable"].includes(func))) { + if (!((urls && urls[0]) || ["store", "newlisturls", "newdatabase", "newtable", "seed"].includes(func))) { console.error("Transports.validFor called with invalid arguments: urls=", urls, "func=", func); // FOr debugging old calling patterns with [ undefined ] return []; } @@ -157,35 +158,6 @@ class Transports { } return this._p_rawstore(tt, data); } - static async p_rawlist(urls) { - urls = await this.p_resolveNames(urls); // If naming is loaded then convert to a name - let tt = this.validFor(urls, "list"); // Valid connected transports that support "store" - if (!tt.length) { - throw new errors.TransportError('Transports.p_rawlist: Cant find transport to "list" urls:'+urls.join(',')); - } - let errs = []; - let ttlines = await Promise.all(tt.map(async function([url, t]) { - try { - debugtransports("Listing %s via %s", url, t.name); - let res = await t.p_rawlist(url); // [sig] - debugtransports("Listing %s via %s retrieved %d items", url, t.name, res.length); - return res; - } catch(err) { - debugtransports("Listing %s via %s failed: %s", url, t.name, err.message); - errs.push(err); - return []; - } - })); // [[sig,sig],[sig,sig]] - if (errs.length >= tt.length) { - // All Transports failed (maybe only 1) - debugtransports("Listing %o failed on all transports", urls); - throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); // New error with concatenated messages - } - let uniques = {}; // Used to filter duplicates - return [].concat(...ttlines) - .filter((x) => (!uniques[x.signature] && (uniques[x.signature] = true))); - } - static async p_rawfetch(urls, opts={}) { /* Fetch the data for a url, transports act on the data, typically storing it. @@ -234,6 +206,56 @@ class Transports { throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); //Throw err with combined messages if none succeed } + // Seeding ===== + // Similar to storing. + static seed({directoryPath=undefined, fileRelativePath=undefined, ipfsHash=undefined, urlToFile=undefined}, cb) { + if (cb) { try { f.call(this, cb) } catch(err) { cb(err)}} else { return new Promise((resolve, reject) => { try { f.call(this, (err, res) => { if (err) {reject(err)} else {resolve(res)} })} catch(err) {reject(err)}})} // Promisify pattern v2 + function f(cb1) { + let tt = this.validFor(undefined, "seed").map(([u, t]) => t); // Valid connected transports that support "seed" + if (!tt.length) { + debugtransports("Seeding: no transports available"); + cb1(null); // Its not (currently) an error to be unable to seed + } else { + const res = {}; + each(tt, // [ Transport] + (t, cb2) => t.seed({directoryPath, fileRelativePath, ipfsHash, urlToFile}, + (err, oneres) => { res[t.name] = err ? { err: err.message } : oneres; cb2(null)}), // Its not an error for t.seed to fail - errors should have been logged by transports + (unusederr) => cb1(null, res)); // Return result of any seeds that succeeded as e.g. { HTTP: {}, IPFS: {ipfsHash:} } + } + } + } + + // List handling =========================================== + + static async p_rawlist(urls) { + urls = await this.p_resolveNames(urls); // If naming is loaded then convert to a name + let tt = this.validFor(urls, "list"); // Valid connected transports that support "store" + if (!tt.length) { + throw new errors.TransportError('Transports.p_rawlist: Cant find transport to "list" urls:'+urls.join(',')); + } + let errs = []; + let ttlines = await Promise.all(tt.map(async function([url, t]) { + try { + debugtransports("Listing %s via %s", url, t.name); + let res = await t.p_rawlist(url); // [sig] + debugtransports("Listing %s via %s retrieved %d items", url, t.name, res.length); + return res; + } catch(err) { + debugtransports("Listing %s via %s failed: %s", url, t.name, err.message); + errs.push(err); + return []; + } + })); // [[sig,sig],[sig,sig]] + if (errs.length >= tt.length) { + // All Transports failed (maybe only 1) + debugtransports("Listing %o failed on all transports", urls); + throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); // New error with concatenated messages + } + let uniques = {}; // Used to filter duplicates + return [].concat(...ttlines) + .filter((x) => (!uniques[x.signature] && (uniques[x.signature] = true))); + } + static async p_rawadd(urls, sig) { /* urls: of lists to add to