fluencelabs/dweb-transports
1
0
Fork 0
mirror of https://github.com/fluencelabs/dweb-transports synced 2025-03-15 02:20:50 +00:00
Code Issues Projects Releases Wiki Activity

Added support for seed() - tested in IPFS

Browse Source
This commit is contained in:
Mitra Ardron 2019-04-14 16:36:33 +10:00
parent 765e4e7607
commit b010189f0b
2 changed files with 87 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
TransportIPFS.js
View File

@ -60,11 +60,15 @@ class TransportIPFS extends Transport {
constructor(options) {
super(options);
if (options.urlUrlstore) {
this.urlUrlstore = options.urlUrlstore;
delete options.urlUrlstore;
}
this.ipfs = undefined; // Undefined till start IPFS
this.options = options; // Dictionary of options
this.name = "IPFS"; // For console log etc
this.supportURLs = ['ipfs'];
this.supportFunctions = ['fetch', 'store', 'createReadStream']; // Does not support reverse
this.supportFunctions = ['fetch', 'store', 'seed', 'createReadStream']; // Does not support reverse
this.status = Transport.STATUS_LOADED;
}
@ -348,6 +352,36 @@ class TransportIPFS extends Transport {
return TransportIPFS.urlFrom(res);
}
seed({directoryPath=undefined, fileRelativePath=undefined, ipfsHash=undefined, urlToFile=undefined}, cb) {
/* Note always passed a cb by Transports.seed - no need to support Promise here
ipfsHash: IPFS hash if known (usually not known)
urlToFile: Where the IPFS server can get the file - must be live before this called as will fetch and hash
TODO support directoryPath/fileRelativePath, but to working around IPFS limitation in https://github.com/ipfs/go-ipfs/issues/4224 will need to check relative to IPFS home, and if not symlink it and add symlink
TODO maybe support adding raw data (using add)
Note neither js-ipfs-http-client nor js-ipfs appear to support urlstore yet, see https://github.com/ipfs/js-ipfs-http-client/issues/969
*/
// This is the URL that the IPFS server uses to get the file from the local mirrorHttp
if (!(this.urlUrlstore && urlToFile)) { // Not doing IPFS
debug("IPFS.seed support requires urlUrlstore and urlToFile"); // Report, though Transports.seed currently ignores this
cb(new Error("IPFS.seed support requires urlUrlstore and urlToFile")); // Report, though Transports.seed currently ignores this
} else {
// Building by hand becase of lack of support in js-ipfs-http-client
const url = `${this.urlUrlstore}?arg=${encodeURIComponent(urlToFile)}`;
// Have to be careful to avoid loops, the call to addIPFS should only be after file is retrieved and cached, and then addIPFS shouldnt be called if already cached
httptools.p_GET(url, {retries:0}, (err, res) => {
if (err) {
debug("IPFS.seed for %s failed in http: %s", urlToFile, err.message);
cb(err); // Note error currently ignored in Transports
} else {
debug("Added %s to IPFS key=", urlToFile, res.Key);
// Check for mismatch - this isn't an error, for example it could be an updated file, old IPFS hash will now fail, but is out of date and shouldnt be shared
if (ipfsHash && ipfsHash !== res.Key) { debug("ipfs hash doesnt match expected metadata has %s daemon returned %s", ipfsHash, res.Key); }
cb(null, res)
}
})
}
}
async p_f_createReadStream(url, {wanturl=false}={}) {
/*
Fetch bytes progressively, using a node.js readable stream, based on a url of the form:

82
Transports.js
View File

@ -4,6 +4,7 @@ const utils = require('./utils');
//process.env.DEBUG = "dweb-transports"; //TODO-DEBUG set at top level
const debugtransports = require('debug')('dweb-transports');
const httptools = require('./httptools');
const each = require('async/each');
class Transports {
/*
@ -56,7 +57,7 @@ class Transports {
throws: CodingError if urls empty or [undefined...]
*/
if (typeof urls === "string") urls = [urls];
if (!((urls && urls[0]) || ["store", "newlisturls", "newdatabase", "newtable"].includes(func))) {
if (!((urls && urls[0]) || ["store", "newlisturls", "newdatabase", "newtable", "seed"].includes(func))) {
console.error("Transports.validFor called with invalid arguments: urls=", urls, "func=", func); // FOr debugging old calling patterns with [ undefined ]
return [];
}
@ -157,35 +158,6 @@ class Transports {
}
return this._p_rawstore(tt, data);
}
static async p_rawlist(urls) {
urls = await this.p_resolveNames(urls); // If naming is loaded then convert to a name
let tt = this.validFor(urls, "list"); // Valid connected transports that support "store"
if (!tt.length) {
throw new errors.TransportError('Transports.p_rawlist: Cant find transport to "list" urls:'+urls.join(','));
}
let errs = [];
let ttlines = await Promise.all(tt.map(async function([url, t]) {
try {
debugtransports("Listing %s via %s", url, t.name);
let res = await t.p_rawlist(url); // [sig]
debugtransports("Listing %s via %s retrieved %d items", url, t.name, res.length);
return res;
} catch(err) {
debugtransports("Listing %s via %s failed: %s", url, t.name, err.message);
errs.push(err);
return [];
}
})); // [[sig,sig],[sig,sig]]
if (errs.length >= tt.length) {
// All Transports failed (maybe only 1)
debugtransports("Listing %o failed on all transports", urls);
throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); // New error with concatenated messages
}
let uniques = {}; // Used to filter duplicates
return [].concat(...ttlines)
.filter((x) => (!uniques[x.signature] && (uniques[x.signature] = true)));
}
static async p_rawfetch(urls, opts={}) {
/*
Fetch the data for a url, transports act on the data, typically storing it.
@ -234,6 +206,56 @@ class Transports {
throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); //Throw err with combined messages if none succeed
}
// Seeding =====
// Similar to storing.
static seed({directoryPath=undefined, fileRelativePath=undefined, ipfsHash=undefined, urlToFile=undefined}, cb) {
if (cb) { try { f.call(this, cb) } catch(err) { cb(err)}} else { return new Promise((resolve, reject) => { try { f.call(this, (err, res) => { if (err) {reject(err)} else {resolve(res)} })} catch(err) {reject(err)}})} // Promisify pattern v2
function f(cb1) {
let tt = this.validFor(undefined, "seed").map(([u, t]) => t); // Valid connected transports that support "seed"
if (!tt.length) {
debugtransports("Seeding: no transports available");
cb1(null); // Its not (currently) an error to be unable to seed
} else {
const res = {};
each(tt, // [ Transport]
(t, cb2) => t.seed({directoryPath, fileRelativePath, ipfsHash, urlToFile},
(err, oneres) => { res[t.name] = err ? { err: err.message } : oneres; cb2(null)}), // Its not an error for t.seed to fail - errors should have been logged by transports
(unusederr) => cb1(null, res)); // Return result of any seeds that succeeded as e.g. { HTTP: {}, IPFS: {ipfsHash:} }
}
}
}
// List handling ===========================================
static async p_rawlist(urls) {
urls = await this.p_resolveNames(urls); // If naming is loaded then convert to a name
let tt = this.validFor(urls, "list"); // Valid connected transports that support "store"
if (!tt.length) {
throw new errors.TransportError('Transports.p_rawlist: Cant find transport to "list" urls:'+urls.join(','));
}
let errs = [];
let ttlines = await Promise.all(tt.map(async function([url, t]) {
try {
debugtransports("Listing %s via %s", url, t.name);
let res = await t.p_rawlist(url); // [sig]
debugtransports("Listing %s via %s retrieved %d items", url, t.name, res.length);
return res;
} catch(err) {
debugtransports("Listing %s via %s failed: %s", url, t.name, err.message);
errs.push(err);
return [];
}
})); // [[sig,sig],[sig,sig]]
if (errs.length >= tt.length) {
// All Transports failed (maybe only 1)
debugtransports("Listing %o failed on all transports", urls);
throw new errors.TransportError(errs.map((err)=>err.message).join(', ')); // New error with concatenated messages
}
let uniques = {}; // Used to filter duplicates
return [].concat(...ttlines)
.filter((x) => (!uniques[x.signature] && (uniques[x.signature] = true)));
}
static async p_rawadd(urls, sig) {
/*
urls: of lists to add to