mirror of
https://github.com/fluencelabs/dweb-transports
synced 2025-03-15 18:30:49 +00:00
396 lines
20 KiB
JavaScript
396 lines
20 KiB
JavaScript
/*
|
|
This is a shim to the IPFS library, (Lists are handled in YJS or OrbitDB)
|
|
See https://github.com/ipfs/js-ipfs but note its often out of date relative to the generic API doc.
|
|
*/
|
|
|
|
const httptools = require('./httptools'); // Expose some of the httptools so that IPFS can use it as a backup
|
|
|
|
// IPFS components
|
|
const IPFS = require('ipfs');
|
|
const CID = require('cids');
|
|
//Removed next two as not needed if use "Kludge" flagged below.
|
|
//const dagPB = require('ipld-dag-pb');
|
|
//const DAGNode = dagPB.DAGNode; // So can check its type
|
|
const unixFs = require('ipfs-unixfs');
|
|
|
|
// Library packages other than IPFS
|
|
const Url = require('url');
|
|
const stream = require('readable-stream'); // Needed for the pullthrough - this is NOT Ipfs streams
|
|
// Alternative to through - as used in WebTorrent
|
|
|
|
// Utility packages (ours) And one-liners
|
|
//No longer reqd: const promisify = require('promisify-es6');
|
|
//const makepromises = require('./utils/makepromises'); // Replaced by direct call to promisify
|
|
|
|
// Other Dweb modules
|
|
const errors = require('./Errors'); // Standard Dweb Errors
|
|
const Transport = require('./Transport.js'); // Base class for TransportXyz
|
|
const Transports = require('./Transports'); // Manage all Transports that are loaded
|
|
const utils = require('./utils'); // Utility functions
|
|
|
|
const defaultoptions = {
|
|
repo: '/tmp/dweb_ipfsv2700', //TODO-IPFS think through where, esp for browser
|
|
//init: false,
|
|
//start: false,
|
|
//TODO-IPFS-Q how is this decentralized - can it run offline? Does it depend on star-signal.cloud.ipfs.team
|
|
config: {
|
|
// Addresses: { Swarm: [ '/dns4/star-signal.cloud.ipfs.team/wss/p2p-webrtc-star']}, // For Y - same as defaults
|
|
// Addresses: { Swarm: [ ] }, // Disable WebRTC to test browser crash, note disables Y so doesnt work.
|
|
//Addresses: {Swarm: ['/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star']}, // from https://github.com/ipfs/js-ipfs#faq 2017-12-05 as alternative to webrtc works sort-of
|
|
Bootstrap: ['/dns4/dweb.me/tcp/4245/wss/ipfs/QmPNgKEjC7wkpu3aHUzKKhZmbEfiGzL5TP1L8zZoHJyXZW'], // Supposedly connects to Dweb IPFS instance, but doesnt work (nor does ".../wss/...")
|
|
},
|
|
//init: true, // Comment out for Y
|
|
EXPERIMENTAL: {
|
|
pubsub: true
|
|
}
|
|
};
|
|
|
|
class TransportIPFS extends Transport {
|
|
/*
|
|
IPFS specific transport
|
|
|
|
Fields:
|
|
ipfs: object returned when starting IPFS
|
|
TODO - this is not complete
|
|
*/
|
|
|
|
constructor(options, verbose) {
|
|
super(options, verbose);
|
|
this.ipfs = undefined; // Undefined till start IPFS
|
|
this.options = options; // Dictionary of options
|
|
this.name = "IPFS"; // For console log etc
|
|
this.supportURLs = ['ipfs'];
|
|
this.supportFunctions = ['fetch', 'store']; // Does not support reverse, createReadStream fails on files uploaded with urlstore TODO reenable when Kyle fixes urlstore
|
|
this.status = Transport.STATUS_LOADED;
|
|
}
|
|
|
|
/*
|
|
_makepromises() {
|
|
//Utility function to promisify Block
|
|
//Replaced promisified utility since only two to promisify
|
|
//this.promisified = {ipfs:{}};
|
|
//makepromises(this.ipfs, this.promisified.ipfs, [ { block: ["put", "get"] }]); // Has to be after this.ipfs defined
|
|
this.promisified = { ipfs: { block: {
|
|
put: promisify(this.ipfs.block.put),
|
|
get: promisify(this.ipfs.block.get)
|
|
}}}
|
|
}
|
|
*/
|
|
p_ipfsstart(verbose) {
|
|
/*
|
|
Just start IPFS - not Y (note used with "yarrays" and will be used for non-IPFS list management)
|
|
Note - can't figure out how to use async with this, as we resolve the promise based on the event callback
|
|
*/
|
|
const self = this;
|
|
return new Promise((resolve, reject) => {
|
|
this.ipfs = new IPFS(this.options);
|
|
this.ipfs.on('ready', () => {
|
|
//this._makepromises();
|
|
resolve();
|
|
});
|
|
this.ipfs.on('error', (err) => reject(err));
|
|
})
|
|
.then(() => self.ipfs.version())
|
|
.then((version) => console.log('IPFS READY',version))
|
|
.catch((err) => {
|
|
console.log("Error caught in p_ipfsstart");
|
|
throw(err);
|
|
});
|
|
}
|
|
|
|
static setup0(options, verbose) {
|
|
/*
|
|
First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.
|
|
*/
|
|
const combinedoptions = Transport.mergeoptions(defaultoptions, options.ipfs);
|
|
if (verbose) console.log("IPFS loading options %o", combinedoptions);
|
|
const t = new TransportIPFS(combinedoptions, verbose); // Note doesnt start IPFS
|
|
Transports.addtransport(t);
|
|
return t;
|
|
}
|
|
|
|
async p_setup1(verbose, cb) {
|
|
try {
|
|
if (verbose) console.log("IPFS starting and connecting");
|
|
this.status = Transport.STATUS_STARTING; // Should display, but probably not refreshed in most case
|
|
if (cb) cb(this);
|
|
await this.p_ipfsstart(verbose); // Throws Error("websocket error") and possibly others.
|
|
this.status = await this.p_status(verbose);
|
|
} catch(err) {
|
|
console.error(this.name, "failed to connect", err);
|
|
this.status = Transport.STATUS_FAILED;
|
|
}
|
|
if (cb) cb(this);
|
|
return this;
|
|
}
|
|
|
|
async p_status(verbose) {
|
|
/*
|
|
Return a numeric code for the status of a transport.
|
|
*/
|
|
this.status = (await this.ipfs.isOnline()) ? Transport.STATUS_CONNECTED : Transport.STATUS_FAILED;
|
|
return super.p_status(verbose);
|
|
}
|
|
|
|
// Everything else - unless documented here - should be opaque to the actual structure of a CID
|
|
// or a url. This code may change as its not clear (from IPFS docs) if this is the right mapping.
|
|
static urlFrom(unknown) {
|
|
/*
|
|
Convert a CID into a standardised URL e.g. ipfs:/ipfs/abc123
|
|
*/
|
|
if (unknown instanceof CID)
|
|
return "ipfs:/ipfs/"+unknown.toBaseEncodedString();
|
|
if (typeof unknown === "object" && unknown.hash) // e.g. from files.add
|
|
return "ipfs:/ipfs/"+unknown.hash;
|
|
if (typeof unknown === "string") // Not used currently
|
|
return "ipfs:/ipfs/"+unknown;
|
|
throw new errors.CodingError("TransportIPFS.urlFrom: Cant convert to url from",unknown);
|
|
}
|
|
|
|
static cidFrom(url) {
|
|
/*
|
|
Convert a URL e.g. ipfs:/ipfs/abc123 into a CID structure suitable for retrieval
|
|
url: String of form "ipfs://ipfs/<hash>" or parsed URL or CID
|
|
returns: CID
|
|
throws: TransportError if cant convert
|
|
*/
|
|
if (url instanceof CID) return url;
|
|
if (typeof(url) === "string") url = Url.parse(url);
|
|
if (url && url["pathname"]) { // On browser "instanceof Url" isn't valid)
|
|
const patharr = url.pathname.split('/');
|
|
if ((!["ipfs:","dweb:"].includes(url.protocol)) || (patharr[1] !== 'ipfs') || (patharr.length < 3))
|
|
throw new errors.TransportError("TransportIPFS.cidFrom bad format for url should be dweb: or ipfs:/ipfs/...: " + url.href);
|
|
if (patharr.length > 3)
|
|
throw new errors.TransportError("TransportIPFS.cidFrom not supporting paths in url yet, should be dweb: or ipfs:/ipfs/...: " + url.href);
|
|
return new CID(patharr[2]);
|
|
} else {
|
|
throw new errors.CodingError("TransportIPFS.cidFrom: Cant convert url",url);
|
|
}
|
|
}
|
|
|
|
static _stringFrom(url) {
|
|
// Tool for ipfsFrom and ipfsGatewayFrom
|
|
if (url instanceof CID)
|
|
return "/ipfs/"+url.toBaseEncodedString();
|
|
if (typeof url === 'object' && url.path) { // It better be URL which unfortunately is hard to test
|
|
return url.path;
|
|
}
|
|
}
|
|
static ipfsFrom(url) {
|
|
/*
|
|
Convert to a ipfspath i.e. /ipfs/Qm....
|
|
Required because of strange differences in APIs between files.cat and dag.get see https://github.com/ipfs/js-ipfs/issues/1229
|
|
*/
|
|
url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/
|
|
if (url.indexOf('/ipfs/') > -1) {
|
|
return url.slice(url.indexOf('/ipfs/'));
|
|
}
|
|
throw new errors.CodingError(`TransportIPFS.ipfsFrom: Cant convert url ${url} into a path starting /ipfs/`);
|
|
}
|
|
|
|
static ipfsGatewayFrom(url) {
|
|
/*
|
|
url: CID, Url, or a string
|
|
returns: https://ipfs.io/ipfs/<cid>
|
|
*/
|
|
url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/
|
|
if (url.indexOf('/ipfs/') > -1) {
|
|
return "https://ipfs.io" + url.slice(url.indexOf('/ipfs/'));
|
|
}
|
|
throw new errors.CodingError(`TransportIPFS.ipfsGatewayFrom: Cant convert url ${url} into a path starting /ipfs/`);
|
|
}
|
|
|
|
static multihashFrom(url) {
|
|
if (url instanceof CID)
|
|
return cid.toBaseEncodedString();
|
|
if (typeof url === 'object' && url.path)
|
|
url = url.path; // /ipfs/Q...
|
|
if (typeof(url) === "string") {
|
|
const idx = url.indexOf("/ipfs/");
|
|
if (idx > -1) {
|
|
return url.slice(idx+6);
|
|
}
|
|
}
|
|
throw new errors.CodingError(`Cant turn ${url} into a multihash`);
|
|
}
|
|
|
|
async p_rawfetch(url, {verbose=false, timeoutMS=60000, relay=false}={}) {
|
|
/*
|
|
Fetch some bytes based on a url of the form ipfs:/ipfs/Qm..... or ipfs:/ipfs/z.... .
|
|
No assumption is made about the data in terms of size or structure, nor can we know whether it was created with dag.put or ipfs add or http /api/v0/add/
|
|
|
|
Where required by the underlying transport it should retrieve a number if its "blocks" and concatenate them.
|
|
Returns a new Promise that resolves currently to a string.
|
|
There may also be need for a streaming version of this call, at this point undefined since we havent (currently) got a use case..
|
|
|
|
:param string url: URL of object being retrieved {ipfs|dweb}:/ipfs/<cid> or /
|
|
:param boolean verbose: true for debugging output
|
|
:resolve buffer: Return the object being fetched. (may in the future return a stream and buffer externally)
|
|
:throws: TransportError if url invalid - note this happens immediately, not as a catch in the promise
|
|
*/
|
|
if (verbose) console.log("IPFS p_rawfetch", utils.stringfrom(url));
|
|
if (!url) throw new errors.CodingError("TransportIPFS.p_rawfetch: requires url");
|
|
const cid = TransportIPFS.cidFrom(url); // Throws TransportError if url bad
|
|
const ipfspath = TransportIPFS.ipfsFrom(url) // Need because dag.get has different requirement than file.cat
|
|
|
|
try {
|
|
const res = await utils.p_timeout(this.ipfs.dag.get(cid), timeoutMS, "Timed out IPFS fetch of "+TransportIPFS._stringFrom(cid)); // Will reject and throw TimeoutError if times out
|
|
// noinspection Annotator
|
|
if (res.remainderPath.length)
|
|
{ // noinspection ExceptionCaughtLocallyJS
|
|
throw new errors.TransportError("Not yet supporting paths in p_rawfetch");
|
|
} //TODO-PATH
|
|
let buff;
|
|
//if (res.value instanceof DAGNode) { // Its file or something added with the HTTP API for example, TODO not yet handling multiple files
|
|
if (res.value.constructor.name === "DAGNode") { // Kludge to replace above, as its not matching the type against the "require" above.
|
|
if (verbose) console.log("IPFS p_rawfetch looks like its a file", url);
|
|
//console.log("Case a or b" - we can tell the difference by looking at (res.value._links.length > 0) but dont need to
|
|
// as since we dont know if we are on node or browser best way is to try the files.cat and if it fails try the block to get an approximate file);
|
|
// Works on Node, but fails on Chrome, cant figure out how to get data from the DAGNode otherwise (its the wrong size)
|
|
buff = await this.ipfs.files.cat(ipfspath); //See js-ipfs v0.27 version and https://github.com/ipfs/js-ipfs/issues/1229 and https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#cat
|
|
|
|
/* Was needed on v0.26, not on v0.27
|
|
if (buff.length === 0) { // Hit the Chrome bug
|
|
// This will get a file padded with ~14 bytes - 4 at front, 4 at end and cant find the other 6 !
|
|
// but it seems to work for PDFs which is what I'm testing on.
|
|
if (verbose) console.log("Kludge alert - files.cat fails in Chrome, trying block.get");
|
|
let blk = await this.promisified.ipfs.block.get(cid);
|
|
buff = blk.data;
|
|
}
|
|
END of v0.26 version */
|
|
} else { //c: not a file
|
|
buff = res.value;
|
|
}
|
|
if (verbose) console.log(`IPFS fetched ${buff.length} from ${ipfspath}`);
|
|
return buff;
|
|
} catch (err) { // TimeoutError or could be some other error from IPFS etc
|
|
console.log("Caught misc error in TransportIPFS.p_rawfetch trying IPFS", err.message);
|
|
try {
|
|
let ipfsurl = TransportIPFS.ipfsGatewayFrom(url);
|
|
return await utils.p_timeout(
|
|
httptools.p_GET(ipfsurl), // Returns a buffer
|
|
timeoutMS, "Timed out IPFS fetch of "+ipfsurl)
|
|
} catch (err) {
|
|
console.log("Caught misc error in TransportIPFS.p_rawfetch trying gateway", err.message);
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
|
|
async p_rawstore(data, {verbose}) {
|
|
/*
|
|
Store a blob of data onto the decentralised transport.
|
|
Returns a promise that resolves to the url of the data
|
|
|
|
:param string|Buffer data: Data to store - no assumptions made to size or content
|
|
:param boolean verbose: true for debugging output
|
|
:resolve string: url of data stored
|
|
*/
|
|
console.assert(data, "TransportIPFS.p_rawstore: requires data");
|
|
const buf = (data instanceof Buffer) ? data : new Buffer(data);
|
|
//return this.promisified.ipfs.block.put(buf).then((block) => block.cid)
|
|
//https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md#dagput
|
|
//let res = await this.ipfs.dag.put(buf,{ format: 'dag-cbor', hashAlg: 'sha2-256' });
|
|
const res = (await this.ipfs.files.add(buf,{ "cid-version": 1, hashAlg: 'sha2-256'}))[0];
|
|
return TransportIPFS.urlFrom(res);
|
|
}
|
|
|
|
// Based on https://github.com/ipfs/js-ipfs/pull/1231/files
|
|
|
|
async p_offsetStream(stream, links, startByte, endByte) {
|
|
let streamPosition = 0
|
|
try {
|
|
for (let l in links) {
|
|
const link = links[l];
|
|
if (!stream.writable) { return } // The stream has been closed
|
|
// DAGNode Links report unixfs object data sizes 14 bytes larger due to the protobuf wrapper
|
|
const bytesInLinkedObjectData = link.size - 14
|
|
if (startByte > (streamPosition + bytesInLinkedObjectData)) {
|
|
// Start byte is after this block so skip it
|
|
streamPosition += bytesInLinkedObjectData;
|
|
} else if (endByte && endByte < streamPosition) { // TODO-STREAM this is copied from https://github.com/ipfs/js-ipfs/pull/1231/files but I think it should be endByte <= since endByte is first byte DONT want
|
|
// End byte was before this block so skip it
|
|
streamPosition += bytesInLinkedObjectData;
|
|
} else {
|
|
let lmh = link.multihash;
|
|
let data;
|
|
await this.ipfs.object.data(lmh)
|
|
.then ((d) => unixFs.unmarshal(d).data)
|
|
.then ((d) => data = d )
|
|
.catch((err) => {console.log("XXX@289 err=",err);});
|
|
if (!stream.writable) { return; } // The stream was closed while we were getting data
|
|
const length = data.length;
|
|
if (startByte > streamPosition && startByte < (streamPosition + length)) {
|
|
// If the startByte is in the current block, skip to the startByte
|
|
data = data.slice(startByte - streamPosition);
|
|
}
|
|
console.log(`Writing ${data.length} to stream`)
|
|
stream.write(data);
|
|
streamPosition += length;
|
|
}
|
|
}
|
|
} catch(err) {
|
|
console.log(err.message);
|
|
}
|
|
}
|
|
async p_f_createReadStream(url, {verbose=false}={}) { // Asynchronously return a function that can be used in createReadStream
|
|
verbose = true;
|
|
if (verbose) console.log("p_f_createReadStream",url);
|
|
const mh = TransportIPFS.multihashFrom(url);
|
|
const links = await this.ipfs.object.links(mh);
|
|
let throughstream; //Holds pointer to stream between calls.
|
|
const self = this;
|
|
function crs(opts) { // This is a synchronous function
|
|
// Return a readable stream that provides the bytes between offsets "start" and "end" inclusive
|
|
console.log("opts=",JSON.stringify(opts));
|
|
/* Can replace rest of crs with this when https://github.com/ipfs/js-ipfs/pull/1231/files lands (hopefully v0.28.3)
|
|
return self.ipfs.catReadableStream(mh, opts ? opts.start : 0, opts && opts.end) ? opts.end+1 : undefined)
|
|
*/
|
|
if (!opts) return throughstream; //TODO-STREAM unclear why called without opts - take this out when figured out
|
|
if (throughstream && throughstream.destroy) throughstream.destroy();
|
|
throughstream = new stream.PassThrough();
|
|
|
|
self.p_offsetStream( // Ignore promise returned, this will write to the stream asynchronously
|
|
throughstream,
|
|
links, // Uses the array of links created above in this function
|
|
opts ? opts.start : 0,
|
|
(opts && opts.end) ? opts.end : undefined);
|
|
return throughstream;
|
|
}
|
|
return crs;
|
|
}
|
|
|
|
static async p_test(opts, verbose) {
|
|
if (verbose) {console.log("TransportIPFS.test")}
|
|
try {
|
|
const transport = await this.p_setup(opts, verbose); // Assumes IPFS already setup
|
|
if (verbose) console.log(transport.name,"setup");
|
|
const res = await transport.p_status(verbose);
|
|
console.assert(res === Transport.STATUS_CONNECTED)
|
|
|
|
let urlqbf;
|
|
const qbf = "The quick brown fox";
|
|
const qbf_url = "ipfs:/ipfs/zdpuAscRnisRkYnEyJAp1LydQ3po25rCEDPPEDMymYRfN1yPK"; // Expected url
|
|
const testurl = "1114"; // Just a predictable number can work with
|
|
const url = await transport.p_rawstore(qbf, {verbose});
|
|
if (verbose) console.log("rawstore returned", url);
|
|
const newcid = TransportIPFS.cidFrom(url); // Its a CID which has a buffer in it
|
|
console.assert(url === qbf_url, "url should match url from rawstore");
|
|
const cidmultihash = url.split('/')[2]; // Store cid from first block in form of multihash
|
|
const newurl = TransportIPFS.urlFrom(newcid);
|
|
console.assert(url === newurl, "Should round trip");
|
|
urlqbf = url;
|
|
const data = await transport.p_rawfetch(urlqbf, {verbose});
|
|
console.assert(data.toString() === qbf, "Should fetch block stored above");
|
|
//console.log("TransportIPFS test complete");
|
|
return transport
|
|
} catch(err) {
|
|
console.log("Exception thrown in TransportIPFS.test:", err.message);
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
}
|
|
Transports._transportclasses["IPFS"] = TransportIPFS;
|
|
exports = module.exports = TransportIPFS;
|