From d8508e07334114e475bef7dc853ee0b4c8d37e3b Mon Sep 17 00:00:00 2001 From: Mitra Ardron Date: Fri, 13 Sep 2019 15:05:44 +1000 Subject: [PATCH] v0.2.1 fix bugs relating to require and refactor script loading --- dist/dweb-transports-bundle.js | 161935 +++++++++++++++++++++++++- dist/dweb-transports-bundle.js.map | 1 + package.json | 2 +- 3 files changed, 161573 insertions(+), 365 deletions(-) create mode 100644 dist/dweb-transports-bundle.js.map diff --git a/dist/dweb-transports-bundle.js b/dist/dweb-transports-bundle.js index 77af7ef..1b7a6a3 100644 --- a/dist/dweb-transports-bundle.js +++ b/dist/dweb-transports-bundle.js @@ -1,389 +1,20569 @@ -!function(e){var t={};function n(r){if(t[r])return t[r].exports;var i=t[r]={i:r,l:!1,exports:{}};return e[r].call(i.exports,i,i.exports,n),i.l=!0,i.exports}n.m=e,n.c=t,n.d=function(e,t,r){n.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:r})},n.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},n.t=function(e,t){if(1&t&&(e=n(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var i in e)n.d(r,i,function(t){return e[t]}.bind(null,i));return r},n.n=function(e){var t=e&&e.__esModule?function t(){return e.default}:function t(){return e};return n.d(t,"a",t),t},n.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},n.p="",n(n.s=309)}([function(e,t,n){"use strict";(function(e){ +/******/ (function(modules) { // webpackBootstrap +/******/ // The module cache +/******/ var installedModules = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ +/******/ // Check if module is in cache +/******/ if(installedModules[moduleId]) { +/******/ return installedModules[moduleId].exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = installedModules[moduleId] = { +/******/ i: moduleId, +/******/ l: false, +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); +/******/ +/******/ // Flag the module as loaded +/******/ module.l = true; +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/******/ +/******/ // expose the modules object (__webpack_modules__) +/******/ __webpack_require__.m = modules; +/******/ +/******/ // expose the module cache +/******/ __webpack_require__.c = installedModules; +/******/ +/******/ // define getter function for harmony exports +/******/ __webpack_require__.d = function(exports, name, getter) { +/******/ if(!__webpack_require__.o(exports, name)) { +/******/ Object.defineProperty(exports, name, { enumerable: true, get: getter }); +/******/ } +/******/ }; +/******/ +/******/ // define __esModule on exports +/******/ __webpack_require__.r = function(exports) { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ +/******/ // create a fake namespace object +/******/ // mode & 1: value is a module id, require it +/******/ // mode & 2: merge all properties of value into the ns +/******/ // mode & 4: return value when already ns object +/******/ // mode & 8|1: behave like require +/******/ __webpack_require__.t = function(value, mode) { +/******/ if(mode & 1) value = __webpack_require__(value); +/******/ if(mode & 8) return value; +/******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value; +/******/ var ns = Object.create(null); +/******/ __webpack_require__.r(ns); +/******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value }); +/******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key)); +/******/ return ns; +/******/ }; +/******/ +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = function(module) { +/******/ var getter = module && module.__esModule ? +/******/ function getDefault() { return module['default']; } : +/******/ function getModuleExports() { return module; }; +/******/ __webpack_require__.d(getter, 'a', getter); +/******/ return getter; +/******/ }; +/******/ +/******/ // Object.prototype.hasOwnProperty.call +/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); }; +/******/ +/******/ // __webpack_public_path__ +/******/ __webpack_require__.p = ""; +/******/ +/******/ +/******/ // Load entry module and return exports +/******/ return __webpack_require__(__webpack_require__.s = "./index.js"); +/******/ }) +/************************************************************************/ +/******/ ({ + +/***/ "./Errors.js": +/*!*******************!*\ + !*** ./Errors.js ***! + \*******************/ +/*! no static exports found */ +/***/ (function(module, exports) { + +errors = {}; + +// Use this when the code logic has been broken - e.g. something is called with an undefined parameter, its preferable to console.assert +// Typically this is an error, that should have been caught higher up. +class CodingError extends Error { + constructor(message) { + super(message || "Coding Error"); + this.name = "CodingError" + } +} +errors.CodingError = CodingError; +// These are equivalent of python exceptions, will log and raise alert in most cases - exceptions aren't caught +class ToBeImplementedError extends Error { + constructor(message) { + super("To be implemented: " + message); + this.name = "ToBeImplementedError" + } +} +errors.ToBeImplementedError = ToBeImplementedError; + +class TransportError extends Error { + constructor(message) { + super(message || "Transport failure"); + this.name = "TransportError" + } +} +errors.TransportError = TransportError; + +class TimeoutError extends Error { + constructor(message) { + super(message || "Timed out"); + this.name = "TimeoutError" + } +} +errors.TimeoutError = TimeoutError; + +class IntentionallyUnimplementedError extends Error { + constructor(message) { + super(message || "Intentionally Unimplemented Function"); + this.name = "IntentionallyUnimplementedError" + } +} +errors.IntentionallyUnimplementedError = IntentionallyUnimplementedError; + + +/*---- Below here are errors copied from previous Dweb-Transport and not currently used */ +/* +class ObsoleteError extends Error { + constructor(message) { + super("Obsolete: " + message); + this.name = "ObsoleteError" + } +} +errors.ObsoleteError = ObsoleteError; + +// Use this when the logic of encryption wont let you do something, typically something higher should have stopped you trying. +// Examples include signing something when you only have a public key. +class EncryptionError extends Error { + constructor(message) { + super(message || "Encryption Error"); + this.name = "EncryptionError" + } +} +errors.EncryptionError = EncryptionError; + +// Use this something that should have been signed isn't - this is externally signed, i.e. a data rather than coding error +class SigningError extends Error { + constructor(message) { + super(message || "Signing Error"); + this.name = "SigningError" + } +} +errors.SigningError = SigningError; + +class ForbiddenError extends Error { + constructor(message) { + super(message || "Forbidden failure"); + this.name = "ForbiddenError" + } +} +errors.ForbiddenError = ForbiddenError; + +class AuthenticationError extends Error { + constructor(message) { + super(message || "Authentication failure"); + this.name = "AuthenticationError" + } +} +errors.AuthenticationError = AuthenticationError; + +class DecryptionFailError extends Error { + constructor(message) { + super(message || "Decryption Failed"); + this.name = "DecryptionFailError" + } +} +errors.DecryptionFailError = DecryptionFailError; + +class SecurityWarning extends Error { + constructor(message) { + super(message || "Security Warning"); + this.name = "SecurityWarning" + } +} +errors.SecurityWarning = SecurityWarning; + +class ResolutionError extends Error { + constructor(message) { + super(message || "Resolution failure"); + this.name = "ResolutionError" + } +} +errors.ResolutionError = ResolutionError; +*/ +exports = module.exports = errors; + + +/***/ }), + +/***/ "./Naming.js": +/*!*******************!*\ + !*** ./Naming.js ***! + \*******************/ +/*! no static exports found */ +/***/ (function(module, exports, __webpack_require__) { + +const debug = __webpack_require__(/*! debug */ "./node_modules/debug/src/browser.js")('dweb-transports:naming'); + +const domains = { + arc: { + "archive.org": { + ".": ["https://dweb.me/archive/archive.html"], + "about": ["https://archive.org/about/"], + "details": ["https://dweb.me/archive/archive.html?item="], + "examples": ["https://dweb.me/archive/examples/"], + "images": ["https://dweb.me/archive/images/"], + "serve": ["https://dweb.archive.org/download/"], + "metadata": [ + "wolk://dweb.archive.org/metadata/", + "gun:/gun/arc/archive.org/metadata/", + "https://dweb.me/arc/archive.org/metadata/"], + "search.php": ["https://dweb.me/archive/archive.html?query="], + "search": ["https://dweb.me/archive/archive.html?query="], + }, + }, + ipfs: [ "http://ipfs.io/ipfs/", "https://dweb.me/ipfs/"], +} + + +function expand(partialUrl, remainder) { + return partialUrl.endsWith("html") + ? [partialUrl, remainder.join('/')] // THis might always be an error. + : partialUrl.endsWith("=") + ? partialUrl + remainder.join('/') + : (partialUrl.endsWith("/")) + ? partialUrl+remainder.join('/') + : undefined; +} +function resolve(parent, table, path) { + /** + * parent = STRING "a/b/c" path matched so far + * table = { key: url || [url]} + * path = "d/e/f" + * returns [ url || [url,remainder]] || undefined + */ + //debug("Resolving %o in %s", path, parent); + const remainder = Array.isArray(path) ? path : path.split('/'); + const name = remainder.shift(); + const found = table[name] || table["."] + if (found) { + if (Array.isArray(found)) { + return (found.map(partialUrl => expand(partialUrl, remainder)).filter(url => !!url)); // [url || [url, remainder]] + } else if (typeof found === "object") { + return resolve([parent, name].join('/'), found, remainder); + } else if (typeof found === "string") { + return [ expand(found, remainder) ] + } + } else { + debug("WARNING unable to resolve %s in %s", name, parent.join('/') || '/' ) + return undefined; // Remainder not found + } +} + +function resolveName(url) { + return url.startsWith("dweb:/") ? resolve([], domains, url.slice(6)) : url; // +} +function naming(names) { + return [].concat(...names.map(n => resolveName(n))) +} +async function p_namingcb(names) { + return new Promise((resolve, reject) => { try { const res = naming(names); resolve(res); } catch(err) {reject(err)}}); // Promisify pattern v2b (no CB) +} + +/* +//TODO find in DM where its catching http://dweb.me and heading back to http://localhost:4244 +const testdata = { + "dweb:/arc/archive.org/metadata/foo": [ + "https://dweb.me/arc/archive.org/metadata/foo", + "gun:/gun/arc/archive.org/metadata/foo", + "wolk://dweb.archive.org/metadata/foo" ], + "dweb:/arc/archive.org/details/foo": [ + "https://dweb.me/archive/archive.html?item=foo"], +} + +function test() { + Object.entries(testdata).forEach(kv => { + const res = resolveName(kv[0]); + if ((!res + || res.length !== kv[1].length) + || res.some(r => !kv[1].includes(r))) { + debug("%s => %s expect %s", kv[0], res, kv[1]); + }}); + p_namingcb(["dweb:/arc/archive.org/details/foo","foo://bar.baz"]) + .then(res => debug("Got %o", res)) + .catch(err => debug("Fail %o", err.message)); +} +test(); +*/ + +exports = module.exports = {naming, p_namingcb}; + + +/***/ }), + +/***/ "./Transport.js": +/*!**********************!*\ + !*** ./Transport.js ***! + \**********************/ +/*! no static exports found */ +/***/ (function(module, exports, __webpack_require__) { + +const Url = __webpack_require__(/*! url */ "./node_modules/url/url.js"); +const errors = __webpack_require__(/*! ./Errors */ "./Errors.js"); // Standard Dweb Errors + +function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})} + + +class Transport { + + constructor(options) { + /* + Doesnt do anything, its all done by SuperClasses, + Superclass should merge with default options, call super + + Fields: + statuselement: If set is an HTML Element that should be adjusted to indicate status (this is managed by Transports, just stored on Transport) + statuscb: Callback when status changes + name: Short name of element e.g. HTTP IPFS WEBTORRENT GUN + */ + } + + //TODO-SPLIT define load() + + static setup0(options) { + /* + First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections. + */ + throw new errors.IntentionallyUnimplementedError("Intentionally undefined function Transport.setup0 should have been subclassed"); + } + + p_setup1(cb) { + /* + Setup the resource and open any P2P connections etc required to be done just once. Asynchronous and should leave status=STATUS_STARTING until it resolves, or STATUS_FAILED if fails. + + cb (t)=>void If set, will be called back as status changes (so could be multiple times) + Resolves to the Transport instance + */ + return this; + } + p_setup2(cb) { + /* + Works like p_setup1 but runs after p_setup1 has completed for all transports. This allows for example YJS to wait for IPFS to be connected in TransportIPFS.setup1() and then connect itself using the IPFS object. + + cb (t)=>void If set, will be called back as status changes (so could be multiple times) + Resolves to the Transport instance + */ + return this; + } + static async p_setup(options, cb) { + /* + A deprecated utility to simply setup0 then p_setup1 then p_setup2 to allow a transport to be started in one step, normally Transports.p_setup should be called instead. + */ + let t = await this.setup0(options) // Sync version that doesnt connect + .p_setup1(cb); // And connect + + return t.p_setup2(cb); // And connect + } + /* Disconnect from the transport service - there is no guarrantee that a restart will be successfull so this is usually only for when exiting */ + stop(refreshstatus, cb) { + // refreshstatus(Transport instance) => optional callback to the UI to update the status on the display + this.status = Transport.STATUS_FAILED; + if (refreshstatus) refreshstatus(this); + cb(null, this); + } + togglePaused(cb) { + /* + Switch the state of the transport between STATUS_CONNECTED and STATUS_PAUSED, + in the paused state it will not be used for transport but, in some cases, will still do background tasks like serving files. + + cb(transport)=>void a callback called after this is run, may be used for example to change the UI + */ + switch (this.status) { + case Transport.STATUS_CONNECTED: + this.status = Transport.STATUS_PAUSED; + break; + case Transport.STATUS_PAUSED: + this.status = Transport.STATUS_CONNECTED; // Superclass might change to STATUS_STARTING if needs to stop/restart + break; + case Transport.STATUS_LOADED: + this.p_setup1(cb).then((t)=>t.p_setup2(cb)); // Allows for updating status progressively as attempts to connect + } + if (cb) cb(this); + } + + async p_status() { + /* + Check the status of the underlying transport. This may update the "status" field from the underlying transport. + returns: a numeric code for the status of a transport. + */ + return this.status; + } + + connected() { + // True if connected (status==STATUS_CONNECTED==0) should not need subclassing + return ! this.status; + } + supports(url, func, {noCache=undefined}={}) { //TODO-API + /* + Determine if this transport supports a certain set of URLs and a func + + :param url: String or parsed URL + :param opts: { noCache } check against supportFeatures + :return: true if this protocol supports these URLs and this func + :throw: TransportError if invalid URL + */ + if (typeof url === "string") { + url = Url.parse(url); // For efficiency, only parse once. + } + if (url && !url.protocol) { + throw new Error("URL failed to specific a scheme (before :) " + url.href) + } //Should be TransportError but out of scope here + // noinspection Annotator supportURLs is defined in subclasses + return ( (!url || this.supportURLs.includes(url.protocol.slice(0, -1))) + && (!func || this.supportFunctions.includes(func)) + && (!noCache || this.supportFeatures.includes("noCache")) + ) + } + + validFor(url, func, opts) { + // By default a transport can handle a url and a func if its connected and supports that url/func + // This shouldnt need subclassing, an exception is HTTP which only applies "connected" against urls heading for the gateway + return this.connected() && this.supports(url, func, opts); + } + + + p_rawstore(data, opts) { + /* + Store a blob of data onto the decentralised transport. + Returns a promise that resolves to the url of the data + + :param string|Buffer data: Data to store - no assumptions made to size or content + :resolve string: url of data stored + */ + throw new errors.ToBeImplementedError("Intentionally undefined function Transport.p_rawstore should have been subclassed"); + } + + async p_rawstoreCaught(data) { + try { + return await this.p_rawstore(data); + } catch (err) { + + } + } + p_store() { + throw new errors.ToBeImplementedError("Undefined function Transport.p_store - may define higher level semantics here (see Python)"); + } + + //noinspection JSUnusedLocalSymbols + + p_rawfetch(url, {timeoutMS=undefined, start=undefined, end=undefined, relay=false}={}) { + /* + Fetch some bytes based on a url, no assumption is made about the data in terms of size or structure. + Where required by the underlying transport it should retrieve a number if its "blocks" and concatenate them. + Returns a new Promise that resolves currently to a string. + There may also be need for a streaming version of this call, at this point undefined. + + :param string url: URL of object being retrieved + :param timeoutMS Max time to wait on transports that support it (IPFS for fetch) + :param start,end Inclusive byte range wanted (must be supported, uses a "slice" on output if transport ignores it. + :param relay If first transport fails, try and retrieve on 2nd, then store on 1st, and so on. + + :resolve string: Return the object being fetched, (note currently returned as a string, may refactor to return Buffer) + :throws: TransportError if url invalid - note this happens immediately, not as a catch in the promise + */ + console.assert(false, "Intentionally undefined function Transport.p_rawfetch should have been subclassed"); + return "UNIMPLEMENTED"; + } + + p_fetch() { + throw new errors.ToBeImplementedError("Undefined function Transport.p_fetch - may define higher level semantics here (see Python)"); + } + + p_rawadd(url, sig) { + /* + Store a new list item, ideally it should be stored so that it can be retrieved either by "signedby" (using p_rawlist) or + by "url" (with p_rawreverse). The underlying transport does not need to guarantee the signature, + an invalid item on a list should be rejected on higher layers. + + :param string url: String identifying an object being added to the list. + :param Signature sig: A signature data structure. + :resolve undefined: + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_rawadd"); + } + + p_rawlist(url) { + /* + Fetch all the objects in a list, these are identified by the url of the public key used for signing. + (Note this is the 'signedby' parameter of the p_rawadd call, not the 'url' parameter + Returns a promise that resolves to the list. + Each item of the list is a dict: {"url": url, "date": date, "signature": signature, "signedby": signedby} + List items may have other data (e.g. reference ids of underlying transport) + + :param string url: String with the url that identifies the list. + :resolve array: An array of objects as stored on the list. + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_rawlist"); + } + + p_list() { + throw new Error("Undefined function Transport.p_list"); + } + p_newlisturls(cl) { + /* + Must be implemented by any list, return a pair of URLS that may be the same, private and public links to the list. + returns: ( privateurl, publicurl) e.g. yjs:xyz/abc or orbitdb:a123 + */ + throw new Error("undefined function Transport.p_newlisturls"); + } + + //noinspection JSUnusedGlobalSymbols + p_rawreverse(url) { + /* + Similar to p_rawlist, but return the list item of all the places where the object url has been listed. + The url here corresponds to the "url" parameter of p_rawadd + Returns a promise that resolves to the list. + + :param string url: String with the url that identifies the object put on a list. + :resolve array: An array of objects as stored on the list. + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_rawreverse"); + } + + listmonitor(url, callback, {current=false}={}) { + /* + Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist. + + :param url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd + :param callback: function(obj) Callback for each new item added to the list + obj is same format as p_rawlist or p_rawreverse + */ + console.log("Undefined function Transport.listmonitor"); // Note intentionally a log, as legitamte to not implement it + } + + + // ==== TO SUPPORT KEY VALUE INTERFACES IMPLEMENT THESE ===== + // Support for Key-Value pairs as per + // https://docs.google.com/document/d/1yfmLRqKPxKwB939wIy9sSaa7GKOzM5PrCZ4W1jRGW6M/edit# + + async p_newdatabase(pubkey) { + /* + Create a new database based on some existing object + pubkey: Something that is, or has a pubkey, by default support Dweb.PublicPrivate, KeyPair or an array of strings as in the output of keypair.publicexport() + returns: {publicurl, privateurl} which may be the same if there is no write authentication + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_newdatabase"); + } + //TODO maybe change the listmonitor / monitor code for to use "on" and the structure of PP.events + //TODO but note https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy about Proxy which might be suitable, prob not as doesnt map well to lists + async p_newtable(pubkey, table) { + /* + Create a new table, + pubkey: Is or has a pubkey (see p_newdatabase) + table: String representing the table - unique to the database + returns: {privateurl, publicurl} which may be the same if there is no write authentication + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_newtable"); + } + + async p_set(url, keyvalues, value) { // url = yjs:/yjs/database/table/key + /* + Set one or more keys in a table. + url: URL of the table + keyvalues: String representing a single key OR dictionary of keys + value: String or other object to be stored (its not defined yet what objects should be supported, e.g. any object ? + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_set"); + } + async p_get(url, keys) { + /* Get one or more keys from a table + url: URL of the table + keys: Array of keys + returns: Dictionary of values found (undefined if not found) + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_get"); + } + + async p_delete(url, keys) { + /* Delete one or more keys from a table + url: URL of the table + keys: Array of keys + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_delete"); + } + + async p_keys(url) { + /* Return a list of keys in a table (suitable for iterating through) + url: URL of the table + returns: Array of strings + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_keys"); + } + async p_getall(url) { + /* Return a dictionary representing the table + url: URL of the table + returns: Dictionary of Key:Value pairs, note take care if this could be large. + */ + throw new errors.ToBeImplementedError("Undefined function Transport.p_keys"); + } + static async p_f_createReadStream(url, {wanturl=false}) { + /* + Provide a function of the form needed by tag and renderMedia library etc + + url Urls of stream + wanturl True if want the URL of the stream (for service workers) + returns f(opts) => stream returning bytes from opts.start || start of file to opts.end-1 || end of file + */ + } + // ------ UTILITY FUNCTIONS, NOT REQD TO BE SUBCLASSED ---- + + static mergeoptions(a) { + /* + Deep merge options dictionaries, careful since searchparameters from URL passed in as null + */ + let c = {}; + for (let i = 0; i < arguments.length; i++) { + let b = arguments[i]; + for (let key in b) { + let val = b[key]; + if (val !== null) { + if ((typeof val === "object") && !Array.isArray(val) && c[key]) { + c[key] = Transport.mergeoptions(a[key], b[key]); + } else { + c[key] = b[key]; + } + } + } + } + return c; + } + + async p_test_list({urlexpectedsubstring=undefined}={}) { + //TODO - this test doesn't work since we dont have Signature nor want to create dependency on it - when works, add to GUN & YJS + {console.log(this.name,"p_test_kvt")} + try { + let table = await this.p_newlisturls("NACL VERIFY:1234567LIST"); + let mapurl = table.publicurl; + console.log("newlisturls=",mapurl); + console.assert((!urlexpectedsubstring) || mapurl.includes(urlexpectedsubstring)); + await this.p_rawadd(mapurl, "testvalue"); + let res = await this.p_rawlist(mapurl); + console.assert(res.length===1 && res[0] === "testvalue"); + await this.p_rawadd(mapurl, {foo: "bar"}); // Try adding an object + res = await this.p_rawlist(mapurl); + console.assert(res.length === 2 && res[1].foo === "bar"); + await this.p_rawadd(mapurl, [1,2,3]); // Try setting to an array + res = await this.p_rawlist(mapurl); + console.assert(res.length === 2 && res[2].length === 3 && res[2][1] === 2); + await delay(200); + console.log(this.name, "p_test_list complete") + } catch(err) { + console.log("Exception thrown in ", this.name, "p_test_list:", err.message); + throw err; + } + + } + async p_test_kvt(urlexpectedsubstring) { + /* + Test the KeyValue functionality of any transport that supports it. + urlexpectedsubstring: Some string expected in the publicurl of the table. + */ + {console.log(this.name,"p_test_kvt")} + try { + let table = await this.p_newtable("NACL VERIFY:1234567KVT","mytable"); + let mapurl = table.publicurl; + console.log("newtable=",mapurl); + console.assert(mapurl.includes(urlexpectedsubstring)); + await this.p_set(mapurl, "testkey", "testvalue"); + let res = await this.p_get(mapurl, "testkey"); + console.assert(res === "testvalue"); + await this.p_set(mapurl, "testkey2", {foo: "bar"}); // Try setting to an object + res = await this.p_get(mapurl, "testkey2"); + console.assert(res.foo === "bar"); + await this.p_set(mapurl, "testkey3", [1,2,3]); // Try setting to an array + res = await this.p_get(mapurl, "testkey3"); + console.assert(res[1] === 2); + res = await this.p_keys(mapurl); + console.assert(res.includes("testkey") && res.includes("testkey3") && res.length === 3); + await this.p_delete(mapurl, ["testkey"]); + res = await this.p_getall(mapurl); + console.log("getall=>",res); + console.assert(res.testkey2.foo === "bar" && res.testkey3["1"] === 2 && !res.testkey); + await delay(200); + console.log(this.name, "p_test_kvt complete") + } catch(err) { + console.log("Exception thrown in ", this.name, "p_test_kvt:", err.message); + throw err; + } + } + + +} +Transport.STATUS_CONNECTED = 0; // Connected - all other numbers are some version of not ok to use +Transport.STATUS_FAILED = 1; // Failed to connect +Transport.STATUS_STARTING = 2; // In the process of connecting +Transport.STATUS_LOADED = 3; // Code loaded, but haven't tried to connect. (this is typically hard coded in subclasses constructor) +Transport.STATUS_PAUSED = 4; // It was launched, probably connected, but now paused so will be ignored by validFor // Note this is copied to dweb-archive/Nav.js so check if change +Transport.STATUSTEXT = ["Connected", "Failed", "Starting", "Loaded", "Paused"]; +exports = module.exports = Transport; + + +/***/ }), + +/***/ "./TransportGUN.js": +/*!*************************!*\ + !*** ./TransportGUN.js ***! + \*************************/ +/*! no static exports found */ +/***/ (function(module, exports, __webpack_require__) { + +/* WEBPACK VAR INJECTION */(function(process) {/* +This Transport layers uses GUN. + +See https://github.com/internetarchive/dweb-mirror/issues/43 for meta issue +*/ +const Url = __webpack_require__(/*! url */ "./node_modules/url/url.js"); +process.env.GUN_ENV = "false"; + +/* This should be done in the caller (see dweb-archive/archive.html for example) +const Gun = require('gun/gun.js'); // gun/gun is the minimized version +// Raw Gun has almost nothing in it, it needs at least the following to work properly. +require('gun/lib/path.js'); // So that .path works + +//WORKAROUND-GUN-STORAGE +// The next step is to stop it failing as soon as its cached 5Mb in localstorage +// see https://github.com/amark/gun/blob/master/test/tmp/indexedDB.html and https://github.com/amark/gun/issues/590 +// but the instructions on how to do this are obviously broken so waiting on @amark to get this working. + +// See https://github.com/internetarchive/dweb-archive/issues/106 unable to get this working (Gun doesnt work well with webpack) +//require('gun/nts'); +//require('gun/lib/wire'); // NodeJS websocket +//require('gun/lib/multicast'); // Local area broadcasting needs passing `multicast: true` in options (which is safe in node + browser) +require('gun/lib/radix.js'); // loaded by store but required for webpack +require('gun/lib/radisk.js'); // loaded by store but required for webpack +require('gun/lib/store.js'); +require('gun/lib/rindexed.js'); +*/ +const debuggun = __webpack_require__(/*! debug */ "./node_modules/debug/src/browser.js")('dweb-transports:gun'); +const canonicaljson = __webpack_require__(/*! @stratumn/canonicaljson */ "./node_modules/@stratumn/canonicaljson/lib/canonicaljson.mjs"); + +// Other Dweb modules +const errors = __webpack_require__(/*! ./Errors */ "./Errors.js"); // Standard Dweb Errors +const Transport = __webpack_require__(/*! ./Transport.js */ "./Transport.js"); // Base class for TransportXyz +const Transports = __webpack_require__(/*! ./Transports */ "./Transports.js"); // Manage all Transports that are loaded +const utils = __webpack_require__(/*! ./utils */ "./utils.js"); // Utility functions + +// Utility packages (ours) And one-liners +//unused currently: function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})} + +let defaultoptions = { + peers: [ "https://dweb.me:4246/gun" ], + localStorage: false // Need to be false to turn localStorage off on browser (do this if include radix/radisk) +}; +//To run a superpeer - cd wherever; node install gun; cd node_modules/gun; npm start - starts server by default on port 8080, or set an "env" - see http.js +//setenv GUN_ENV false; node examples/http.js 4246 +//Make sure to open of the port (typically in /etc/ferm) +// TODO-GUN - copy example from systemctl here + +/* + WORKING AROUND GUN WEIRDNESS/SUBOPTIMAL (of course, whats weird/sub-optimal to me, might be ideal to someone else) - search the code to see where worked around + + WORKAROUND-GUN-UNDERSCORE .once() and possibly .on() send an extra GUN internal field "_" which needs filtering. Reported and hopefully will get fixed + .once behaves differently on node or the browser - this is a bug https://github.com/amark/gun/issues/586 and for now this code doesnt work on Node + WORKAROUND-GUN-CURRENT: .once() and .on() deliver existing values as well as changes, reported & hopefully will get way to find just new ones. + WORKAROUND-GUN-DELETE: There is no way to delete an item, setting it to null is recorded and is by convention a deletion. BUT the field will still show up in .once and .on, + WORKAROUND-GUN-PROMISES: GUN is not promisified, there is only one place we care, and that is .once (since .on is called multiple times). + WORKAROUND-GUN-ERRORS: GUN does an unhelpful job with errors, for example returning undefined when it cant find something (e.g. if connection to superpeer is down), + for now just throw an error on undefined + WORKAROUND-GUN-STORAGE: GUN defaults to local storage, which then fails on 5Mb or more of data, need to use radix, which has to be included and has bizarre config requirement I can't figure out + TODO-GUN, handle error callbacks which are available in put etc + Errors and Promises: Note that GUN's use of promises is seriously unexpected (aka weird), see https://gun.eco/docs/SEA#errors + instead of using .reject or throwing an error at async it puts the error in SEA.err, so how that works in async parallel context is anyone's guess + */ + +class TransportGUN extends Transport { + /* + GUN specific transport - over IPFS + + Fields: + gun: object returned when starting GUN + */ + + constructor(options) { + super(options); + this.options = options; // Dictionary of options + this.gun = undefined; + this.name = "GUN"; // For console log etc + this.supportURLs = ['gun']; + this.supportFunctions = [ 'fetch', //'store' + 'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor', + 'add', 'list', 'listmonitor', 'newlisturls']; + this.supportFeatures = []; // Doesnt support noCache and is mutable + this.status = Transport.STATUS_LOADED; + } + + connection(url) { + /* + TODO-GUN need to determine what a "rooted" Url is in gun, is it specific to a superpeer for example + Utility function to get Gun object for this URL (note this isn't async) + url: URL string or structure, to find list of of form [gun|dweb]:/gun//[/debuggun("Got late result of: %o", data)); // Checking for bug in GUN issue#586 - ignoring result + if (!val) + throw new errors.TransportError("GUN unable to retrieve: "+url.href); // WORKAROUND-GUN-ERRORS - gun doesnt throw errors when it cant find something + let o = typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync (see same code on p_get and p_rawfetch) + //TODO-GUN this is a hack because the metadata such as metadata/audio is getting cached in GUN and in this case is wrong. + if (o.metadata && o.metadata.thumbnaillinks && o.metadata.thumbnaillinks.find(t => t.includes('ipfs/zb2'))) { + throw new errors.TransportError("GUN retrieving legacy data at: "+url.href) + } + return o; + } + + + // ===== LISTS ======== + + // noinspection JSCheckFunctionSignatures + async p_rawlist(url) { + /* + Fetch all the objects in a list, these are identified by the url of the public key used for signing. + (Note this is the 'signedby' parameter of the p_rawadd call, not the 'url' parameter + Returns a promise that resolves to the list. + Each item of the list is a dict: {"url": url, "date": date, "signature": signature, "signedby": signedby} + List items may have other data (e.g. reference ids of underlying transport) + + :param string url: String with the url that identifies the list. + :resolve array: An array of objects as stored on the list. + */ + try { + let g = this.connection(url); + let data = await this._p_once(g); + let res = data ? Object.keys(data).filter(k => k !== '_').sort().map(k => data[k]) : []; //See WORKAROUND-GUN-UNDERSCORE + // .filter((obj) => (obj.signedby.includes(url))); // upper layers verify, which filters + debuggun("p_rawlist found", ...utils.consolearr(res)); + return res; + } catch(err) { + // Will be logged by Transports + throw(err); + } + } + + listmonitor(url, callback, {current=false}={}) { + /* + Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist. + + url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd + callback: function(obj) Callback for each new item added to the list + obj is same format as p_rawlist or p_rawreverse + current true if should send list of existing elements + */ + let g = this.connection(url); + if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new. + g.once(data => { + this.monitored = data ? Object.keys(data) : []; // Keep a copy - could actually just keep high water mark unless getting partial knowledge of state of array. + g.map().on((v, k) => { + if (!(this.monitored.includes(k)) && (k !== '_')) { //See WORKAROUND-GUN-UNDERSCORE + this.monitored.push(k); + callback(JSON.parse(v)); + } + }); + }); + } else { + g.map().on((v, k) => callback("set", k, JSON.parse(v))); + } + } + + // noinspection JSCheckFunctionSignatures + async p_rawadd(url, sig) { + /* + Store a new list item, it should be stored so that it can be retrieved either by "signedby" (using p_rawlist) or + by "url" (with p_rawreverse). The underlying transport does not need to guarantee the signature, + an invalid item on a list should be rejected on higher layers. + + :param string url: String identifying list to post to + :param Signature sig: Signature object containing at least: + date - date of signing in ISO format, + urls - array of urls for the object being signed + signature - verifiable signature of date+urls + signedby - urls of public key used for the signature + :resolve undefined: + */ + // noinspection JSUnresolvedVariable + // Logged by Transports + console.assert(url && sig.urls.length && sig.signature && sig.signedby.length, "TransportGUN.p_rawadd args", url, sig); + this.connection(url) + .set( canonicaljson.stringify( sig.preflight( Object.assign({}, sig)))); + } + + // noinspection JSCheckFunctionSignatures + async p_newlisturls(cl) { + let u = await this._p_newgun(cl); + return [ u, u]; + } + + //=======KEY VALUE TABLES ======== + + // noinspection JSMethodCanBeStatic + async _p_newgun(pubkey) { + if (pubkey.hasOwnProperty("keypair")) + pubkey = pubkey.keypair.signingexport(); + // By this point pubkey should be an export of a public key of form xyz:abc where xyz + // specifies the type of public key (NACL VERIFY being the only kind we expect currently) + return `gun:/gun/${encodeURIComponent(pubkey)}`; + } + async p_newdatabase(pubkey) { + /* + Request a new database + For GUN it doesnt actually create anything, just generates the URLs + TODO-GUN simple version first - userid based on my keypair first, then switch to Gun's userid and its keypair + Include gun/sea.js; user.create(,); user.auth(,); # See gun.eco/docs/Auth + + returns: {publicurl: "gun:/gun/", privateurl: "gun:/gun/"> + */ + let u = await this._p_newgun(pubkey); + return {publicurl: u, privateurl: u}; + } + + async p_newtable(pubkey, table) { + /* + Request a new table + For GUN it doesnt actually create anything, just generates the URLs + + returns: {publicurl: "gun:/gun//
", privateurl: "gun:/gun//
"> + */ + if (!pubkey) throw new errors.CodingError("p_newtable currently requires a pubkey"); + let database = await this.p_newdatabase(pubkey); + // If have use cases without a database, then call p_newdatabase first + return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it + } + + async p_set(url, keyvalues, value) { // url = yjs:/yjs/database/table + /* + Set key values + keyvalues: string (key) in which case value should be set there OR + object in which case value is ignored + */ + let table = this.connection(url); + if (typeof keyvalues === "string") { + table.path(keyvalues).put(canonicaljson.stringify(value)); + } else { + // Store all key-value pairs without destroying any other key/value pairs previously set + console.assert(!Array.isArray(keyvalues), "TransportGUN - shouldnt be passsing an array as the keyvalues"); + table.put( + Object.keys(keyvalues).reduce( + function(previous, key) { previous[key] = canonicaljson.stringify(keyvalues[key]); return previous; }, + {} + )) + } + } + + async p_get(url, keys) { + let table = this.connection(url); + if (Array.isArray(keys)) { + throw new errors.ToBeImplementedError("p_get(url, [keys]) isn't supported - because of ambiguity better to explicitly loop on set of keys or use getall and filter"); + /* + return keys.reduce(function(previous, key) { + let val = table.get(key); + previous[key] = typeof val === "string" ? JSON.parse(val) : val; // Handle undefined + return previous; + }, {}); + */ + } else { + let val = await this._p_once(table.get(keys)); // Resolves to value + return typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync (see same code on p_get and p_rawfetch) + } + } + + async p_delete(url, keys) { + let table = this.connection(url); + if (typeof keys === "string") { + table.path(keys).put(null); + } else { + keys.map((key) => table.path(key).put(null)); // This looks like it is sync + } + } + + //WORKAROUND-GUN-PROMISE suggest p_once as a good single addition + //TODO-GUN expand this to workaround Gun weirdness with errors. + _p_once(gun) { // Note in some cases (e.g. p_getall) this will resolve to a object, others a string/number (p_get) + // TODO-GUN Temporarily added a 2000ms delay to workaround https://github.com/internetarchive/dweb-archive/issues/106 / https://github.com/amark/gun/issues/762 + return new Promise((resolve) => gun.once(resolve, {wait: 2000})); + } + + async p_keys(url) { + let res = await this._p_once(this.connection(url)); + return Object.keys(res) + .filter(k=> (k !== '_') && (res[k] !== null)); //See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE + } + + async p_getall(url) { + let res = await this._p_once(this.connection(url)); + return Object.keys(res) + .filter(k=> (k !== '_') && res[k] !== null) //See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE + .reduce( function(previous, key) { previous[key] = JSON.parse(res[key]); return previous; }, {}); + } + + async monitor(url, callback, {current=false}={}) { + /* + Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall. + Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => GUN.monitor)(b: dispatchEvent) + + url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd + callback: function({type, key, value}) Callback for each new item added to the list (type = "set"|"delete") + current Send existing items to the callback as well + */ + let g = this.connection(url); + if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new. + g.once(data => { + this.monitored = Object.assign({},data); // Make a copy of data (this.monitored = data won't work as just points at same structure) + g.map().on((v, k) => { + if ((v !== this.monitored[k]) && (k !== '_')) { //See WORKAROUND-GUN-UNDERSCORE + this.monitored[k] = v; + callback("set", k, JSON.parse(v)); + } + }); + }); + } else { + g.map().on((v, k) => callback("set", k, JSON.parse(v))); + } + } + + static async p_test() { + debuggun("p_test"); + try { + let t = this.setup0({}); //TODO-GUN when works with peers commented out, try passing peers: [] + await t.p_setup1(); // Not passing cb yet + await t.p_setup2(); // Not passing cb yet - this one does nothing on GUN + // noinspection JSIgnoredPromiseFromCall + t.p_test_kvt("gun:/gun/NACL"); + //t.p_test_list("gun:/gun/NACL"); //TODO test_list needs fixing to not create a dependency on Signature + } catch(err) { + console.warn("Exception thrown in TransportGUN.test:", err.message); + throw err; + } + } + + // noinspection JSUnusedGlobalSymbols + static async demo_bugs() { + let gun = new Gun(); + gun.get('foo').get('bar').put('baz'); + console.log("Expect {bar: 'baz'} but get {_:..., bar: 'baz'}"); + gun.get('foo').once(data => console.log(data)); + gun.get('zip').get('bar').set('alice'); + console.log("Expect {12345: 'alice'} but get {_:..., 12345: 'alice'}"); + gun.get('foo').once(data => console.log(data)); + // Returns extra "_" field + } +} +Transports._transportclasses["GUN"] = TransportGUN; +TransportGUN.requires = ['gun/gun.js', 'gun/lib/path.js', 'gun/nts', 'gun/lib/wire', 'gun/lib/multicast', 'gun/lib/radix.js', + 'gun/lib/radisk.js', 'gun/lib/store.js', 'gun/lib/rindexed.js'] + .map(s => 'https://cdn.jsdelivr.net/npm/' + s); +exports = module.exports = TransportGUN; + +/* WEBPACK VAR INJECTION */}.call(this, __webpack_require__(/*! ./node_modules/process/browser.js */ "./node_modules/process/browser.js"))) + +/***/ }), + +/***/ "./TransportHTTP.js": +/*!**************************!*\ + !*** ./TransportHTTP.js ***! + \**************************/ +/*! no static exports found */ +/***/ (function(module, exports, __webpack_require__) { + +const Transport = __webpack_require__(/*! ./Transport */ "./Transport.js"); // Base class for TransportXyz +const Transports = __webpack_require__(/*! ./Transports */ "./Transports.js"); // Manage all Transports that are loaded +const httptools = __webpack_require__(/*! ./httptools */ "./httptools.js"); // Expose some of the httptools so that IPFS can use it as a backup +const Url = __webpack_require__(/*! url */ "./node_modules/url/url.js"); +const stream = __webpack_require__(/*! readable-stream */ "./node_modules/readable-stream/readable-browser.js"); +const debug = __webpack_require__(/*! debug */ "./node_modules/debug/src/browser.js")('dweb-transports:http'); +const canonicaljson = __webpack_require__(/*! @stratumn/canonicaljson */ "./node_modules/@stratumn/canonicaljson/lib/canonicaljson.mjs"); + + + +defaulthttpoptions = { + urlbase: 'https://dweb.me', + heartbeat: { delay: 30000 } // By default check twice a minute +}; + +servercommands = { // What the server wants to see to return each of these + rawfetch: "contenthash", // was content/rawfetch which should still work. + rawstore: "contenturl/rawstore", + rawadd: "void/rawadd", + rawlist: "metadata/rawlist", + get: "get/table", + set: "set/table", + delete: "delete/table", + keys: "keys/table", + getall: "getall/table" +}; + + +class TransportHTTP extends Transport { + /* Subclass of Transport for handling HTTP - see API.md for docs + + options { + urlbase: e.g. https://dweb.me Where to go for URLS like /arc/... + heartbeat: { + delay // Time in milliseconds between checks - 30000 might be appropriate - if missing it wont do a heartbeat + statusCB // Callback cb(transport) when status changes + } + } + */ + + constructor(options) { + super(options); // These are now options.http + this.options = options; + this.urlbase = options.urlbase; // e.g. https://dweb.me + this.supportURLs = ['contenthash', 'http','https']; + this.supportFunctions = ['fetch', 'store', 'add', 'list', 'reverse', 'newlisturls', "get", "set", "keys", "getall", "delete", "newtable", "newdatabase"]; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller + this.supportFeatures = ['noCache']; + if (typeof window === "undefined") { + // running in node, can support createReadStream, (browser can't - see createReadStream below) + this.supportFunctions.push("createReadStream"); + } + // noinspection JSUnusedGlobalSymbols + this.supportFeatures = ['fetch.range', 'noCache']; + this.name = "HTTP"; // For console log etc + this.status = Transport.STATUS_LOADED; + } + + static setup0(options) { + let combinedoptions = Transport.mergeoptions(defaulthttpoptions, options.http); + try { + let t = new TransportHTTP(combinedoptions); + Transports.addtransport(t); + return t; + } catch (err) { + console.error("HTTP unable to setup0", err.message); + throw err; + } + } + + p_setup1(statusCB) { + return new Promise((resolve, unusedReject) => { + this.status = Transport.STATUS_STARTING; + if (statusCB) statusCB(this); + this.updateStatus((unusedErr, unusedRes) => { + if (statusCB) statusCB(this); + this.startHeartbeat(this.options.heartbeat); + resolve(this); // Note always resolve even if error from p_status as have set status to failed + }); + }) + } + + async p_status(cb) { //TODO-API + /* + Return (via cb or promise) a numeric code for the status of a transport. + */ + if (cb) { try { this.updateStatus(cb) } catch(err) { cb(err)}} else { return new Promise((resolve, reject) => { try { this.updateStatus((err, res) => { if (err) {reject(err)} else {resolve(res)} })} catch(err) {reject(err)}})} // Promisify pattern v2f + } + updateStatus(cb) { //TODO-API + this.updateInfo((err, res) => { + if (err) { + debug("Error status call to info failed %s", err.message); + this.status = Transport.STATUS_FAILED; + cb(null, this.status); // DOnt pass error up, the status indicates the error + } else { + this.info = res; // Save result + this.status = Transport.STATUS_CONNECTED; + cb(null, this.status); + } + }); + } + + startHeartbeat({delay=undefined, statusCB=undefined}) { + if (delay) { + debug("HTTP Starting Heartbeat") + this.HTTPheartbeatTimer = setInterval(() => { + this.updateStatus((err, res)=>{ // Pings server and sets status + if (statusCB) statusCB(this); // repeatedly call callback if supplies + }, (unusedErr, unusedRes)=>{}); // Dont wait for status to complete + }, delay); + } + } + stopHeartbeat() { + if (this.HTTPheartbeatTimer) { + debug("HTTP stopping heartbeat"); + clearInterval(this.HTTPheartbeatTimer);} + } + stop(refreshstatus, cb) { + this.stopHeartbeat(); + this.status = Transport.STATUS_FAILED; + if (refreshstatus) { refreshstatus(this); } + cb(null, this); + } + + _cmdurl(command) { + return `${this.urlbase}/${command}` + } + _url(url, command, parmstr) { + if (!url) throw new errors.CodingError(`${command}: requires url`); + if (typeof url !== "string") { url = url.href } + url = url.replace('contenthash:/contenthash', this._cmdurl(command)) ; // Note leaves http: and https: urls unchanged + url = url.replace('getall/table', command); + url = url + (parmstr ? "?"+parmstr : ""); + return url; + } + + validFor(url, func, opts) { + // Overrides Transport.prototype.validFor because HTTP's connection test is only really for dweb.me + // in particular this allows urls like https://be-api.us.archive.org + return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func, opts); + } + // noinspection JSCheckFunctionSignatures + async p_rawfetch(url, opts={}) { + /* + Fetch from underlying transport, + Fetch is used both for contenthash requests and table as when passed to SmartDict.p_fetch may not know what we have + url: Of resource - which is turned into the HTTP url in p_httpfetch + opts: {start, end, retries, noCache} see p_GET for documentation + throws: TransportError if fails + */ + //if (!(url && url.includes(':') )) + // throw new errors.CodingError("TransportHTTP.p_rawfetch bad url: "+url); + //if (url.href.includes('contenthash//')) + // console.error("XXX@91", url) + if (((typeof url === "string") ? url : url.href).includes('/getall/table')) { + throw new Error("Probably dont want to be calling p_rawfetch on a KeyValueTable, especially since dont know if its keyvaluetable or subclass"); //TODO-NAMING + } else { + return await httptools.p_GET(this._url(url, servercommands.rawfetch), opts); + } + } + + p_rawlist(url) { + // obj being loaded + // Locate and return a block, based on its url + if (!url) throw new errors.CodingError("TransportHTTP.p_rawlist: requires url"); + return httptools.p_GET(this._url(url, servercommands.rawlist)); + } + rawreverse() { throw new errors.ToBeImplementedError("Undefined function TransportHTTP.rawreverse"); } + + async p_rawstore(data) { + /* + Store data on http server, + data: string + resolves to: {string}: url + throws: TransportError on failure in p_POST > p_httpfetch + */ + //PY: res = self._sendGetPost(True, "rawstore", headers={"Content-Type": "application/octet-stream"}, urlargs=[], data=data) + console.assert(data, "TransportHttp.p_rawstore: requires data"); + const res = await httptools.p_POST(this._cmdurl(servercommands.rawstore), {data, contenttype: "application/octet-stream"}); // resolves to URL + let parsedurl = Url.parse(res); + let pathparts = parsedurl.pathname.split('/'); + return `contenthash:/contenthash/${pathparts.slice(-1)}` + + } + + p_rawadd(url, sig) { + // Logged by Transports + if (!url || !sig) throw new errors.CodingError("TransportHTTP.p_rawadd: invalid parms", url, sig); + const data = canonicaljson.stringify(sig.preflight(Object.assign({},sig)))+"\n"; + return httptools.p_POST(this._url(url, servercommands.rawadd), {data, contenttype: "application/json"}); // Returns immediately + } + + p_newlisturls(cl) { + let u = cl._publicurls.map(urlstr => Url.parse(urlstr)) + .find(parsedurl => + ((parsedurl.protocol === "https:" && ["gateway.dweb.me", "dweb.me"].includes(parsedurl.host) + && (parsedurl.pathname.includes('/content/rawfetch') || parsedurl.pathname.includes('/contenthash/'))) + || (parsedurl.protocol === "contenthash:") && (parsedurl.pathname.split('/')[1] === "contenthash"))); + if (!u) { + // noinspection JSUnresolvedVariable + u = `contenthash:/contenthash/${ cl.keypair.verifyexportmultihashsha256_58() }`; // Pretty random, but means same test will generate same list and server is expecting base58 of a hash + } + return [u,u]; + } + + // ============================== Stream support + + /* + Code disabled until have a chance to test it with