dweb-transports/TransportGUN.js

367 lines
18 KiB
JavaScript
Raw Normal View History

2018-07-06 20:41:45 -07:00
/*
This Transport layers uses GUN.
*/
const Url = require('url');
2018-07-10 15:35:55 -07:00
process.env.GUN_ENV = "false";
2018-07-13 11:52:42 -07:00
const Gun = require('gun/gun.js'); // TODO-GUN switchback to gun/gun at some point to get minimized version
2018-07-10 15:35:55 -07:00
require('gun/lib/path.js');
const debuggun = require('debug')('dweb-transports:gun');
2018-07-06 20:41:45 -07:00
// Other Dweb modules
const errors = require('./Errors'); // Standard Dweb Errors
const Transport = require('./Transport.js'); // Base class for TransportXyz
const Transports = require('./Transports'); // Manage all Transports that are loaded
const utils = require('./utils'); // Utility functions
// Utility packages (ours) And one-liners
2018-07-10 15:35:55 -07:00
//unused currently: function delay(ms, val) { return new Promise(resolve => {setTimeout(() => { resolve(val); },ms)})}
2018-07-06 20:41:45 -07:00
let defaultoptions = {
2018-07-20 20:00:15 -07:00
peers: [ "https://dweb.me:4246/gun" ]
//localstore: true #True is default TODO-GUN check if false turns it off, or defaults to a different store.
};
2018-07-10 15:35:55 -07:00
//To run a superpeer - cd wherever; node install gun; cd node_modules/gun; npm start - starts server by default on port 8080, or set an "env" - see http.js
2018-07-13 13:17:16 -07:00
//setenv GUN_ENV false; node examples/http.js 4246
2018-07-10 15:35:55 -07:00
//Make sure to open of the port (typically in /etc/ferm)
2018-07-20 20:00:15 -07:00
// TODO-GUN - copy example from systemctl here
2018-07-10 15:35:55 -07:00
/*
WORKING AROUND GUN WEIRNESS/SUBOPTIMAL (of course, whats weird/sub-optimal to me, might be ideal to someone else) - search the code to see where worked around
2018-07-10 15:35:55 -07:00
WORKAROUND-GUN-UNDERSCORE .once() and possibly .on() send an extra GUN internal field "_" which needs filtering. Reported and hopefully will get fixed
2018-07-20 20:00:15 -07:00
.once behaves differently on node or the browser - this is a bug https://github.com/amark/gun/issues/586 and for now this code doesnt work on Node
WORKAROUND-GUN-CURRENT: .once() and .on() deliver existing values as well as changes, reported & hopefully will get way to find just new ones.
WORKAROUND-GUN-DELETE: There is no way to delete an item, setting it to null is recorded and is by convention a deletion. BUT the field will still show up in .once and .on,
WORKAROUND-GUN-PROMISES: GUN is not promisified, there is only one place we care, and that is .once (since .on is called multiple times).
WORKAROUND-GUN-ERRORS: GUN does an unhelpful job with errors, for example returning undefined when it cant find something (e.g. if connection to superpeer is down),
for now just throw an error on undefined
2018-07-20 20:00:15 -07:00
Errors and Promises: Note that GUN's use of promises is seriously uexpected (aka weird), see https://gun.eco/docs/SEA#errors
instead of using .reject or throwing an error at async it puts the error in SEA.err, so how that works in async parallel context is anyone's guess
2018-07-10 15:35:55 -07:00
*/
2018-07-06 20:41:45 -07:00
class TransportGUN extends Transport {
/*
GUN specific transport - over IPFS
Fields:
gun: object returned when starting GUN
*/
constructor(options) {
super(options);
2018-07-13 13:17:16 -07:00
this.options = options; // Dictionary of options
2018-07-06 20:41:45 -07:00
this.gun = undefined;
this.name = "GUN"; // For console log etc
this.supportURLs = ['gun'];
2018-07-20 20:00:15 -07:00
this.supportFunctions = [ 'fetch', //'store'
'connection', 'get', 'set', 'getall', 'keys', 'newdatabase', 'newtable', 'monitor',
2018-07-22 16:52:58 -07:00
'add', 'list', 'listmonitor', 'newlisturls'];
2018-07-06 20:41:45 -07:00
this.status = Transport.STATUS_LOADED;
}
connection(url) {
2018-07-06 20:41:45 -07:00
/*
2018-07-20 20:00:15 -07:00
TODO-GUN need to determine what a "rooted" Url is in gun, is it specific to a superpeer for example
Utility function to get Gun object for this URL (note this isn't async)
url: URL string or structure, to find list of of form [gun|dweb]:/gun/<database>/<table>[/<key] but could be arbitrary gun path
2018-07-09 19:08:56 -07:00
resolves: Gun a connection to use for get's etc, undefined if fails
2018-07-06 20:41:45 -07:00
*/
url = Url.parse(url); // Accept string or Url structure
2018-07-20 20:00:15 -07:00
let patharray = url.pathname.split('/'); //[ 'gun', database, table ] but could be arbitrary length path
patharray.shift(); // Loose leading ""
patharray.shift(); // Loose "gun"
debuggun("path=", patharray);
return this.gun.path(patharray); // Not sure how this could become undefined as it will return g before the path is walked, but if do a lookup on this "g" then should get undefined
2018-07-06 20:41:45 -07:00
}
static setup0(options) {
2018-07-06 20:41:45 -07:00
/*
First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.
options: { gun: { }, } Set of options - "gun" is used for those to pass direct to Gun
*/
2018-07-13 13:17:16 -07:00
let combinedoptions = Transport.mergeoptions(defaultoptions, options.gun);
debuggun("options %o", combinedoptions);
let t = new TransportGUN(combinedoptions); // Note doesnt start IPFS or OrbitDB
t.gun = new Gun(t.options); // This doesnt connect, just creates db structure
Transports.addtransport(t);
2018-07-06 20:41:45 -07:00
return t;
}
async p_setup1(cb) {
2018-07-06 20:41:45 -07:00
/*
This sets up for GUN.
2018-07-10 15:35:55 -07:00
Throws: TODO-GUN-DOC document possible error behavior
2018-07-06 20:41:45 -07:00
*/
try {
this.status = Transport.STATUS_STARTING; // Should display, but probably not refreshed in most case
2018-07-06 20:41:45 -07:00
if (cb) cb(this);
2018-07-09 19:08:56 -07:00
//TODO-GUN-TEST - try connect and retrieve info then look at ._.opt.peers
await this.p_status();
2018-07-06 20:41:45 -07:00
} catch(err) {
console.error(this.name,"failed to start",err);
this.status = Transport.STATUS_FAILED;
}
if (cb) cb(this);
return this;
}
async p_status() {
2018-07-06 20:41:45 -07:00
/*
2018-07-09 19:08:56 -07:00
Return an integer for the status of a transport see Transport
2018-07-06 20:41:45 -07:00
*/
2018-07-09 19:08:56 -07:00
//TODO-GUN-TEST - try connect and retrieve info then look at ._.opt.peers
this.status = Transport.STATUS_CONNECTED; //TODO-GUN how do I know if/when I'm connected (see comment on p_setup1 as well)
2018-07-06 20:41:45 -07:00
return this.status;
}
2018-07-20 20:00:15 -07:00
// ===== DATA ======
async p_rawfetch(url) {
url = Url.parse(url); // Accept url as string or object
let g = this.connection(url); // Goes all the way to the key
2018-07-20 20:00:15 -07:00
let val = await this._p_once(g);
if (!val) throw new errors.TransportError("GUN unable to retrieve: "+url.href); // WORKAROUND-GUN-ERRORS - gun doesnt throw errors when it cant find something
let o = typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync (see same code on p_get and p_rawfetch)
//TODO-GUN this is a hack because the metadata such as metadata/audio is getting cached in GUN and in this case is wrong.
if (o.metadata && o.metadata.thumbnaillinks && o.metadata.thumbnaillinks.find(t => t.includes('ipfs/zb2'))) {
throw new errors.TransportError("GUN retrieving legacy data at: "+url.href)
}
return o;
2018-07-20 20:00:15 -07:00
}
2018-07-09 19:08:56 -07:00
// ===== LISTS ========
2018-07-06 20:41:45 -07:00
// noinspection JSCheckFunctionSignatures
async p_rawlist(url) {
2018-07-09 19:08:56 -07:00
/*
Fetch all the objects in a list, these are identified by the url of the public key used for signing.
(Note this is the 'signedby' parameter of the p_rawadd call, not the 'url' parameter
Returns a promise that resolves to the list.
Each item of the list is a dict: {"url": url, "date": date, "signature": signature, "signedby": signedby}
List items may have other data (e.g. reference ids of underlying transport)
:param string url: String with the url that identifies the list.
:resolve array: An array of objects as stored on the list.
*/
try {
let g = this.connection(url);
2018-07-13 11:52:42 -07:00
let data = await this._p_once(g);
2018-07-20 20:00:15 -07:00
let res = data ? Object.keys(data).filter(k => k !== '_').sort().map(k => data[k]) : []; //See WORKAROUND-GUN-UNDERSCORE
2018-07-09 19:08:56 -07:00
// .filter((obj) => (obj.signedby.includes(url))); // upper layers verify, which filters
debuggun("p_rawlist found", ...utils.consolearr(res));
2018-07-09 19:08:56 -07:00
return res;
} catch(err) {
// Will be logged by Transports
2018-07-09 19:08:56 -07:00
throw(err);
}
}
listmonitor(url, callback, {current=false}={}) {
2018-07-06 20:41:45 -07:00
/*
2018-07-09 19:08:56 -07:00
Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_rawlist to get any more items not returned by p_rawlist.
2018-07-06 20:41:45 -07:00
2018-07-09 19:08:56 -07:00
url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
callback: function(obj) Callback for each new item added to the list
obj is same format as p_rawlist or p_rawreverse
current true if should send list of existing elements
*/
let g = this.connection(url);
2018-07-20 20:00:15 -07:00
if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new.
2018-07-13 11:52:42 -07:00
g.once(data => {
2018-07-16 19:59:07 -07:00
this.monitored = data ? Object.keys(data) : []; // Keep a copy - could actually just keep high water mark unless getting partial knowledge of state of array.
2018-07-13 11:52:42 -07:00
g.map().on((v, k) => {
2018-07-20 20:00:15 -07:00
if (!(this.monitored.includes(k)) && (k !== '_')) { //See WORKAROUND-GUN-UNDERSCORE
2018-07-22 16:52:58 -07:00
this.monitored.push(k);
2018-07-13 11:52:42 -07:00
callback(JSON.parse(v));
}
});
2018-07-09 19:08:56 -07:00
});
} else {
2018-07-13 11:52:42 -07:00
g.map().on((v, k) => callback("set", k, JSON.parse(v)));
2018-07-09 19:08:56 -07:00
}
}
// noinspection JSCheckFunctionSignatures
async p_rawadd(url, sig) {
2018-07-09 19:08:56 -07:00
/*
Store a new list item, it should be stored so that it can be retrieved either by "signedby" (using p_rawlist) or
by "url" (with p_rawreverse). The underlying transport does not need to guarantee the signature,
an invalid item on a list should be rejected on higher layers.
:param string url: String identifying list to post to
:param Signature sig: Signature object containing at least:
date - date of signing in ISO format,
urls - array of urls for the object being signed
signature - verifiable signature of date+urls
signedby - urls of public key used for the signature
:resolve undefined:
2018-07-06 20:41:45 -07:00
*/
2018-07-10 15:35:55 -07:00
// noinspection JSUnresolvedVariable
// Logged by Transports
2018-07-09 19:08:56 -07:00
console.assert(url && sig.urls.length && sig.signature && sig.signedby.length, "TransportGUN.p_rawadd args", url, sig);
this.connection(url)
2018-07-09 19:08:56 -07:00
.set( JSON.stringify( sig.preflight( Object.assign({}, sig))));
}
// noinspection JSCheckFunctionSignatures
async p_newlisturls(cl) {
let u = await this._p_newgun(cl);
2018-07-10 15:35:55 -07:00
return [ u, u];
2018-07-09 19:08:56 -07:00
}
//=======KEY VALUE TABLES ========
// noinspection JSMethodCanBeStatic
async _p_newgun(pubkey) {
2018-07-06 20:41:45 -07:00
if (pubkey.hasOwnProperty("keypair"))
pubkey = pubkey.keypair.signingexport();
2018-07-06 20:41:45 -07:00
// By this point pubkey should be an export of a public key of form xyz:abc where xyz
// specifies the type of public key (NACL VERIFY being the only kind we expect currently)
2018-07-10 15:35:55 -07:00
return `gun:/gun/${encodeURIComponent(pubkey)}`;
2018-07-06 20:41:45 -07:00
}
async p_newdatabase(pubkey) {
2018-07-09 19:08:56 -07:00
/*
Request a new database
For GUN it doesnt actually create anything, just generates the URLs
2018-07-10 15:35:55 -07:00
TODO-GUN simple version first - userid based on my keypair first, then switch to Gun's userid and its keypair
2018-07-09 19:08:56 -07:00
Include gun/sea.js; user.create(<alias>,<passphrase>); user.auth(<alias>,<passphrase>); # See gun.eco/docs/Auth
returns: {publicurl: "gun:/gun/<publickey>", privateurl: "gun:/gun/<publickey>">
*/
let u = await this._p_newgun(pubkey);
2018-07-10 15:35:55 -07:00
return {publicurl: u, privateurl: u};
2018-07-09 19:08:56 -07:00
}
2018-07-06 20:41:45 -07:00
async p_newtable(pubkey, table) {
2018-07-06 20:41:45 -07:00
/*
Request a new table
For GUN it doesnt actually create anything, just generates the URLs
returns: {publicurl: "gun:/gun/<publickey>/<table>", privateurl: "gun:/gun/<publickey>/<table>">
*/
if (!pubkey) throw new errors.CodingError("p_newtable currently requires a pubkey");
let database = await this.p_newdatabase(pubkey);
2018-07-06 20:41:45 -07:00
// If have use cases without a database, then call p_newdatabase first
return { privateurl: `${database.privateurl}/${table}`, publicurl: `${database.publicurl}/${table}`} // No action required to create it
}
async p_set(url, keyvalues, value) { // url = yjs:/yjs/database/table
2018-07-06 20:41:45 -07:00
/*
Set key values
keyvalues: string (key) in which case value should be set there OR
object in which case value is ignored
*/
let table = this.connection(url);
2018-07-06 20:41:45 -07:00
if (typeof keyvalues === "string") {
table.path(keyvalues).put(JSON.stringify(value));
} else {
2018-07-10 15:35:55 -07:00
// Store all key-value pairs without destroying any other key/value pairs previously set
console.assert(!Array.isArray(keyvalues), "TransportGUN - shouldnt be passsing an array as the keyvalues");
table.put(
Object.keys(keyvalues).reduce(
function(previous, key) { previous[key] = JSON.stringify(keyvalues[key]); return previous; },
{}
))
2018-07-06 20:41:45 -07:00
}
}
2018-07-10 15:35:55 -07:00
async p_get(url, keys) {
let table = this.connection(url);
2018-07-06 20:41:45 -07:00
if (Array.isArray(keys)) {
throw new errors.ToBeImplementedError("p_get(url, [keys]) isn't supported - because of ambiguity better to explicitly loop on set of keys or use getall and filter");
2018-07-10 15:35:55 -07:00
/*
2018-07-06 20:41:45 -07:00
return keys.reduce(function(previous, key) {
let val = table.get(key);
previous[key] = typeof val === "string" ? JSON.parse(val) : val; // Handle undefined
return previous;
}, {});
2018-07-10 15:35:55 -07:00
*/
2018-07-06 20:41:45 -07:00
} else {
let val = await this._p_once(table.get(keys)); // Resolves to value
2018-07-20 20:00:15 -07:00
return typeof val === "string" ? JSON.parse(val) : val; // This looks like it is sync (see same code on p_get and p_rawfetch)
2018-07-06 20:41:45 -07:00
}
}
async p_delete(url, keys) {
let table = this.connection(url);
2018-07-06 20:41:45 -07:00
if (typeof keys === "string") {
table.path(keys).put(null);
} else {
keys.map((key) => table.path(key).put(null)); // This looks like it is sync
}
}
2018-07-20 20:00:15 -07:00
//WORKAROUND-GUN-PROMISE suggest p_once as a good single addition
//TODO-GUN expand this to workaround Gun weirdness with errors.
2018-07-22 16:52:58 -07:00
_p_once(gun) { // Note in some cases (e.g. p_getall) this will resolve to a object, others a string/number (p_get)
return new Promise((resolve) => gun.once(resolve));
2018-07-06 20:41:45 -07:00
}
2018-07-10 15:35:55 -07:00
async p_keys(url) {
let res = await this._p_once(this.connection(url));
2018-07-10 15:35:55 -07:00
return Object.keys(res)
2018-07-20 20:00:15 -07:00
.filter(k=> (k !== '_') && (res[k] !== null)); //See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE
2018-07-06 20:41:45 -07:00
}
2018-07-10 15:35:55 -07:00
async p_getall(url) {
let res = await this._p_once(this.connection(url));
2018-07-10 15:35:55 -07:00
return Object.keys(res)
2018-07-20 20:00:15 -07:00
.filter(k=> (k !== '_') && res[k] !== null) //See WORKAROUND-GUN-UNDERSCORE and WORKAROUND-GUN-DELETE
2018-07-10 15:35:55 -07:00
.reduce( function(previous, key) { previous[key] = JSON.parse(res[key]); return previous; }, {});
2018-07-06 20:41:45 -07:00
}
async monitor(url, callback, {current=false}={}) {
2018-07-06 20:41:45 -07:00
/*
Setup a callback called whenever an item is added to a list, typically it would be called immediately after a p_getall to get any more items not returned by p_getall.
Stack: KVT()|KVT.p_new => KVT.monitor => (a: Transports.monitor => GUN.monitor)(b: dispatchEvent)
2018-07-09 19:08:56 -07:00
url: string Identifier of list (as used by p_rawlist and "signedby" parameter of p_rawadd
callback: function({type, key, value}) Callback for each new item added to the list (type = "set"|"delete")
current Send existing items to the callback as well
2018-07-06 20:41:45 -07:00
*/
let g = this.connection(url);
2018-07-20 20:00:15 -07:00
if (!current) { // See WORKAROUND-GUN-CURRENT have to keep an extra copy to compare for which calls are new.
g.once(data => {
this.monitored = Object.assign({},data); // Make a copy of data (this.monitored = data won't work as just points at same structure)
g.map().on((v, k) => {
2018-07-20 20:00:15 -07:00
if ((v !== this.monitored[k]) && (k !== '_')) { //See WORKAROUND-GUN-UNDERSCORE
this.monitored[k] = v;
callback("set", k, JSON.parse(v));
}
});
2018-07-09 19:08:56 -07:00
});
} else {
2018-07-13 11:52:42 -07:00
g.map().on((v, k) => callback("set", k, JSON.parse(v)));
2018-07-09 19:08:56 -07:00
}
2018-07-06 20:41:45 -07:00
}
static async p_test() {
debuggun("p_test");
2018-07-06 20:41:45 -07:00
try {
let t = this.setup0({}); //TODO-GUN when works with peers commented out, try passing peers: []
await t.p_setup1(); // Not passing cb yet
await t.p_setup2(); // Not passing cb yet - this one does nothing on GUN
// noinspection JSIgnoredPromiseFromCall
t.p_test_kvt("gun:/gun/NACL");
//t.p_test_list("gun:/gun/NACL"); //TODO test_list needs fixing to not create a dependency on Signature
2018-07-06 20:41:45 -07:00
} catch(err) {
console.warn("Exception thrown in TransportGUN.test:", err.message);
2018-07-06 20:41:45 -07:00
throw err;
}
}
// noinspection JSUnusedGlobalSymbols
2018-07-10 15:35:55 -07:00
static async demo_bugs() {
let gun = new Gun();
gun.get('foo').get('bar').put('baz');
console.log("Expect {bar: 'baz'} but get {_:..., bar: 'baz'}");
gun.get('foo').once(data => console.log(data));
gun.get('zip').get('bar').set('alice');
console.log("Expect {12345: 'alice'} but get {_:..., 12345: 'alice'}");
gun.get('foo').once(data => console.log(data));
// Returns extra "_" field
}
2018-07-06 20:41:45 -07:00
}
Transports._transportclasses["GUN"] = TransportGUN;
exports = module.exports = TransportGUN;