implement mongodb storage

This commit is contained in:
Ivan Socolsky 2015-04-20 17:04:26 -03:00
parent 706079da82
commit 1b2b0dc146
4 changed files with 338 additions and 269 deletions

View File

@ -1,5 +1,9 @@
var Model = {}; var Model = {};
Model.Wallet = require('./wallet'); Model.Wallet = require('./wallet');
Model.Copayer = require('./copayer');
Model.TxProposal = require('./txproposal');
Model.Address = require('./address');
Model.Notification = require('./notification');
module.exports = Model; module.exports = Model;

View File

@ -1,7 +1,6 @@
'use strict'; 'use strict';
var _ = require('lodash'); var _ = require('lodash');
var levelup = require('mongodb');
var async = require('async'); var async = require('async');
var $ = require('preconditions').singleton(); var $ = require('preconditions').singleton();
var log = require('npmlog'); var log = require('npmlog');
@ -11,11 +10,7 @@ var util = require('util');
var mongodb = require('mongodb'); var mongodb = require('mongodb');
var Wallet = require('./model/wallet'); var Model = require('./model');
var Copayer = require('./model/copayer');
var Address = require('./model/address');
var TxProposal = require('./model/txproposal');
var Notification = require('./model/notification');
var Storage = function(opts) { var Storage = function(opts) {
opts = opts || {}; opts = opts || {};
@ -34,87 +29,51 @@ var Storage = function(opts) {
} }
}; };
var zeroPad = function(x, length) {
return _.padLeft(parseInt(x), length, '0');
};
var walletPrefix = function(id) {
return 'w!' + id;
};
var opKey = function(key) {
return key ? '!' + key : '';
};
var MAX_TS = _.repeat('9', 14);
var KEY = {
WALLET: function(walletId) {
return walletPrefix(walletId) + '!main';
},
COPAYER: function(id) {
return 'copayer!' + id;
},
TXP: function(walletId, txProposalId) {
return walletPrefix(walletId) + '!txp' + opKey(txProposalId);
},
NOTIFICATION: function(walletId, notificationId) {
return walletPrefix(walletId) + '!not' + opKey(notificationId);
},
PENDING_TXP: function(walletId, txProposalId) {
return walletPrefix(walletId) + '!ptxp' + opKey(txProposalId);
},
ADDRESS: function(walletId, address) {
return walletPrefix(walletId) + '!addr' + opKey(address);
},
};
Storage.prototype.fetchWallet = function(id, cb) { Storage.prototype.fetchWallet = function(id, cb) {
this.db.get(KEY.WALLET(id), function(err, data) { this.db.collection('wallets').findOne({
if (err) { id: id
if (err.notFound) return cb(); }, function(err, result) {
return cb(err); if (err) return cb(err);
} if (!result) return cb();
return cb(null, Wallet.fromObj(data)); return cb(null, Model.Wallet.fromObj(result));
}); });
}; };
Storage.prototype.storeWallet = function(wallet, cb) { Storage.prototype.storeWallet = function(wallet, cb) {
this.db.put(KEY.WALLET(wallet.id), wallet, cb); this.db.collection('wallets').update({
id: wallet.id
}, wallet, {
w: 1,
upsert: true,
}, cb);
}; };
Storage.prototype.storeWalletAndUpdateCopayersLookup = function(wallet, cb) { Storage.prototype.storeWalletAndUpdateCopayersLookup = function(wallet, cb) {
var ops = []; return this.storeWallet(wallet, cb);
ops.push({
type: 'put',
key: KEY.WALLET(wallet.id),
value: wallet
});
_.each(wallet.copayers, function(copayer) {
var value = {
walletId: wallet.id,
requestPubKey: copayer.requestPubKey,
};
ops.push({
type: 'put',
key: KEY.COPAYER(copayer.id),
value: value
});
});
this.db.batch(ops, cb);
}; };
Storage.prototype.fetchCopayerLookup = function(copayerId, cb) { Storage.prototype.fetchCopayerLookup = function(copayerId, cb) {
this.db.get(KEY.COPAYER(copayerId), function(err, data) { this.db.collection('wallets').findOne({
if (err) { 'copayers.id': copayerId
if (err.notFound) return cb(); }, {
return cb(err); fields: {
} id: 1,
return cb(null, data); copayers: 1,
},
}, function(err, result) {
if (err) return cb(err);
if (!result) return cb();
var copayer = _.find(result.copayers, {
id: copayerId
});
return cb(null, {
walletId: result.id,
requestPubKey: copayer.requestPubKey,
});
}); });
}; };
// TODO: should be done client-side
Storage.prototype._completeTxData = function(walletId, txs, cb) { Storage.prototype._completeTxData = function(walletId, txs, cb) {
var txList = [].concat(txs); var txList = [].concat(txs);
this.fetchWallet(walletId, function(err, wallet) { this.fetchWallet(walletId, function(err, wallet) {
@ -131,12 +90,14 @@ Storage.prototype._completeTxData = function(walletId, txs, cb) {
Storage.prototype.fetchTx = function(walletId, txProposalId, cb) { Storage.prototype.fetchTx = function(walletId, txProposalId, cb) {
var self = this; var self = this;
this.db.get(KEY.TXP(walletId, txProposalId), function(err, data) {
if (err) { this.db.collection('txs').findOne({
if (err.notFound) return cb(); id: txProposalId,
return cb(err); walletId: walletId
} }, function(err, result) {
return self._completeTxData(walletId, TxProposal.fromObj(data), cb); if (err) return cb(err);
if (!result) return cb();
return self._completeTxData(walletId, Model.TxProposal.fromObj(result), cb);
}); });
}; };
@ -144,22 +105,19 @@ Storage.prototype.fetchTx = function(walletId, txProposalId, cb) {
Storage.prototype.fetchPendingTxs = function(walletId, cb) { Storage.prototype.fetchPendingTxs = function(walletId, cb) {
var self = this; var self = this;
var txs = []; this.db.collection('txs').find({
var key = KEY.PENDING_TXP(walletId); walletId: walletId,
this.db.createReadStream({ isPending: true
gte: key, }).sort({
lt: key + '~' createdOn: 1
}) }).toArray(function(err, result) {
.on('data', function(data) { if (err) return cb(err);
txs.push(TxProposal.fromObj(data.value)); if (!result) return cb();
}) var txs = _.map(result, function(tx) {
.on('error', function(err) { return Model.TxProposal.fromObj(tx);
if (err.notFound) return cb();
return cb(err);
})
.on('end', function() {
return self._completeTxData(walletId, txs, cb);
}); });
return self._completeTxData(walletId, txs, cb);
});
}; };
/** /**
@ -173,31 +131,30 @@ Storage.prototype.fetchPendingTxs = function(walletId, cb) {
Storage.prototype.fetchTxs = function(walletId, opts, cb) { Storage.prototype.fetchTxs = function(walletId, opts, cb) {
var self = this; var self = this;
var txs = [];
opts = opts || {}; opts = opts || {};
opts.limit = _.isNumber(opts.limit) ? parseInt(opts.limit) : -1;
opts.minTs = _.isNumber(opts.minTs) ? zeroPad(opts.minTs, 11) : 0;
opts.maxTs = _.isNumber(opts.maxTs) ? zeroPad(opts.maxTs, 11) : MAX_TS;
var key = KEY.TXP(walletId, opts.minTs); var tsFilter = {};
var endkey = KEY.TXP(walletId, opts.maxTs); if (_.isNumber(opts.minTs)) tsFilter.$gte = opts.minTs;
if (_.isNumber(opts.maxTs)) tsFilter.$lte = opts.maxTs;
this.db.createReadStream({ var filter = {
gt: key, walletId: walletId
lt: endkey + '~', };
reverse: true, if (!_.isEmpty(tsFilter)) filter.createdOn = tsFilter;
limit: opts.limit,
}) var mods = {};
.on('data', function(data) { if (_.isNumber(opts.limit)) mods.limit = opts.limit;
txs.push(TxProposal.fromObj(data.value));
}) this.db.collection('txs').find(filter, mods).sort({
.on('error', function(err) { createdOn: 1
if (err.notFound) return cb(); }).toArray(function(err, result) {
return cb(err); if (err) return cb(err);
}) if (!result) return cb();
.on('end', function() { var txs = _.map(result, function(tx) {
return self._completeTxData(walletId, txs, cb); return Model.TxProposal.fromObj(tx);
}); });
return self._completeTxData(walletId, txs, cb);
});
}; };
@ -210,181 +167,133 @@ Storage.prototype.fetchTxs = function(walletId, opts, cb) {
* @param opts.limit * @param opts.limit
*/ */
Storage.prototype.fetchNotifications = function(walletId, opts, cb) { Storage.prototype.fetchNotifications = function(walletId, opts, cb) {
var txs = []; var self = this;
opts = opts || {}; opts = opts || {};
opts.limit = _.isNumber(opts.limit) ? parseInt(opts.limit) : -1;
opts.minTs = _.isNumber(opts.minTs) ? zeroPad(opts.minTs, 11) : 0;
opts.maxTs = _.isNumber(opts.maxTs) ? zeroPad(opts.maxTs, 11) : MAX_TS;
var key = KEY.NOTIFICATION(walletId, opts.minTs); var tsFilter = {};
var endkey = KEY.NOTIFICATION(walletId, opts.maxTs); if (_.isNumber(opts.minTs)) tsFilter.$gte = opts.minTs;
if (_.isNumber(opts.maxTs)) tsFilter.$lte = opts.maxTs;
this.db.createReadStream({ var filter = {
gt: key, walletId: walletId
lt: endkey + '~', };
reverse: opts.reverse, if (!_.isEmpty(tsFilter)) filter.createdOn = tsFilter;
limit: opts.limit,
}) var mods = {};
.on('data', function(data) { if (_.isNumber(opts.limit)) mods.limit = opts.limit;
txs.push(Notification.fromObj(data.value));
}) this.db.collection('notifications').find(filter, mods).sort({
.on('error', function(err) { createdOn: 1
if (err.notFound) return cb(); }).toArray(function(err, result) {
return cb(err); if (err) return cb(err);
}) if (!result) return cb();
.on('end', function() { var notifications = _.map(result, function(notification) {
return cb(null, txs); return Model.Notification.fromObj(notification);
}); });
return cb(null, notifications);
});
}; };
// TODO: remove walletId from signature
Storage.prototype.storeNotification = function(walletId, notification, cb) { Storage.prototype.storeNotification = function(walletId, notification, cb) {
this.db.put(KEY.NOTIFICATION(walletId, notification.id), notification, cb); this.db.collection('notifications').insert(notification, {
w: 1
}, cb);
}; };
// TODO: remove walletId from signature
// TODO should we store only txp.id on keys for indexing
// or the whole txp? For now, the entire record makes sense
// (faster + easier to access)
Storage.prototype.storeTx = function(walletId, txp, cb) { Storage.prototype.storeTx = function(walletId, txp, cb) {
var ops = [{ txp.isPending = txp.isPending(); // Persist attribute to use when querying
type: 'put', this.db.collection('txs').update({
key: KEY.TXP(walletId, txp.id), id: txp.id,
value: txp, walletId: walletId
}]; }, txp, {
w: 1,
if (txp.isPending()) { upsert: true,
ops.push({ }, cb);
type: 'put',
key: KEY.PENDING_TXP(walletId, txp.id),
value: txp,
});
} else {
ops.push({
type: 'del',
key: KEY.PENDING_TXP(walletId, txp.id),
});
}
this.db.batch(ops, cb);
}; };
Storage.prototype.removeTx = function(walletId, txProposalId, cb) { Storage.prototype.removeTx = function(walletId, txProposalId, cb) {
var ops = [{ this.db.collection('txs').findAndRemove({
type: 'del', id: txProposalId,
key: KEY.TXP(walletId, txProposalId), walletId: walletId
}, { }, {
type: 'del', w: 1
key: KEY.PENDING_TXP(walletId, txProposalId), }, cb);
}];
this.db.batch(ops, cb);
};
Storage.prototype._delByKey = function(key, cb) {
var self = this;
var keys = [];
this.db.createKeyStream({
gte: key,
lt: key + '~',
})
.on('data', function(key) {
keys.push(key);
})
.on('error', function(err) {
if (err.notFound) return cb();
return cb(err);
})
.on('end', function(err) {
self.db.batch(_.map(keys, function(k) {
return {
key: k,
type: 'del'
};
}), function(err) {
return cb(err);
});
});
};
Storage.prototype._removeCopayers = function(walletId, cb) {
var self = this;
this.fetchWallet(walletId, function(err, w) {
if (err || !w) return cb(err);
self.db.batch(_.map(w.copayers, function(c) {
return {
type: 'del',
key: KEY.COPAYER(c.id),
};
}), cb);
});
}; };
Storage.prototype.removeWallet = function(walletId, cb) { Storage.prototype.removeWallet = function(walletId, cb) {
var self = this; var self = this;
async.series([ async.parallel([
function(next) { function(next) {
// This should be the first step. Will check the wallet exists this.db.collections('wallets').findAndRemove({
self._removeCopayers(walletId, next); id: walletId
}, next);
}, },
function(next) { function(next) {
self._delByKey(walletPrefix(walletId), cb); this.db.collections('addresses').findAndRemove({
walletId: walletId
}, next);
},
function(next) {
this.db.collections('txs').findAndRemove({
walletId: walletId
}, next);
},
function(next) {
this.db.collections('notifications').findAndRemove({
walletId: walletId
}, next);
}, },
], cb); ], cb);
}; };
Storage.prototype.fetchAddresses = function(walletId, cb) { Storage.prototype.fetchAddresses = function(walletId, cb) {
var addresses = []; var self = this;
var key = KEY.ADDRESS(walletId);
this.db.createReadStream({ this.db.collection('addresses').find({
gte: key, walletId: walletId,
lt: key + '~' }).sort({
}) createdOn: 1
.on('data', function(data) { }).toArray(function(err, result) {
addresses.push(Address.fromObj(data.value)); if (err) return cb(err);
}) if (!result) return cb();
.on('error', function(err) { var addresses = _.map(result, function(address) {
if (err.notFound) return cb(); return Model.Address.fromObj(address);
return cb(err);
})
.on('end', function() {
return cb(null, addresses);
}); });
return cb(null, addresses);
});
}; };
Storage.prototype.storeAddressAndWallet = function(wallet, addresses, cb) { Storage.prototype.storeAddressAndWallet = function(wallet, addresses, cb) {
var ops = _.map([].concat(addresses), function(address) { var self = this;
return { this.db.collection('addresses').insert([].concat(addresses), {
type: 'put', w: 1
key: KEY.ADDRESS(wallet.id, address.address), }, function(err) {
value: address, if (err) return cb(err);
}; self.storeWallet(wallet, cb);
}); });
ops.unshift({
type: 'put',
key: KEY.WALLET(wallet.id),
value: wallet,
});
this.db.batch(ops, cb);
}; };
Storage.prototype._dump = function(cb, fn) { Storage.prototype._dump = function(cb, fn) {
fn = fn || console.log; fn = fn || console.log;
this.db.readStream() var self = this;
.on('data', function(data) { this.db.collections(function(err, collections) {
fn(util.inspect(data, { if (err) return cb(err);
depth: 10 async.eachSeries(collections, function(col, next) {
})); fn('--------' + col);
}) col.find().toArray(function(err, item) {
.on('end', function() { fn(item);
if (cb) return cb(); next(err);
}); });
}, cb);
});
}; };
module.exports = Storage; module.exports = Storage;

View File

@ -52,7 +52,8 @@
"memdown": "^1.0.0", "memdown": "^1.0.0",
"mocha": "^1.18.2", "mocha": "^1.18.2",
"sinon": "^1.10.3", "sinon": "^1.10.3",
"supertest": "*" "supertest": "*",
"tingodb": "^0.3.4"
}, },
"scripts": { "scripts": {
"start": "node bws.js", "start": "node bws.js",
@ -60,14 +61,11 @@
"test": "./node_modules/.bin/mocha", "test": "./node_modules/.bin/mocha",
"coveralls": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage" "coveralls": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage"
}, },
"contributors": [ "contributors": [{
{ "name": "Ivan Socolsky",
"name": "Ivan Socolsky", "email": "ivan@bitpay.com"
"email": "ivan@bitpay.com" }, {
}, "name": "Matias Alejo Garcia",
{ "email": "ematiu@gmail.com"
"name": "Matias Alejo Garcia", }]
"email": "ematiu@gmail.com"
}
]
} }

View File

@ -1,24 +1,62 @@
'use strict'; 'use strict';
var _ = require('lodash'); var _ = require('lodash');
var async = require('async');
var chai = require('chai'); var chai = require('chai');
var sinon = require('sinon'); var sinon = require('sinon');
var should = chai.should(); var should = chai.should();
var levelup = require('levelup'); var levelup = require('levelup');
var memdown = require('memdown'); var memdown = require('memdown');
var Storage = require('../lib/storage'); var mongodb = require('mongodb');
var StorageLevelDb = require('../lib/storage');
var StorageMongoDb = require('../lib/storage_mongo');
var Model = require('../lib/model'); var Model = require('../lib/model');
describe.only('Storage', function() { function initStorageLevelDb(cb) {
var storage; var db = levelup(memdown, {
beforeEach(function() { valueEncoding: 'json'
var db = levelup(memdown, { });
valueEncoding: 'json' return cb(null, db);
};
function initStorageMongoDb(cb) {
var url = 'mongodb://localhost:27017/bws';
mongodb.MongoClient.connect(url, function(err, db) {
should.not.exist(err);
db.dropDatabase(function(err) {
return cb(null, db);
}); });
storage = new Storage({ });
db: db };
var Storage, initDb;
var useLevel = false;
if (useLevel) {
Storage = StorageLevelDb;
initDb = initStorageLevelDb;
} else {
Storage = StorageMongoDb;
initDb = initStorageMongoDb;
}
describe('Storage', function() {
var storage;
beforeEach(function(done) {
initDb(function(err, db) {
should.not.exist(err);
storage = new Storage({
db: db
});
done();
}); });
}); });
describe('Store & fetch wallet', function() { describe('Store & fetch wallet', function() {
@ -51,4 +89,124 @@ describe.only('Storage', function() {
}); });
}); });
}); });
describe('Copayer lookup', function() {
it('should correctly store and fetch copayer lookup', function(done) {
var wallet = Model.Wallet.create({
id: '123',
name: 'my wallet',
m: 2,
n: 3,
});
_.each(_.range(3), function(i) {
var copayer = Model.Copayer.create({
name: 'copayer ' + i,
xPubKey: 'xPubKey ' + i,
requestPubKey: 'requestPubKey ' + i,
});
wallet.addCopayer(copayer);
});
should.exist(wallet);
storage.storeWalletAndUpdateCopayersLookup(wallet, function(err) {
should.not.exist(err);
storage.fetchCopayerLookup(wallet.copayers[1].id, function(err, lookup) {
should.not.exist(err);
should.exist(lookup);
lookup.walletId.should.equal('123');
lookup.requestPubKey.should.equal('requestPubKey 1');
done();
})
});
});
it('should not return error if copayer not found', function(done) {
storage.fetchCopayerLookup('2', function(err, lookup) {
should.not.exist(err);
should.not.exist(lookup);
done();
});
});
});
describe('Transaction proposals', function() {
var wallet, proposals;
beforeEach(function(done) {
wallet = Model.Wallet.create({
id: '123',
name: 'my wallet',
m: 2,
n: 3,
});
_.each(_.range(3), function(i) {
var copayer = Model.Copayer.create({
name: 'copayer ' + i,
xPubKey: 'xPubKey ' + i,
requestPubKey: 'requestPubKey ' + i,
});
wallet.addCopayer(copayer);
});
should.exist(wallet);
storage.storeWalletAndUpdateCopayersLookup(wallet, function(err) {
should.not.exist(err);
proposals = _.map(_.range(4), function(i) {
var tx = Model.TxProposal.create({
walletId: '123',
creatorId: wallet.copayers[0].id,
amount: i + 100,
});
if (i % 2 == 0) {
tx.status = 'rejected';
tx.isPending().should.be.false;
}
return tx;
});
async.each(proposals, function(tx, next) {
storage.storeTx('123', tx, next);
}, function(err) {
should.not.exist(err);
done();
});
});
});
it('should fetch tx', function(done) {
storage.fetchTx('123', proposals[0].id, function(err, tx) {
should.not.exist(err);
should.exist(tx);
tx.id.should.equal(proposals[0].id);
tx.walletId.should.equal(proposals[0].walletId);
tx.creatorName.should.equal('copayer 0');
done();
});
});
it('should fetch all pending txs', function(done) {
storage.fetchPendingTxs('123', function(err, txs) {
should.not.exist(err);
should.exist(txs);
txs.length.should.equal(2);
txs = _.sortBy(txs, 'amount');
txs[0].amount.should.equal(101);
txs[1].amount.should.equal(103);
done();
});
});
it('should remove tx', function(done) {
storage.removeTx('123', proposals[0].id, function(err) {
should.not.exist(err);
storage.fetchTx('123', proposals[0].id, function(err, tx) {
should.not.exist(err);
should.not.exist(tx);
storage.fetchTxs('123', {}, function(err, txs) {
should.not.exist(err);
should.exist(txs);
txs.length.should.equal(3);
_.any(txs, {
id: proposals[0].id
}).should.be.false;
done();
});
});
});
});
});
}); });