zcash-hackworks
/
insight-ui-zcash
mirror of https://github.com/zcash-hackworks/insight-ui-zcash.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

smart sync. Skips genesisTX

This commit is contained in:
Matias Alejo Garcia 2014-01-17 16:36:34 -03:00
parent 82beb27a3c
commit 80d3c24755
6 changed files with 97 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/models/Block.js
View File

@ -55,6 +55,7 @@ BlockSchema.statics.customCreate = function(block, cb) {
newBlock.hash = block.hash;
newBlock.nextBlockHash = block.nextBlockHash;
Transaction.createFromArray(block.tx, newBlock.time, function(err, inserted_txs) {
if (err) return cb(err);

7
app/models/Transaction.js
View File

@ -19,6 +19,9 @@ var mongoose = require('mongoose'),
var CONCURRENCY = 5;
// TODO: use bitcore networks module
var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
/**
*/
var TransactionSchema = new Schema({
@ -113,6 +116,10 @@ TransactionSchema.statics.createFromArray = function(txs, time, next) {
TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) {
// Is it from genesis block? (testnet==livenet)
// TODO: parse it from networks.genesisTX
if (txid === genesisTXID) return cb();
this.queryInfo(txid, function(err, info) {
if (err || !info) return cb(err);

106
lib/HistoricSync.js
View File

@ -16,6 +16,10 @@ function spec() {
this.block_count= 0;
this.block_total= 0;
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
var genesisHashReversed = new Buffer(32);
this.network.genesisBlock.hash.copy(genesisHashReversed);
this.genesis = genesisHashReversed.reverse().toString('hex');
this.sync = new Sync(opts);
}
@ -44,12 +48,10 @@ function spec() {
HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
var that = this;
var self = this;
// recursion end.
if (!blockHash || (blockEnd && blockEnd === blockHash) ) {
return cb();
}
if (!blockHash ) return cb();
var existed = 0;
var blockInfo;
@ -61,17 +63,16 @@ function spec() {
Block.findOne({hash:blockHash}, function(err,block){
if (err) { p(err); return c(err); }
if (block) {
existed = 1;
blockObj = block;
existed =1;
blockObj =block;
}
return c();
});
},
//show some (inacurate) status
function(c) {
if (that.block_count % 1000 === 1) {
progress_bar('sync status:', that.block_count, that.block_total);
if (self.block_count % 1000 === 1) {
progress_bar('sync status:', self.block_count, self.block_total);
}
return c();
},
@ -81,7 +82,7 @@ function spec() {
// TODO: if we store prev/next, no need to go to RPC
// if (blockObj && blockObj.nextBlockHash) return c();
that.rpc.getBlock(blockHash, function(err, ret) {
self.rpc.getBlock(blockHash, function(err, ret) {
if (err) return c(err);
blockInfo = ret;
@ -91,8 +92,10 @@ function spec() {
//store it
function(c) {
if (existed) return c();
that.sync.storeBlock(blockInfo.result, function(err) {
self.sync.storeBlock(blockInfo.result, function(err) {
existed = err && err.toString().match(/E11000/);
if (err && ! existed) return c(err);
return c();
});
@ -110,39 +113,36 @@ function spec() {
],
function (err){
if (err)
p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count);
if (opts.uptoexisting && existed) {
p('DONE. Found existing block: ', blockHash);
return cb(err);
}
if (err)
p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count);
if (blockEnd && blockEnd === blockHash) {
p('DONE. Found END block: ', blockHash);
return cb(err);
}
// Continue
if (blockInfo && blockInfo.result) {
block_total++;
self.block_count++;
if (opts.prev && blockInfo.result.previousblockhash) {
return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
}
if (opts.next && blockInfo.result.nextblockhash)
return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
}
return cb(err);
});
};
HistoricSync.prototype.syncBlocks = function(start, end, opts, cb) {
var that = this;
p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));
return that.getPrevNextBlock( start, end, opts , cb);
};
HistoricSync.prototype.import_history = function(opts, next) {
var that = this;
var self = this;
var retry_attemps = 100;
var retry_secs = 2;
@ -154,7 +154,7 @@ function spec() {
function(cb) {
if (opts.destroy) {
p('Deleting Blocks...');
that.db.collections.blocks.drop(cb);
self.db.collections.blocks.drop(cb);
} else {
return cb();
}
@ -162,7 +162,7 @@ function spec() {
function(cb) {
if (opts.destroy) {
p('Deleting TXs...');
that.db.collections.transactions.drop(cb);
self.db.collections.transactions.drop(cb);
} else {
return cb();
}
@ -170,16 +170,16 @@ function spec() {
function(cb) {
if (opts.destroy) {
p('Deleting TXItems...');
that.db.collections.transactionitems.drop(cb);
self.db.collections.transactionitems.drop(cb);
} else {
return cb();
}
},
function(cb) {
that.rpc.getInfo(function(err, res) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err);
that.block_total = res.result.blocks;
self.block_total = res.result.blocks;
return cb();
});
},
@ -187,7 +187,7 @@ function spec() {
function(cb) {
if (!opts.reverse) return cb();
that.rpc.getBlockCount(function(err, res) {
self.rpc.getBlockCount(function(err, res) {
if (err) return cb(err);
block_height = res.result;
return cb();
@ -196,7 +196,7 @@ function spec() {
function(cb) {
if (!opts.reverse) return cb();
that.rpc.getBlockHash(block_height, function(err, res) {
self.rpc.getBlockHash(block_height, function(err, res) {
if (err) return cb(err);
block_best = res.result;
@ -210,17 +210,20 @@ function spec() {
function sync() {
if (opts.reverse) {
start = block_best;
end = that.network.genesisBlock.hash.reverse().toString('hex');
end = self.genesis;
opts.prev = true;
}
else {
start = that.network.genesisBlock.hash.reverse().toString('hex');
start = self.genesis;
end = null;
opts.next = true;
}
that.syncBlocks(start, end, opts, function(err) {
p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));
self.getPrevNextBlock( start, end, opts , function(err) {
if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
setTimeout(function() {
p('Retrying in %d secs', retry_secs);
@ -228,7 +231,7 @@ function spec() {
}, retry_secs * 1000);
}
else
return next(err, that.block_count);
return next(err, self.block_count);
});
}
if (!err)
@ -238,13 +241,28 @@ function spec() {
});
};
// Reverse Imports (upto if we have genesis block?)
// upto if we have genesis block?
HistoricSync.prototype.smart_import = function(next) {
var that = this;
var opts = {
prev: 1,
};
that.import_history(opts, next);
var self = this;
Block.findOne({hash:self.genesis}, function(err, b){
if (err) return next(err);
if (!b) {
p('Could not find Genesis block. Running FULL SYNC');
}
else {
p('Genesis block found. Syncing upto know blocks.');
}
var opts = {
reverse: 1,
uptoexisting: b ? true: false,
};
return self.import_history(opts, next);
});
};

12
lib/Sync.js
View File

@ -19,6 +19,7 @@ function spec() {
var that = this;
Block.customCreate(block, function(err, block, inserted_txs){
if (err) return cb(err);
if (block && that.opts.broadcast_blocks) {
sockets.broadcast_block(block);
@ -56,17 +57,6 @@ function spec() {
};
Sync.prototype.syncBlocks = function(start, end, isForward, cb) {
var that = this;
console.log('Syncing Blocks, starting \n\tfrom: %s \n\tend: %s \n\tisForward:',
start, end, isForward);
return that.getPrevNextBlock( start, end,
isForward ? { next: 1 } : { prev: 1}, cb);
};
Sync.prototype.init = function(opts, cb) {
var that = this;

4
server.js
View File

@ -50,9 +50,7 @@ if (!config.disableHistoricSync) {
skip_db_connection: true,
networkName: config.network
}, function() {
hs.import_history({
reverse: 1,
}, function(){
hs.smart_import(function(){
console.log('historic_sync finished!');
});
});

43
util/sync.js
View File

@ -24,33 +24,40 @@ var historicSync = new HistoricSync({
networkName: program.network
});
/* TODO: Sure?
if (program.remove) {
// TODO: Sure?
}
*/
async.series([
function(cb) {
historicSync.init(program, cb);
},
function(cb) {
historicSync.import_history({
network: program.network,
destroy: program.destroy,
reverse: program.reverse,
uptoexisting: program.uptoexisting,
smart: program.smart,
}, function(err, count) {
if (err) {
console.log('CRITICAL ERROR: ', err);
}
else {
console.log('Finished. [%d blocks]', count);
}
cb();
});
if (program.smart) {
historicSync.smart_import(cb);
}
else {
historicSync.import_history({
destroy: program.destroy,
reverse: program.reverse,
uptoexisting: program.uptoexisting,
}, cb);
}
},
function(cb) {
historicSync.close();
cb();
}]);
return cb();
},
],
function(err, count) {
if (err) {
console.log('CRITICAL ERROR: ', err);
}
else {
console.log('Finished. [%d blocks synced]', count[1]);
}
return;
});