davide/purple-explorer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Flatten tx count value for faster lookups with large blockchains

Browse Source
This commit is contained in:
joeuhren
2020-12-03 14:16:34 -07:00
parent 64f7072680
commit 6d74c3e6de
4 changed files with 64 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/database.js
+40 -30
View File
@@ -115,7 +115,6 @@ function find_tx(txid, cb) {
}
function save_tx(txid, blockheight, cb) {
//var s_timer = new Date().getTime();
lib.get_rawtransaction(txid, function(tx){
if (tx != 'There was an error. Check your console.') {
lib.prepare_vin(tx, function(vin) {
@@ -148,10 +147,9 @@ function save_tx(txid, blockheight, cb) {
});
newTx.save(function(err) {
if (err) {
return cb(err);
return cb(err, false);
} else {
//console.log('txid: ');
return cb();
return cb(null, vout.length > 0);
}
});
});
@@ -160,7 +158,7 @@ function save_tx(txid, blockheight, cb) {
});
});
} else {
return cb('tx not found: ' + txid);
return cb('tx not found: ' + txid, false);
}
});
}
@@ -378,7 +376,7 @@ module.exports = {
} else {
lib.syncLoop(block.tx.length, function (loop) {
var i = loop.iteration();
save_tx(block.tx[i], block.height, function(err){
save_tx(block.tx[i], block.height, function(err, tx_has_vout) {
if (err) {
loop.next();
} else {
@@ -420,12 +418,14 @@ module.exports = {
},
get_last_txs_ajax: function(start, length, min, cb) {
Tx.find({'total': {$gte: min}}).countDocuments(function(err, count){
Tx.find({'total': {$gte: min}}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec(function(err, txs){
// Lookup the coin stats to get the txes value which is used to determine the total # of records
Stats.findOne({coin:settings.coin}, function(err, stats) {
// Get last transactions where there is at least 1 vout
Tx.find({'total': {$gte: min}, 'vout': { $gte: { $size: 1 }}}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec(function(err, txs) {
if (err) {
return cb(err);
} else {
return cb(txs, count);
return cb(txs, stats.txes);
}
});
});
@@ -750,30 +750,36 @@ module.exports = {
// updates stats data for given coin; called by sync.js
update_db: function(coin, cb) {
lib.get_blockcount( function (count) {
if (!count){
if (!count) {
console.log('Unable to connect to explorer API');
return cb(false);
}
lib.get_supply( function (supply){
lib.get_supply( function (supply) {
lib.get_connectioncount(function (connections) {
Stats.findOneAndUpdate({coin: coin}, {
$set: {
coin: coin,
count : count,
supply: supply,
connections: connections
Stats.findOne({coin: coin}, function(err, stats) {
if (stats) {
Stats.updateOne({coin: coin}, {
coin: coin,
count : count,
supply: supply,
connections: connections
}, function(err) {
if (err) {
console.log("Error during Stats Update: ", err);
}
return cb({
coin: coin,
count : count,
supply: supply,
connections: connections,
last: (stats.last ? stats.last : 0),
txes: (stats.txes ? stats.txes : 0)
});
});
} else {
console.log("Error during Stats Update: ", (err ? err : 'cannot find stats collection'));
return cb(false);
}
}, {
new: true
}, function(err, new_stats) {
if(err) {
console.log("Error during Stats Update:", err);
}
return cb({coin: coin,
count : count,
supply: supply,
connections: connections,
last: (new_stats.last ? new_stats.last : 0)});
});
});
});
@@ -781,7 +787,7 @@ module.exports = {
},
// updates tx, address & richlist db's; called by sync.js
update_tx_db: function(coin, start, end, timeout, cb) {
update_tx_db: function(coin, start, end, txes, timeout, cb) {
is_locked("db_index", function (exists) {
if (exists) {
console.log("db_index lock file exists...");
@@ -801,6 +807,7 @@ module.exports = {
if (block_height % 5000 === 0) {
Stats.updateOne({coin: coin}, {
last: block_height - 1,
txes: txes,
last_txs: '' //not used anymore left to clear out existing objects
}, function() {});
}
@@ -816,12 +823,14 @@ module.exports = {
next_tx();
}, timeout);
} else {
save_tx(txid, block_height, function(err){
save_tx(txid, block_height, function(err, tx_has_vout) {
if (err) {
console.log(err);
} else {
console.log('%s: %s', block_height, txid);
}
if (tx_has_vout)
txes++;
setTimeout( function(){
tx = null;
next_tx();
@@ -853,6 +862,7 @@ module.exports = {
Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){
Stats.updateOne({coin: coin}, {
last: end,
txes: txes,
last_txs: '' //not used anymore left to clear out existing objects
}, function() {
remove_lock("db_index", function(){
models/stats.js
+1 -1
View File
@@ -8,7 +8,7 @@ var StatsSchema = new Schema({
//difficulty: { type: Object, default: {} },
//hashrate: { type: String, default: 'N/A' },
supply: { type: Number, default: 0 },
//last_txs: { type: Array, default: [] },
txes: { type: Number, default: 0 },
connections: { type: Number, default: 0 },
last_price: { type: Number, default: 0 },
last_usd_price: { type: Number, default: 0 },
scripts/benchmark.js
+1 -1
View File
@@ -27,7 +27,7 @@ mongoose.connect(dbString, { useNewUrlParser: true, useCreateIndex: true, useUni
Tx.deleteMany({}, function(err) {
Address.deleteMany({}, function(err2) {
var s_timer = new Date().getTime();
db.update_tx_db(settings.coin, 1, COUNT, settings.update_timeout, function(){
db.update_tx_db(settings.coin, 1, COUNT, 0, settings.update_timeout, function(){
var e_timer = new Date().getTime();
Tx.countDocuments({}, function(txerr, txcount){
Address.countDocuments({}, function(aerr, acount){
scripts/sync.js
+22 -7
View File
@@ -20,9 +20,10 @@ function usage() {
console.log('market Market data: summaries, orderbooks, trade history & chartdata')
console.log('');
console.log('mode: (required for index database only)');
console.log('update Updates index from last sync to current block');
console.log('check checks index for (and adds) any missing transactions/addresses');
console.log('reindex Clears index then resyncs from genesis to current block');
console.log('update Updates index from last sync to current block');
console.log('check Checks index for (and adds) any missing transactions/addresses');
console.log('reindex Clears index then resyncs from genesis to current block');
console.log('reindex-txcount Rescan and flatten the tx count value for faster access');
console.log('');
console.log('notes:');
console.log('* \'current block\' is the latest created block when script is executed.');
@@ -51,6 +52,8 @@ if (process.argv[2] == 'index') {
break;
case 'reindex-rich':
mode = 'reindex-rich';
case 'reindex-txcount':
mode = 'reindex-txcount';
break;
default:
usage();
@@ -164,7 +167,7 @@ is_locked(function (exists) {
}, function() {
console.log('index cleared (reindex)');
});
db.update_tx_db(settings.coin, 1, stats.count, settings.update_timeout, function(){
db.update_tx_db(settings.coin, 1, stats.count, stats.txes, settings.update_timeout, function(){
db.update_richlist('received', function(){
db.update_richlist('balance', function(){
db.get_stats(settings.coin, function(nstats){
@@ -179,7 +182,7 @@ is_locked(function (exists) {
});
});
} else if (mode == 'check') {
db.update_tx_db(settings.coin, 1, stats.count, settings.check_timeout, function(){
db.update_tx_db(settings.coin, 1, stats.count, stats.txes, settings.check_timeout, function(){
db.get_stats(settings.coin, function(nstats){
console.log('check complete (block: %s)', nstats.last);
exit();
@@ -194,7 +197,7 @@ is_locked(function (exists) {
nLast = data.blockindex;
}
db.update_tx_db(settings.coin, nLast, stats.count, settings.update_timeout, function(){
db.update_tx_db(settings.coin, nLast, stats.count, stats.txes, settings.update_timeout, function(){
db.update_richlist('received', function(){
db.update_richlist('balance', function(){
db.get_stats(settings.coin, function(nstats){
@@ -207,7 +210,7 @@ is_locked(function (exists) {
});
} else if (mode == 'reindex-rich') {
console.log('update started');
db.update_tx_db(settings.coin, stats.last, stats.count, settings.check_timeout, function(){
db.update_tx_db(settings.coin, stats.last, stats.count, stats.txes, settings.check_timeout, function(){
console.log('update finished');
db.check_richlist(settings.coin, function(exists){
if (exists == true) {
@@ -233,6 +236,18 @@ is_locked(function (exists) {
});
});
});
} else if (mode == 'reindex-txcount') {
console.log('calculating tx count.. please wait..');
// Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout
Tx.find({'total': {$gte: 0}, 'vout': { $gte: { $size: 1 }}}).countDocuments(function(err, count) {
console.log('found tx count: ' + count.toString());
Stats.updateOne({coin: settings.coin}, {
txes: count
}, function() {
console.log('tx count update complete');
exit();
});
});
}
});
}