diff --git a/lib/database.js b/lib/database.js index 238d9f0..91eb0b3 100644 --- a/lib/database.js +++ b/lib/database.js @@ -115,7 +115,6 @@ function find_tx(txid, cb) { } function save_tx(txid, blockheight, cb) { - //var s_timer = new Date().getTime(); lib.get_rawtransaction(txid, function(tx){ if (tx != 'There was an error. Check your console.') { lib.prepare_vin(tx, function(vin) { @@ -148,10 +147,9 @@ function save_tx(txid, blockheight, cb) { }); newTx.save(function(err) { if (err) { - return cb(err); + return cb(err, false); } else { - //console.log('txid: '); - return cb(); + return cb(null, vout.length > 0); } }); }); @@ -160,7 +158,7 @@ function save_tx(txid, blockheight, cb) { }); }); } else { - return cb('tx not found: ' + txid); + return cb('tx not found: ' + txid, false); } }); } @@ -378,7 +376,7 @@ module.exports = { } else { lib.syncLoop(block.tx.length, function (loop) { var i = loop.iteration(); - save_tx(block.tx[i], block.height, function(err){ + save_tx(block.tx[i], block.height, function(err, tx_has_vout) { if (err) { loop.next(); } else { @@ -420,12 +418,14 @@ module.exports = { }, get_last_txs_ajax: function(start, length, min, cb) { - Tx.find({'total': {$gte: min}}).countDocuments(function(err, count){ - Tx.find({'total': {$gte: min}}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec(function(err, txs){ + // Lookup the coin stats to get the txes value which is used to determine the total # of records + Stats.findOne({coin:settings.coin}, function(err, stats) { + // Get last transactions where there is at least 1 vout + Tx.find({'total': {$gte: min}, 'vout': { $gte: { $size: 1 }}}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec(function(err, txs) { if (err) { return cb(err); } else { - return cb(txs, count); + return cb(txs, stats.txes); } }); }); @@ -750,30 +750,36 @@ module.exports = { // updates stats data for given coin; called by sync.js update_db: function(coin, cb) { lib.get_blockcount( function (count) { - if (!count){ + if (!count) { console.log('Unable to connect to explorer API'); return cb(false); } - lib.get_supply( function (supply){ + lib.get_supply( function (supply) { lib.get_connectioncount(function (connections) { - Stats.findOneAndUpdate({coin: coin}, { - $set: { - coin: coin, - count : count, - supply: supply, - connections: connections + Stats.findOne({coin: coin}, function(err, stats) { + if (stats) { + Stats.updateOne({coin: coin}, { + coin: coin, + count : count, + supply: supply, + connections: connections + }, function(err) { + if (err) { + console.log("Error during Stats Update: ", err); + } + return cb({ + coin: coin, + count : count, + supply: supply, + connections: connections, + last: (stats.last ? stats.last : 0), + txes: (stats.txes ? stats.txes : 0) + }); + }); + } else { + console.log("Error during Stats Update: ", (err ? err : 'cannot find stats collection')); + return cb(false); } - }, { - new: true - }, function(err, new_stats) { - if(err) { - console.log("Error during Stats Update:", err); - } - return cb({coin: coin, - count : count, - supply: supply, - connections: connections, - last: (new_stats.last ? new_stats.last : 0)}); }); }); }); @@ -781,7 +787,7 @@ module.exports = { }, // updates tx, address & richlist db's; called by sync.js - update_tx_db: function(coin, start, end, timeout, cb) { + update_tx_db: function(coin, start, end, txes, timeout, cb) { is_locked("db_index", function (exists) { if (exists) { console.log("db_index lock file exists..."); @@ -801,6 +807,7 @@ module.exports = { if (block_height % 5000 === 0) { Stats.updateOne({coin: coin}, { last: block_height - 1, + txes: txes, last_txs: '' //not used anymore left to clear out existing objects }, function() {}); } @@ -816,12 +823,14 @@ module.exports = { next_tx(); }, timeout); } else { - save_tx(txid, block_height, function(err){ + save_tx(txid, block_height, function(err, tx_has_vout) { if (err) { console.log(err); } else { console.log('%s: %s', block_height, txid); } + if (tx_has_vout) + txes++; setTimeout( function(){ tx = null; next_tx(); @@ -853,6 +862,7 @@ module.exports = { Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ Stats.updateOne({coin: coin}, { last: end, + txes: txes, last_txs: '' //not used anymore left to clear out existing objects }, function() { remove_lock("db_index", function(){ diff --git a/models/stats.js b/models/stats.js index 872030c..31cb77e 100644 --- a/models/stats.js +++ b/models/stats.js @@ -8,7 +8,7 @@ var StatsSchema = new Schema({ //difficulty: { type: Object, default: {} }, //hashrate: { type: String, default: 'N/A' }, supply: { type: Number, default: 0 }, - //last_txs: { type: Array, default: [] }, + txes: { type: Number, default: 0 }, connections: { type: Number, default: 0 }, last_price: { type: Number, default: 0 }, last_usd_price: { type: Number, default: 0 }, diff --git a/scripts/benchmark.js b/scripts/benchmark.js index 829c839..5d1c3ae 100644 --- a/scripts/benchmark.js +++ b/scripts/benchmark.js @@ -27,7 +27,7 @@ mongoose.connect(dbString, { useNewUrlParser: true, useCreateIndex: true, useUni Tx.deleteMany({}, function(err) { Address.deleteMany({}, function(err2) { var s_timer = new Date().getTime(); - db.update_tx_db(settings.coin, 1, COUNT, settings.update_timeout, function(){ + db.update_tx_db(settings.coin, 1, COUNT, 0, settings.update_timeout, function(){ var e_timer = new Date().getTime(); Tx.countDocuments({}, function(txerr, txcount){ Address.countDocuments({}, function(aerr, acount){ diff --git a/scripts/sync.js b/scripts/sync.js index a6f85eb..486891a 100644 --- a/scripts/sync.js +++ b/scripts/sync.js @@ -20,9 +20,10 @@ function usage() { console.log('market Market data: summaries, orderbooks, trade history & chartdata') console.log(''); console.log('mode: (required for index database only)'); - console.log('update Updates index from last sync to current block'); - console.log('check checks index for (and adds) any missing transactions/addresses'); - console.log('reindex Clears index then resyncs from genesis to current block'); + console.log('update Updates index from last sync to current block'); + console.log('check Checks index for (and adds) any missing transactions/addresses'); + console.log('reindex Clears index then resyncs from genesis to current block'); + console.log('reindex-txcount Rescan and flatten the tx count value for faster access'); console.log(''); console.log('notes:'); console.log('* \'current block\' is the latest created block when script is executed.'); @@ -51,6 +52,8 @@ if (process.argv[2] == 'index') { break; case 'reindex-rich': mode = 'reindex-rich'; + case 'reindex-txcount': + mode = 'reindex-txcount'; break; default: usage(); @@ -164,7 +167,7 @@ is_locked(function (exists) { }, function() { console.log('index cleared (reindex)'); }); - db.update_tx_db(settings.coin, 1, stats.count, settings.update_timeout, function(){ + db.update_tx_db(settings.coin, 1, stats.count, stats.txes, settings.update_timeout, function(){ db.update_richlist('received', function(){ db.update_richlist('balance', function(){ db.get_stats(settings.coin, function(nstats){ @@ -179,7 +182,7 @@ is_locked(function (exists) { }); }); } else if (mode == 'check') { - db.update_tx_db(settings.coin, 1, stats.count, settings.check_timeout, function(){ + db.update_tx_db(settings.coin, 1, stats.count, stats.txes, settings.check_timeout, function(){ db.get_stats(settings.coin, function(nstats){ console.log('check complete (block: %s)', nstats.last); exit(); @@ -194,7 +197,7 @@ is_locked(function (exists) { nLast = data.blockindex; } - db.update_tx_db(settings.coin, nLast, stats.count, settings.update_timeout, function(){ + db.update_tx_db(settings.coin, nLast, stats.count, stats.txes, settings.update_timeout, function(){ db.update_richlist('received', function(){ db.update_richlist('balance', function(){ db.get_stats(settings.coin, function(nstats){ @@ -207,7 +210,7 @@ is_locked(function (exists) { }); } else if (mode == 'reindex-rich') { console.log('update started'); - db.update_tx_db(settings.coin, stats.last, stats.count, settings.check_timeout, function(){ + db.update_tx_db(settings.coin, stats.last, stats.count, stats.txes, settings.check_timeout, function(){ console.log('update finished'); db.check_richlist(settings.coin, function(exists){ if (exists == true) { @@ -233,6 +236,18 @@ is_locked(function (exists) { }); }); }); + } else if (mode == 'reindex-txcount') { + console.log('calculating tx count.. please wait..'); + // Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout + Tx.find({'total': {$gte: 0}, 'vout': { $gte: { $size: 1 }}}).countDocuments(function(err, count) { + console.log('found tx count: ' + count.toString()); + Stats.updateOne({coin: settings.coin}, { + txes: count + }, function() { + console.log('tx count update complete'); + exit(); + }); + }); } }); }