Fixed multi-threaded sync + related improvements
-The block_parallel_tasks feature has been improved and fixed so that it is now safe to cancel (Ctrl+C) or kill (kill cmd not kill -9) the task and resume the sync later without missing transactions. The new block_parallel_tasks default is 8 threads which seems to be the sweet spot for any type of cpu -Numerous improvements to the benchamark script to utilize new benchmark settings, auto-add credentials to the benchmark database, reuse the same sync code as the regular block sync instead of using a copy of the code and more -Added a new cmd to run the benchmark script `npm run benchmark` -README updated to include the new benchmark script instrutions + include multi-threaded sync as a feature
This commit is contained in:
Joe Uhren
-141
@@ -65,54 +65,6 @@ function find_richlist(coin, cb) {
|
||||
});
|
||||
}
|
||||
|
||||
function update_address(hash, blockheight, txid, amount, type, cb) {
|
||||
var to_sent = false;
|
||||
var to_received = false;
|
||||
var addr_inc = {}
|
||||
|
||||
if (hash == 'coinbase')
|
||||
addr_inc.sent = amount;
|
||||
else {
|
||||
if (type == 'vin') {
|
||||
addr_inc.sent = amount;
|
||||
addr_inc.balance = -amount;
|
||||
} else {
|
||||
addr_inc.received = amount;
|
||||
addr_inc.balance = amount;
|
||||
}
|
||||
}
|
||||
|
||||
Address.findOneAndUpdate({a_id: hash}, {
|
||||
$inc: addr_inc
|
||||
}, {
|
||||
new: true,
|
||||
upsert: true
|
||||
}).then((address) => {
|
||||
if (hash != 'coinbase') {
|
||||
AddressTx.findOneAndUpdate({a_id: hash, txid: txid}, {
|
||||
$inc: {
|
||||
amount: addr_inc.balance
|
||||
},
|
||||
$set: {
|
||||
a_id: hash,
|
||||
blockindex: blockheight,
|
||||
txid: txid
|
||||
}
|
||||
}, {
|
||||
new: true,
|
||||
upsert: true
|
||||
}).then((addresstx) => {
|
||||
return cb();
|
||||
}).catch((err) => {
|
||||
return cb(err);
|
||||
});
|
||||
} else
|
||||
return cb();
|
||||
}).catch((err) => {
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
|
||||
function find_tx(txid, cb) {
|
||||
Tx.findOne({txid: txid}).then((tx) => {
|
||||
if (tx)
|
||||
@@ -202,15 +154,6 @@ function check_remove_db_field(model_obj, field_name, cb) {
|
||||
});
|
||||
}
|
||||
|
||||
function hex_to_ascii(hex) {
|
||||
var str = '';
|
||||
|
||||
for (var i = 0; i < hex.length; i += 2)
|
||||
str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
|
||||
|
||||
return str;
|
||||
}
|
||||
|
||||
function init_markets(cb) {
|
||||
let installed_markets = [];
|
||||
|
||||
@@ -2046,90 +1989,6 @@ module.exports = {
|
||||
}
|
||||
},
|
||||
|
||||
save_tx: function(txid, blockheight, block, cb) {
|
||||
lib.get_rawtransaction(txid, function(tx) {
|
||||
if (tx && tx != `${settings.localization.ex_error}: ${settings.localization.check_console}`) {
|
||||
lib.prepare_vin(tx, function(vin, tx_type_vin) {
|
||||
lib.prepare_vout(tx.vout, txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) {
|
||||
lib.syncLoop(nvin.length, function (loop) {
|
||||
var i = loop.iteration();
|
||||
|
||||
// check if address is inside an array
|
||||
if (Array.isArray(nvin[i].addresses)) {
|
||||
// extract the address
|
||||
nvin[i].addresses = nvin[i].addresses[0];
|
||||
}
|
||||
|
||||
update_address(nvin[i].addresses, blockheight, txid, nvin[i].amount, 'vin', function() {
|
||||
loop.next();
|
||||
});
|
||||
}, function() {
|
||||
lib.syncLoop(vout.length, function (subloop) {
|
||||
var t = subloop.iteration();
|
||||
|
||||
// check if address is inside an array
|
||||
if (Array.isArray(vout[t].addresses)) {
|
||||
// extract the address
|
||||
vout[t].addresses = vout[t].addresses[0];
|
||||
}
|
||||
|
||||
if (vout[t].addresses) {
|
||||
update_address(vout[t].addresses, blockheight, txid, vout[t].amount, 'vout', function() {
|
||||
subloop.next();
|
||||
});
|
||||
} else
|
||||
subloop.next();
|
||||
}, function() {
|
||||
lib.calculate_total(vout, function(total) {
|
||||
var op_return = null;
|
||||
var algo = null;
|
||||
|
||||
// check if the op_return value should be decoded and saved
|
||||
if (settings.transaction_page.show_op_return) {
|
||||
// loop through vout to find the op_return value
|
||||
tx.vout.forEach(function (vout_data) {
|
||||
// check if the op_return value exists
|
||||
if (vout_data.scriptPubKey != null && vout_data.scriptPubKey.asm != null && vout_data.scriptPubKey.asm.indexOf('OP_RETURN') > -1) {
|
||||
// decode the op_return value
|
||||
op_return = hex_to_ascii(vout_data.scriptPubKey.asm.replace('OP_RETURN', '').trim());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// check if the algo value should be saved
|
||||
if (settings.block_page.multi_algorithm.show_algo) {
|
||||
// get the algo value
|
||||
algo = block[settings.block_page.multi_algorithm.key_name];
|
||||
}
|
||||
|
||||
var newTx = new Tx({
|
||||
txid: tx.txid,
|
||||
vin: (vin == null || vin.length == 0 ? [] : nvin),
|
||||
vout: vout,
|
||||
total: total.toFixed(8),
|
||||
timestamp: tx.time,
|
||||
blockhash: tx.blockhash,
|
||||
blockindex: blockheight,
|
||||
tx_type: (tx_type_vout == null ? tx_type_vin : tx_type_vout),
|
||||
op_return: op_return,
|
||||
algo: algo
|
||||
});
|
||||
|
||||
newTx.save().then(() => {
|
||||
return cb(null, vout.length > 0);
|
||||
}).catch((err) => {
|
||||
return cb(err, false);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
} else
|
||||
return cb('tx not found: ' + txid, false);
|
||||
});
|
||||
},
|
||||
|
||||
// get the list of orphans from local collection
|
||||
get_orphans: function(start, length, cb) {
|
||||
// get the count of orphaned blocks
|
||||
|
||||
Reference in New Issue
Block a user