davide/purple-explorer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved block sync speed

Browse Source 
-A number of functions have been rewritten to be more optimized and faster: calculate_total, is_unique, convert_to_satoshi, get_input_addresses, processVoutAddresses, prepare_vout, prepare_vin
-Txes are now written to database via bulk writes which helps improve the sync speed and also controls memory usage with batching to write data once a certain threshold is reached
-update_address function changed to update_addresses since it now bulk writes the addresses in batches to improve sync speed and also controls memory usage with batching to write data once a certain threshold is reached
-The syncLoop function has been completely removed from the project and replaced with async library loops or even normal "for" loops in some cases which greatly improves sync speeds over large batches of data
-Fixed an issue with the flattened count of txes that is saved to the coinstats collection which could save incorrectly when using more than 1 thread
-Fixed an issue with the block sync which caused an unwanted delay when syncing less blocks than the amount of threads used to sync the data
-Fixed an issue with vout data processing that could sometimes populate data out of order
-Added a new sync.batch_size setting used to determine how many records (txes, addresses, addresstxes) should be saved in a single database transaction
-Added a new wait_for_bulk_database_save setting used to increase the block sync speed at the cost of not returning any error msgs for data that failed to save
-get_input_addresses function no longer returns in the exports section of the explorer.js file since it is only referenced in that file
-Updated explorerspec tests to use the newest function changes for any tests that needed to be updated

Special thanks to Karzo from Pepecoin for help with the bulkwrite code changes!
This commit is contained in:
Joe Uhren
2025-02-02 19:10:17 -07:00
parent 0b0ef817f1
commit 3a2f679201
10 changed files with 966 additions and 867 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/block_sync.js
+348 -138
View File
@@ -8,17 +8,17 @@ const async = require('async');
let stopSync = false;
let stackSizeErrorId = null;
function check_delete_tx(tx, block_height, tx_count, timeout, cb) {
function check_delete_tx(tx, block_height, timeout, cb) {
// check if the tx object exists and does not match the current block height
if (tx && tx.blockindex != block_height) {
// the transaction exists but does not match the correct block height, therefore it should be deleted
module.exports.delete_and_cleanup_tx(tx.txid, tx.blockindex, tx_count, timeout, function(updated_tx_count) {
module.exports.delete_and_cleanup_tx(tx.txid, tx.blockindex, timeout, function(updated_tx_count) {
// finished removing the transaction
return cb(updated_tx_count, true);
});
} else {
// tx dosn't exist or block heights match so nothing to do
return cb(tx_count, false);
return cb(0, false);
}
}
@@ -71,50 +71,112 @@ function hex_to_ascii(hex) {
return str;
}
function update_address(hash, blockheight, txid, amount, type, cb) {
let addr_inc = {}
function update_addresses(addresses, blockheight, txid, cb) {
const bulkAddresses = addresses.map((address) => {
return {
updateOne: {
filter: { a_id: address.hash },
update: {
$setOnInsert: { a_id: address.hash },
$inc: { sent: address.sent, balance: address.balance, received: address.received },
},
upsert: true
}
};
});
if (hash == 'coinbase')
addr_inc.sent = amount;
else {
if (type == 'vin') {
addr_inc.sent = amount;
addr_inc.balance = -amount;
} else {
addr_inc.received = amount;
addr_inc.balance = amount;
const bulkAddressTxes = addresses
.filter((address) => address.hash !== 'coinbase')
.map((address) => {
return {
updateOne: {
filter: { a_id: address.hash, txid: txid },
update: {
$setOnInsert: { a_id: address.hash, blockindex: blockheight, txid: txid },
$inc: { amount: (address.type == 'vin' ? -address.amount : address.amount) }
},
upsert: true
}
};
});
try {
let completed = 0;
let errorCalled = false;
// callback to run when one of the 2 table writes is complete
function onDone(err) {
// check if the previous process already returned an error
if (errorCalled) {
// stop if there was already an error
return;
} else {
// check if there was an error
if (err) {
// ensure the next data set knows there was already an error
errorCalled = true;
// return the error
return cb(err);
}
// increment the completed counter
completed += 1;
// check if both data sets completed
if (completed === 2) {
// finished updating address data
return cb();
}
}
}
// start processing both sets of data in parallel
processInBatches(Address, bulkAddresses, onDone);
processInBatches(AddressTx, bulkAddressTxes, onDone);
} catch (err) {
return cb(err);
}
}
function processInBatches(collection, data, cb) {
const batch_size = settings.sync.batch_size;
let index = 0;
function processNextBatch() {
// check if all records were saved
if (index >= data.length) {
// all records were saved
return cb();
}
// get the next batch of data
const batch = data.slice(index, index + batch_size);
// increment the index by the batch size
index += batch_size;
try {
// asynchronously write data to the collection database
collection.bulkWrite(batch, { ordered: false, writeConcern: { w: (settings.sync.wait_for_bulk_database_save ? 1 : 0) } }).then((result) => {
// process the next batch of records
processNextBatch();
}).catch((err) => {
console.log(err);
// process the next batch of records
processNextBatch();
});
} catch(err) {
console.log(err);
// process the next batch of records
processNextBatch();
}
}
Address.findOneAndUpdate({a_id: hash}, {
$inc: addr_inc
}, {
new: true,
upsert: true
}).then((address) => {
if (hash != 'coinbase') {
AddressTx.findOneAndUpdate({a_id: hash, txid: txid}, {
$inc: {
amount: addr_inc.balance
},
$set: {
a_id: hash,
blockindex: blockheight,
txid: txid
}
}, {
new: true,
upsert: true
}).then((addresstx) => {
return cb();
}).catch((err) => {
return cb(err);
});
} else
return cb();
}).catch((err) => {
return cb(err);
});
// start processing records
processNextBatch();
}
function finalize_update_tx_db(coin, check_only, end, txes, cb) {
@@ -143,49 +205,103 @@ function finalize_update_tx_db(coin, check_only, end, txes, cb) {
module.exports = {
save_tx: function(txid, blockheight, block, cb) {
lib.get_rawtransaction(txid, function(tx) {
if (tx && tx != `${settings.localization.ex_error}: ${settings.localization.check_console}`) {
lib.prepare_vin(tx, function(vin, tx_type_vin) {
lib.prepare_vout(tx.vout, txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) {
// check if vout is null which indicates an error
if (vout != null) {
lib.syncLoop(nvin.length, function (loop) {
const i = loop.iteration();
// check if address is inside an array
if (Array.isArray(nvin[i].addresses)) {
// extract the address
nvin[i].addresses = nvin[i].addresses[0];
}
update_address(nvin[i].addresses, blockheight, txid, nvin[i].amount, 'vin', function() {
loop.next();
});
}, function() {
lib.syncLoop(vout.length, function (subloop) {
const t = subloop.iteration();
try {
lib.get_rawtransaction(txid, function(tx) {
if (tx && tx != `${settings.localization.ex_error}: ${settings.localization.check_console}`) {
lib.prepare_vin(tx, function(vin, tx_type_vin) {
lib.prepare_vout(tx.vout, txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) {
// check if vout is null which indicates an error
if (vout != null) {
let addressBatch = [];
// add all vin addresses to the batch array
nvin.forEach(function(input) {
// check if address is inside an array
if (Array.isArray(vout[t].addresses)) {
if (Array.isArray(input.addresses)) {
// extract the address
vout[t].addresses = vout[t].addresses[0];
input.addresses = input.addresses[0];
}
if (vout[t].addresses) {
update_address(vout[t].addresses, blockheight, txid, vout[t].amount, 'vout', function() {
subloop.next();
});
} else
subloop.next();
}, function() {
lib.calculate_total(vout, function(total) {
var op_return = null;
var algo = null;
// check if a vin address exists
if (input.addresses) {
const index = lib.is_unique(addressBatch, input.addresses, 'hash');
let sent = 0;
let balance = 0;
if (input.addresses == 'coinbase')
sent = input.amount;
else {
sent = input.amount;
balance = -input.amount;
}
// check if the address already exists in the array
if (index == -1) {
// unique address
addressBatch.push({
hash: input.addresses,
amount: input.amount,
type: 'vin',
sent: sent,
balance: balance,
received: 0
});
} else {
// address already exists
addressBatch[index].amount += input.amount;
addressBatch[index].sent += sent;
addressBatch[index].balance += balance;
}
}
});
// add all vout addresses to the batch array
vout.forEach(function(output) {
// check if address is inside an array
if (Array.isArray(output.addresses)) {
// extract the address
output.addresses = output.addresses[0];
}
// check if a vout address exists
if (output.addresses) {
const index = lib.is_unique(addressBatch, output.addresses, 'hash');
const balance = output.amount;
const received = output.amount;
// check if the address already exists in the array
if (index == -1) {
// unique address
addressBatch.push({
hash: output.addresses,
amount: output.amount,
type: 'vout',
sent: 0,
balance: balance,
received: received
});
} else {
// address already exists
addressBatch[index].amount -= output.amount;
addressBatch[index].balance += balance;
addressBatch[index].received += received;
}
}
});
// save the addresses to the database
update_addresses(addressBatch, blockheight, txid, function(err) {
if (err)
return cb(err, false, null);
else {
const total = lib.calculate_total(vout);
let op_return = null;
let algo = null;
// check if the op_return value should be decoded and saved
if (settings.transaction_page.show_op_return) {
// loop through vout to find the op_return value
tx.vout.forEach(function (vout_data) {
tx.vout.forEach(function(vout_data) {
// check if the op_return value exists
if (vout_data.scriptPubKey != null && vout_data.scriptPubKey.asm != null && vout_data.scriptPubKey.asm.indexOf('OP_RETURN') > -1) {
// decode the op_return value
@@ -200,7 +316,8 @@ module.exports = {
algo = block[settings.block_page.multi_algorithm.key_name];
}
const newTx = new Tx({
// return the transaction data
return cb(null, vout.length > 0, new Tx({
txid: tx.txid,
vin: (vin == null || vin.length == 0 ? [] : nvin),
vout: vout,
@@ -211,30 +328,27 @@ module.exports = {
tx_type: (tx_type_vout == null ? tx_type_vin : tx_type_vout),
op_return: op_return,
algo: algo
});
newTx.save().then(() => {
return cb(null, vout.length > 0);
}).catch((err) => {
return cb(err, false);
});
});
}));
}
});
});
} else {
// create a custom error that will be specifically checked for later (NOTE: tx_type_vout contains the error code in this special case)
const customError = new Error(tx_type_vout);
} else {
// create a custom error that will be specifically checked for later
// NOTE: tx_type_vout contains the error code in this special case
const customError = new Error(tx_type_vout);
customError.code = tx_type_vout;
customError.code = tx_type_vout;
// return the custom error
return cb(customError, false);
}
// return the custom error
return cb(customError, false, null);
}
});
});
});
} else
return cb('tx not found: ' + txid, false);
});
} else
return cb('tx not found: ' + txid, false, null);
});
} catch(err) {
return cb(`Error querying database for txid ${txid}: ${err}`, false, null);
}
},
// updates tx & address balances
@@ -329,6 +443,84 @@ module.exports = {
lib.get_block(blockhash, function(block) {
if (block) {
let tx_counter = 0;
let txBatch = [];
// create a queue for batching txes for this block
const txBatchQueue = async.queue(function(tx, done) {
// add the tx to an array
txBatch.push(tx);
// check if the batch of txes should be saved
if (txBatch.length >= settings.sync.batch_size) {
// save the current batch of txes to the database now
flushTxBatch(done);
} else {
// continue without saving txes
done();
}
}, 1);
// add a function used to bulkWrite txes to the database
function flushTxBatch(txBatchCallback) {
// copy current batch of txes to a local variable
const localTxBatch = txBatch;
// clear the global array of batched txes
txBatch = [];
// check if there are actually any txes to save
if (!localTxBatch || localTxBatch.length === 0) {
// no txes to save
return txBatchCallback();
} else {
// get the transaction batch ready to bulk update
const bulkTxes = localTxBatch.map(tx => {
// convert tx to plain JS object
const plainTx = tx.toObject();
// remove the _id field to prevent issues with some blockchains that can reuse non-standard txids
delete plainTx._id;
return {
updateOne: {
filter: { txid: plainTx.txid },
update: [
{
$replaceWith: {
$cond: {
if: { $gt: [ plainTx.blockindex, { $ifNull: ["$blockindex", -1] } ] },
then: {
$mergeObjects: [
// if a doc exists, keep that _id
{ _id: "$_id" },
// overwrite everything else with plainTx
plainTx
]
},
else: "$$ROOT"
}
}
}
],
upsert: true
}
};
});
try {
// write the transactions to the database
Tx.bulkWrite(bulkTxes, { ordered: false, writeConcern: { w: (settings.sync.wait_for_bulk_database_save ? 1 : 0) } }).then(() => {
return txBatchCallback();
}).catch((err) => {
console.log(err);
return txBatchCallback();
});
} catch(err) {
console.log(err);
return txBatchCallback();
}
}
}
// loop through all txes in this block
async.eachLimit(block.tx, parallel_tasks, function(txid, next_tx) {
@@ -350,42 +542,54 @@ module.exports = {
}, timeout);
} else {
// check if the transaction exists but doesn't match the current block height
check_delete_tx(tx, block_height, txes, timeout, function(updated_txes, tx_deleted) {
check_delete_tx(tx, block_height, timeout, function(updated_txes, tx_deleted) {
// update the running tx count
txes = updated_txes;
txes += updated_txes;
// check if this tx should be added to the local database
if (tx_deleted || !tx) {
// save the transaction to local database
module.exports.save_tx(txid, block_height, block, function(err, tx_has_vout) {
module.exports.save_tx(txid, block_height, block, function(err, tx_has_vout, newTx) {
if (err) {
// check the error code
if (err.code == 'StackSizeError') {
// ensure the process halts after stopping all sync threads
stackSizeErrorId = txid;
} else if (err.code === 11000) {
// output a nicer error msg for the 11000 error code "duplicate key error collection" which can happen in some blockchains with non-standard txids being reused
console.log(`${settings.localization.ex_warning}: ${block_height}: ${txid} already exists`);
} else
console.log(err);
}
else
setTimeout(function() {
tx = null;
tx_counter--;
// check if the script is stopping
if ((stopSync && check_only != 2) || stackSizeErrorId) {
// stop the loop
next_tx({});
} else
next_tx();
}, timeout);
} else {
console.log('%s: %s', block_height, txid);
if (tx_has_vout)
txes++;
if (tx_has_vout)
txes++;
setTimeout(function() {
tx = null;
tx_counter--;
// add the tx to a queue
txBatchQueue.push(newTx, function(queue_err) {
setTimeout(function() {
tx = null;
tx_counter--;
// check if the script is stopping
if ((stopSync && check_only != 2) || stackSizeErrorId) {
// stop the loop
next_tx({});
} else
next_tx();
}, timeout);
// check if the script is stopping
if ((stopSync && check_only != 2) || stackSizeErrorId) {
// stop the loop
next_tx({});
} else
next_tx();
}, timeout);
});
}
});
} else {
// skip adding the current tx
@@ -437,22 +641,28 @@ module.exports = {
blockhash = null;
block = null;
// reset the slot in the block array back to 0
block_numbers[slotIndex] = 0;
// save the remaining txes for this block
flushTxBatch(function(batch_err) {
if (batch_err)
console.error(batch_err);
// check if the script is stopping
if ((stopSync && check_only != 2) || stackSizeErrorId) {
// stop the loop
finished_tasks++;
next_block({});
} else {
// check if the last block is finished or in process and increment the finished counter
if (processed_last_block)
// reset the slot in the block array back to 0
block_numbers[slotIndex] = 0;
// check if the script is stopping
if ((stopSync && check_only != 2) || stackSizeErrorId) {
// stop the loop
finished_tasks++;
next_block({});
} else {
// check if the last block is finished or in process and increment the finished counter
if (processed_last_block)
finished_tasks++;
// proceed to next block
next_block();
}
// proceed to next block
next_block();
}
});
}
}, timeout);
});
@@ -516,7 +726,7 @@ module.exports = {
// check if all threads have properly finished or else the retry limit has been reached
// NOTE: the retry limit should never need to be used but is put in place to prevent an
// infinite loop just in case something goes very wrong
if (finished_tasks === parallel_tasks || retryAttempts >= retryLimit) {
if (finished_tasks === ((end - start + 1) >= parallel_tasks ? parallel_tasks : (end - start + 1)) || retryAttempts >= retryLimit) {
// stop waiting for all threads to finish
clearInterval(handle);
@@ -539,7 +749,9 @@ module.exports = {
});
},
delete_and_cleanup_tx: function(txid, block_height, tx_count, timeout, cb) {
delete_and_cleanup_tx: function(txid, block_height, timeout, cb) {
let tx_count = 0;
// lookup all address tx records associated with the current tx
AddressTx.find({txid: txid}).exec().then((address_txes) => {
if (address_txes.length == 0) {
@@ -704,14 +916,12 @@ module.exports = {
}
// loop through the address txes
lib.syncLoop(addressTxArray.length, function(address_loop) {
var a = address_loop.iteration();
async.eachSeries(addressTxArray, function(addressTx, address_loop) {
// fix the balance, sent and received data for the current address
fix_address_data(addressTxArray[a], function() {
fix_address_data(addressTx, function() {
setTimeout(function() {
// move to the next address record
address_loop.next();
address_loop();
}, timeout);
});
}, function() {
lib/database.js
+92 -110
View File
@@ -1,19 +1,20 @@
var mongoose = require('mongoose'),
Stats = require('../models/stats'),
Markets = require('../models/markets'),
Masternode = require('../models/masternode'),
Address = require('../models/address'),
AddressTx = require('../models/addresstx'),
Tx = require('../models/tx'),
Orphans = require('../models/orphans'),
Richlist = require('../models/richlist'),
Peers = require('../models/peers'),
Heavy = require('../models/heavy'),
NetworkHistory = require('../models/networkhistory'),
ClaimAddress = require('../models/claimaddress'),
lib = require('./explorer'),
settings = require('./settings'),
fs = require('fs');
const mongoose = require('mongoose');
const Stats = require('../models/stats');
const Markets = require('../models/markets');
const Masternode = require('../models/masternode');
const Address = require('../models/address');
const AddressTx = require('../models/addresstx');
const Tx = require('../models/tx');
const Orphans = require('../models/orphans');
const Richlist = require('../models/richlist');
const Peers = require('../models/peers');
const Heavy = require('../models/heavy');
const NetworkHistory = require('../models/networkhistory');
const ClaimAddress = require('../models/claimaddress');
const lib = require('./explorer');
const settings = require('./settings');
const fs = require('fs');
const async = require('async');
function find_address(hash, caseSensitive, cb) {
if (caseSensitive) {
@@ -250,24 +251,22 @@ function remove_inactive_markets(installed_markets, cb) {
// check if the database has any markets installed
if (db_markets != null && db_markets.length > 0) {
// loop through the list of markets in the database
lib.syncLoop(db_markets.length, function(market_loop) {
let m = market_loop.iteration();
async.eachSeries(db_markets, function(current_market, market_loop) {
// check if this market is installed
if (installed_markets.findIndex(x => x.market.toUpperCase() == db_markets[m].market.toUpperCase() && x.coin_symbol.toUpperCase() == db_markets[m].coin_symbol.toUpperCase() && x.pair_symbol.toUpperCase() == db_markets[m].pair_symbol.toUpperCase()) == -1) {
if (installed_markets.findIndex(x => x.market.toUpperCase() == current_market.market.toUpperCase() && x.coin_symbol.toUpperCase() == current_market.coin_symbol.toUpperCase() && x.pair_symbol.toUpperCase() == current_market.pair_symbol.toUpperCase()) == -1) {
// remove this market from the database because it is not installed or active
Markets.deleteOne({market: db_markets[m].market, coin_symbol: db_markets[m].coin_symbol, pair_symbol: db_markets[m].pair_symbol}).then(() => {
Markets.deleteOne({market: current_market.market, coin_symbol: current_market.coin_symbol, pair_symbol: current_market.pair_symbol}).then(() => {
// move to the next market record
market_loop.next();
market_loop();
}).catch((err) => {
console.log(err);
// move to the next market record
market_loop.next();
market_loop();
});
} else {
// move to the next market record
market_loop.next();
market_loop();
}
}, function() {
// finished removing inactive markets
@@ -299,35 +298,30 @@ function init_heavy(cb) {
function init_claimaddress(coin, cb) {
// first, get the stats data
Stats.findOne({coin: coin}).then((stats) => {
var newer_claim_address = false;
let newer_claim_address = false;
// check if stats were found
if (stats) {
// check if the claim address data was already moved to the new collection
if (stats.newer_claim_address != null && stats.newer_claim_address == true)
newer_claim_address = true;
}
// check if stats were found and the claim address data was already moved to the new collection
if (stats && stats.newer_claim_address != null && stats.newer_claim_address == true)
newer_claim_address = true;
// check if the claim address data should be moved to a new collection
if (!newer_claim_address) {
// find all addresses with a custom claim address name
Address.find({$and: [{"name": {$ne: ""}}, {"name": {$ne: null}}]}).exec().then((addresses) => {
Address.find({$and: [{'name': {$ne: ''}}, {'name': {$ne: null}}]}).exec().then((addresses) => {
// loop through the claimed addresses
lib.syncLoop(addresses.length, function(address_loop) {
var a = address_loop.iteration();
async.eachSeries(addresses, function(current_address, address_loop) {
// create a new claimaddress record
var claim_address = new ClaimAddress({
a_id: addresses[a].a_id,
claim_name: addresses[a].name
const claim_address = new ClaimAddress({
a_id: current_address.a_id,
claim_name: current_address.name
});
// add new claim address to collection
claim_address.save().then(() => {
address_loop.next();
address_loop();
}).catch((err) => {
console.log(err);
address_loop.next();
address_loop();
});
}, function() {
// finished moving all claimed address data to the new collection
@@ -864,17 +858,17 @@ module.exports = {
},
get_txs: function(block, cb) {
var txs = [];
let txs = [];
lib.syncLoop(block.tx.length, function (loop) {
var i = loop.iteration();
find_tx(block.tx[i], function(tx) {
async.eachSeries(block.tx, function(current_tx, loop) {
find_tx(current_tx[i], function(tx) {
if (tx) {
// add tx to the list
txs.push(tx);
loop.next();
} else
loop.next();
}
// move to next tx
loop();
});
}, function() {
return cb(txs);
@@ -951,14 +945,10 @@ module.exports = {
},
get_address_txs_ajax: function(hash, start, length, cb) {
var totalCount = 0;
AddressTx.find({a_id: hash}).countDocuments().then((count) => {
totalCount = count;
AddressTx.find({a_id: hash}).countDocuments().then((totalCount) => {
AddressTx.aggregate([
{ $match: { a_id: hash } },
{ $sort: {blockindex: -1} },
{ $sort: { blockindex: -1 } },
{ $skip: Number(start) },
{
$group: {
@@ -972,29 +962,27 @@ module.exports = {
balance: '$balance'
}
},
{ $sort: {blockindex: -1} }
{ $sort: { blockindex: -1 } }
]).then((balance_sum) => {
AddressTx.find({a_id: hash}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec().then((address_tx) => {
var txs = [];
var count = address_tx.length;
var running_balance = balance_sum.length > 0 ? balance_sum[0].balance : 0;
var txs = [];
let txs = [];
let running_balance = (balance_sum.length > 0 ? balance_sum[0].balance : 0);
lib.syncLoop(count, function (loop) {
var i = loop.iteration();
find_tx(address_tx[i].txid, function (tx) {
if (tx && !txs.includes(tx)) {
async.eachSeries(address_tx, function(current_addresstx, loop) {
find_tx(current_addresstx.txid, function(tx) {
if (tx) {
// set the balance for this tx
tx.balance = running_balance;
txs.push(tx);
loop.next();
} else if (!txs.includes(tx)) {
txs.push("1. Not found");
loop.next();
} else
loop.next();
running_balance = running_balance - address_tx[i].amount;
// add tx to list of txes
txs.push(tx);
// subtract from the running balance
running_balance -= current_addresstx.amount;
}
// move to next address tx
loop();
});
}, function () {
return cb(txs, totalCount);
@@ -1133,7 +1121,7 @@ module.exports = {
},
get_distribution: function(richlist, stats, cb) {
var distribution = {
const distribution = {
supply: stats.supply,
t_1_25: {percent: 0, total: 0 },
t_26_50: {percent: 0, total: 0 },
@@ -1142,32 +1130,30 @@ module.exports = {
t_101plus: {percent: 0, total: 0 }
};
lib.syncLoop(richlist.balance.length, function (loop) {
var i = loop.iteration();
var count = i + 1;
var percentage = ((richlist.balance[i].balance / 100000000) / stats.supply) * 100;
async.timesSeries(richlist.balance.length, function(i, loop) {
const percentage = ((richlist.balance[i].balance / 100000000) / stats.supply) * 100;
if (count <= 25 ) {
if ((i + 1) <= 25) {
distribution.t_1_25.percent = distribution.t_1_25.percent + percentage;
distribution.t_1_25.total = distribution.t_1_25.total + (richlist.balance[i].balance / 100000000);
}
if (count <= 50 && count > 25) {
if ((i + 1) <= 50 && (i + 1) > 25) {
distribution.t_26_50.percent = distribution.t_26_50.percent + percentage;
distribution.t_26_50.total = distribution.t_26_50.total + (richlist.balance[i].balance / 100000000);
}
if (count <= 75 && count > 50) {
if ((i + 1) <= 75 && (i + 1) > 50) {
distribution.t_51_75.percent = distribution.t_51_75.percent + percentage;
distribution.t_51_75.total = distribution.t_51_75.total + (richlist.balance[i].balance / 100000000);
}
if (count <= 100 && count > 75) {
if ((i + 1) <= 100 && (i + 1) > 75) {
distribution.t_76_100.percent = distribution.t_76_100.percent + percentage;
distribution.t_76_100.total = distribution.t_76_100.total + (richlist.balance[i].balance / 100000000);
}
loop.next();
loop();
}, function() {
distribution.t_101plus.percent = parseFloat(100 - distribution.t_76_100.percent - distribution.t_51_75.percent - distribution.t_26_50.percent - distribution.t_1_25.percent - (settings.richlist_page.burned_coins.include_burned_coins_in_distribution == true && richlist.burned > 0 ? ((richlist.burned / 100000000) / stats.supply) * 100 : 0)).toFixed(2);
distribution.t_101plus.total = parseFloat(distribution.supply - distribution.t_76_100.total - distribution.t_51_75.total - distribution.t_26_50.total - distribution.t_1_25.total - (settings.richlist_page.burned_coins.include_burned_coins_in_distribution == true && richlist.burned > 0 ? (richlist.burned / 100000000) : 0)).toFixed(8);
@@ -1187,23 +1173,21 @@ module.exports = {
// updates heavycoin stats
// height: current block height, count: amount of votes to store
update_heavy: function(coin, height, count, cb) {
var newVotes = [];
lib.get_maxmoney(function(maxmoney) {
lib.get_maxvote(function(maxvote) {
lib.get_vote(function(vote) {
lib.get_phase(function(phase) {
lib.get_reward(function(reward) {
module.exports.get_stats(settings.coin.name, function(stats) {
lib.get_estnext(function(estnext) {
lib.get_nextin(function(nextin) {
let newVotes = [];
lib.get_maxmoney( function (maxmoney) {
lib.get_maxvote( function (maxvote) {
lib.get_vote( function (vote) {
lib.get_phase( function (phase) {
lib.get_reward( function (reward) {
module.exports.get_stats(settings.coin.name, function (stats) {
lib.get_estnext( function (estnext) {
lib.get_nextin( function (nextin) {
lib.syncLoop(count, function (loop) {
var i = loop.iteration();
lib.get_blockhash(height - i, function (hash) {
lib.get_block(hash, function (block) {
async.timesSeries(count, function(i, loop) {
lib.get_blockhash(height - i, function(hash) {
lib.get_block(hash, function(block) {
newVotes.push({ count: height - i, reward: block.reward, vote: (block && block.vote ? block.vote : 0) });
loop.next();
loop();
});
});
}, function() {
@@ -1219,7 +1203,7 @@ module.exports = {
votes: newVotes
}).then(() => {
// update reward_last_updated value
module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (new_cb) {
module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (update_success) {
console.log('Heavycoin update complete');
return cb();
});
@@ -1830,10 +1814,10 @@ module.exports = {
},
populate_claim_address_names: function(tx, cb) {
var addresses = [];
const addresses = [];
// loop through vin addresses
tx.vin.forEach(function (vin) {
tx.vin.forEach(function(vin) {
// check if this address already exists
if (addresses.indexOf(vin.addresses) == -1) {
// add address to array
@@ -1842,7 +1826,7 @@ module.exports = {
});
// loop through vout addresses
tx.vout.forEach(function (vout) {
tx.vout.forEach(function(vout) {
// check if this address already exists
if (addresses.indexOf(vout.addresses) == -1) {
// add address to array
@@ -1851,31 +1835,29 @@ module.exports = {
});
// loop through address array
lib.syncLoop(addresses.length, function(loop) {
var a = loop.iteration();
module.exports.get_claim_name(addresses[a], function(claim_name) {
async.eachSeries(addresses, function(current_address, loop) {
module.exports.get_claim_name(current_address, function(claim_name) {
if (claim_name != null && claim_name != '') {
// look for address in vin
for (v = 0; v < tx.vin.length; v++) {
for (let v = 0; v < tx.vin.length; v++) {
// check if this is the correct address
if (tx.vin[v].addresses == addresses[a]) {
if (tx.vin[v].addresses == current_address) {
// add claim name to array
tx.vin[v]['claim_name'] = claim_name;
}
}
// look for address in vout
for (v = 0; v < tx.vout.length; v++) {
for (let v = 0; v < tx.vout.length; v++) {
// check if this is the correct address
if (tx.vout[v].addresses == addresses[a]) {
if (tx.vout[v].addresses == current_address) {
// add claim name to array
tx.vout[v]['claim_name'] = claim_name;
}
}
}
loop.next();
loop();
});
}, function() {
// return modified tx object
lib/explorer.js
+284 -397
View File
@@ -1,10 +1,9 @@
var request = require('postman-request'),
async = require('async'),
settings = require('./settings'),
Address = require('../models/address');
var base_server = 'http://127.0.0.1:' + settings.webserver.port + "/";
var base_url = base_server + 'api/';
const request = require('postman-request');
const async = require('async');
const settings = require('./settings');
const Address = require('../models/address');
const base_server = 'http://127.0.0.1:' + settings.webserver.port + '/';
const base_url = base_server + 'api/';
const onode = require('./node');
const client = new onode.Client(settings.wallet);
@@ -76,7 +75,7 @@ function convertHashUnits(hashes) {
}
}
function processVoutAddresses(address_list, vout_value, arr_vout, cb) {
function processVoutAddresses(address_list, vout_value, arr_vout) {
// check if there are any addresses to process
if (address_list != null && address_list.length > 0) {
// check if vout address is inside an array
@@ -85,28 +84,21 @@ function processVoutAddresses(address_list, vout_value, arr_vout, cb) {
address_list[0] = address_list[0][0];
}
const amount_sat = module.exports.convert_to_satoshi(parseFloat(vout_value));
const index = module.exports.is_unique(arr_vout, address_list[0], 'addresses');
// check if vout address is unique, if so add to array, if not add its amount to existing index
module.exports.is_unique(arr_vout, address_list[0], 'addresses', function(unique, index) {
if (unique == true) {
// unique vout
module.exports.convert_to_satoshi(parseFloat(vout_value), function(amount_sat) {
arr_vout.push({addresses: address_list[0], amount: amount_sat});
return cb(arr_vout);
});
} else {
// already exists
module.exports.convert_to_satoshi(parseFloat(vout_value), function(amount_sat) {
arr_vout[index].amount = arr_vout[index].amount + amount_sat;
return cb(arr_vout);
});
}
});
} else {
// no address, move to next vout
return cb(arr_vout);
if (index == -1) {
// unique vout
arr_vout.push({addresses: address_list[0], amount: amount_sat});
} else {
// already exists
arr_vout[index].amount += amount_sat;
}
}
// return the list of addresses
return arr_vout;
}
function encodeP2PKaddress(p2pk_descriptor, cb) {
@@ -182,12 +174,146 @@ function normalizeMasternodeCount(raw_count) {
}
}
function get_input_addresses(input, vout, cb) {
let addresses = [];
if (input.coinbase) {
let amount = 0;
for (let i = 0; i < vout.length; i++)
amount += parseFloat(vout[i].value);
addresses.push({ hash: 'coinbase', amount: amount });
return cb(addresses, null);
} else {
module.exports.get_rawtransaction(input.txid, function(tx) {
if (tx) {
let tx_type = null;
async.eachSeries(tx.vout, function(current_vout, loop) {
if (current_vout.n == input.vout) {
if (current_vout.scriptPubKey.addresses || current_vout.scriptPubKey.address) {
let new_address = current_vout.scriptPubKey.address || current_vout.scriptPubKey.addresses[0];
// check if address is inside an array
if (Array.isArray(new_address)) {
// extract the address
new_address = new_address[0];
}
const index = module.exports.is_unique(addresses, new_address, 'hash');
if (index == -1)
addresses.push({hash: new_address, amount: current_vout.value});
else
addresses[index].amount += current_vout.value;
loop();
} else {
// no addresses defined
// check if bitcoin features are enabled
if (settings.blockchain_specific.bitcoin.enabled == true) {
// assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address
encodeP2PKaddress(current_vout.scriptPubKey.asm, function(p2pkh_address) {
// check if the address was encoded properly
if (p2pkh_address != null) {
// mark this tx as p2pk
tx_type = 'p2pk';
// check if address is inside an array
if (Array.isArray(p2pkh_address)) {
// extract the address
p2pkh_address = p2pkh_address[0];
}
// save the P2PKH address
const index = module.exports.is_unique(addresses, p2pkh_address, 'hash');
if (index == -1)
addresses.push({hash: p2pkh_address, amount: current_vout.value});
else
addresses[index].amount += current_vout.value;
loop();
} else {
// could not decipher the address, save as unknown and move to next vin
console.log('Failed to find vin address from tx ' + input.txid);
const index = module.exports.is_unique(addresses, 'unknown_address', 'hash');
if (index == -1)
addresses.push({hash: 'unknown_address', amount: current_vout.value});
else
addresses[index].amount += current_vout.value;
loop();
}
});
} else {
// could not decipher the address, save as unknown and move to next vin
console.log('Failed to find vin address from tx ' + input.txid);
const index = module.exports.is_unique(addresses, 'unknown_address', 'hash');
if (index == -1)
addresses.push({hash: 'unknown_address', amount: current_vout.value});
else
addresses[index].amount += current_vout.value;
loop();
}
}
} else
loop();
}, function() {
return cb(addresses, tx_type);
});
} else
return cb();
});
}
}
function finalize_vout(first_vout, arr_vout, arr_vin, tx_type, txid, cb) {
if (typeof first_vout !== 'undefined' && first_vout.scriptPubKey.type == 'nonstandard') {
if (arr_vin.length > 0 && arr_vout.length > 0) {
if (arr_vin[0].addresses == arr_vout[0].addresses) {
// pos
arr_vout[0].amount -= arr_vin[0].amount;
arr_vin.shift();
// check if any vin remains
if (arr_vin == null || arr_vin.length == 0) {
// empty vin should be linked to coinbase
arr_vin = [{coinbase: 'coinbase'}];
let new_vout = [];
// loop through the arr_vout to create a copy of the data with coin amounts only for use with prepare_vin()
for (i = 0; i < arr_vout.length; i++)
new_vout.push({ value: arr_vout[i].amount / 100000000 });
// call the prepare_vin again to populate the vin data correctly
module.exports.prepare_vin({ txid: txid, vin: arr_vin, vout: new_vout}, function(return_vin, return_tx_type_vin) {
return cb(arr_vout, return_vin, return_tx_type_vin);
});
} else
return cb(arr_vout, arr_vin, tx_type);
} else
return cb(arr_vout, arr_vin, tx_type);
} else
return cb(arr_vout, arr_vin, tx_type);
} else
return cb(arr_vout, arr_vin, tx_type);
}
module.exports = {
convert_to_satoshi: function(amount, cb) {
// fix to 8dp & convert to string
var fixed = amount.toFixed(8).toString();
convert_to_satoshi: function(amount) {
// fix to 8 decimals and convert to string
const fixed = amount.toFixed(8).toString();
// remove decimal (.) and return integer
return cb(parseInt(fixed.replace('.', '')));
return parseInt(fixed.replace('.', ''));
},
get_hashrate: function(cb) {
@@ -810,77 +936,13 @@ module.exports = {
}
},
// synchonous loop used to interate through an array,
// avoid use unless absolutely neccessary
syncLoop: function(iterations, process, exit) {
var index = 0,
done = false,
shouldExit = false;
var loop = {
next: function() {
if (done) {
if (shouldExit && exit) {
// exit if we're done
exit();
}
// stop the loop if we're done
return;
}
// if we're not finished
if (index < iterations) {
// increment our index
index++;
if (index % 100 === 0) {
// clear stack
setTimeout(function() {
// run our process, pass in the loop
process(loop);
}, 1);
} else {
// run our process, pass in the loop
process(loop);
}
} else {
// otherwise we're done
// make sure we say we're done
done = true;
if (exit) {
// call the callback on exit
exit();
}
}
},
iteration: function() {
// return the loop number we're on
return index - 1;
},
break: function(end) {
// end the loop
done = true;
// passing end as true means we still call the exit callback
shouldExit = end;
}
};
loop.next();
return loop;
},
balance_supply: function(cb) {
Address.find({}, 'balance').where('balance').gt(0).exec().then((docs) => {
var count = 0;
let count = 0;
module.exports.syncLoop(docs.length, function (loop) {
var i = loop.iteration();
count = count + docs[i].balance;
loop.next();
async.eachSeries(docs, function(current_doc, loop) {
count += current_doc.balance;
loop();
}, function() {
return cb(count);
});
@@ -1080,222 +1142,160 @@ module.exports = {
});
},
is_unique: function(array, object, key_name, cb) {
var unique = true;
var index = null;
is_unique: function(array, object, key_name) {
for (let i = 0; i < array.length; i++) {
if (array[i][key_name] === object)
return i;
}
module.exports.syncLoop(array.length, function (loop) {
var i = loop.iteration();
if (array[i][key_name] == object) {
unique = false;
index = i;
loop.break(true);
loop.next();
} else
loop.next();
}, function() {
return cb(unique, index);
});
return -1;
},
calculate_total: function(vout, cb) {
var total = 0;
calculate_total: function(vout) {
let total = 0;
module.exports.syncLoop(vout.length, function (loop) {
var i = loop.iteration();
for (let i = 0; i < vout.length; i++)
total += vout[i].amount;
total = total + vout[i].amount;
loop.next();
}, function() {
return cb(total);
});
return total;
},
prepare_vout: function(vout, txid, vin, vhidden, cb) {
var arr_vout = [];
var arr_vin = vin;
var tx_type = null;
let arr_vout = [];
let arr_vin = vin;
let tx_type = null;
try {
module.exports.syncLoop(vout.length, function (loop) {
var i = loop.iteration();
async.eachSeries(vout, function(current_vout, loop) {
// make sure vout has an address
if (vout[i].scriptPubKey.type != 'nonstandard' && vout[i].scriptPubKey.type != 'nulldata') {
if (current_vout.scriptPubKey.type != 'nonstandard' && current_vout.scriptPubKey.type != 'nulldata') {
// check if this is a zerocoin tx
if (vout[i].scriptPubKey.type != 'zerocoinmint') {
var address_list = vout[i].scriptPubKey.addresses;
if (current_vout.scriptPubKey.type != 'zerocoinmint') {
const address_list = current_vout.scriptPubKey.addresses;
// check if there are one or more addresses in the vout
if (address_list == null || address_list.length == 0) {
// no addresses defined
// check if there is a single address defined
if (vout[i].scriptPubKey.address == null) {
if (current_vout.scriptPubKey.address == null) {
// no single address defined
// check if bitcoin features are enabled
if (settings.blockchain_specific.bitcoin.enabled == true) {
// assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address
encodeP2PKaddress(vout[i].scriptPubKey.asm, function(p2pkh_address) {
encodeP2PKaddress(current_vout.scriptPubKey.asm, function(p2pkh_address) {
// check if the address was encoded properly
if (p2pkh_address != null) {
// mark this tx as p2pk
tx_type = 'p2pk';
// process vout addresses
processVoutAddresses(p2pkh_address, vout[i].value, arr_vout, function(vout_array) {
// save updated array
arr_vout = vout_array;
// move to next vout
loop.next();
});
// process vout addresses and save the updated array
arr_vout = processVoutAddresses(p2pkh_address, current_vout.value, arr_vout);
} else {
// could not decipher the address, save as unknown and move to next vout
// could not decipher the address, save as unknown
console.log('Failed to find vout address from tx ' + txid);
// process vout addresses
processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) {
// save updated array
arr_vout = vout_array;
// move to next vout
loop.next();
});
// process vout addresses and save the updated array
arr_vout = processVoutAddresses(['unknown_address'], current_vout.value, arr_vout);
}
// move to next vout
loop();
});
} else {
// could not decipher the address, save as unknown and move to next vout
console.log('Failed to find vout address from tx ' + txid);
// process vout addresses
processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) {
// save updated array
arr_vout = vout_array;
// move to next vout
loop.next();
});
// process vout addresses and save the updated array
arr_vout = processVoutAddresses(['unknown_address'], current_vout.value, arr_vout);
// move to next vout
loop();
}
} else {
// process vout address
processVoutAddresses([vout[i].scriptPubKey.address], vout[i].value, arr_vout, function(vout_array) {
// save updated array
arr_vout = vout_array;
// move to next vout
loop.next();
});
// process vout addresses and save the updated array
arr_vout = processVoutAddresses([current_vout.scriptPubKey.address], current_vout.value, arr_vout);
// move to next vout
loop();
}
} else {
// process vout addresses
processVoutAddresses(address_list, vout[i].value, arr_vout, function(vout_array) {
// save updated array
arr_vout = vout_array;
// move to next vout
loop.next();
});
// process vout addresses and save the updated array
arr_vout = processVoutAddresses(address_list, current_vout.value, arr_vout);
// move to next vout
loop();
}
} else {
// TODO: add support for zerocoin transactions
console.log('Zerocoin tx found. skipping for now as it is unsupported');
tx_type = "zerocoin";
loop.next();
loop();
}
} else {
// no address, move to next vout
loop.next();
loop();
}
}, function() {
// check if zksnarks is enabled
if (settings.blockchain_specific.zksnarks.enabled == true) {
// check for hidden/anonymous outputs
if (vhidden != null && vhidden.length > 0) {
tx_type = "zksnarks";
// loop through all hidden/anonymous outputs
module.exports.syncLoop(vhidden.length, function (loop) {
var i = loop.iteration();
// check if zksnarks is enabled and there are any hidden/anonymous outputs
if (settings.blockchain_specific.zksnarks.enabled == true && vhidden != null && vhidden.length > 0) {
const hidden_address = 'hidden_address';
tx_type = 'zksnarks';
if (vhidden[i].vpub_old > 0) {
// process vout addresses
processVoutAddresses(['hidden_address'], parseFloat(vhidden[i].vpub_old), arr_vout, function(vout_array) {
// save updated array
arr_vout = vout_array;
// move to next vout
loop.next();
});
} else {
if ((!vout || vout.length == 0) && (!vin || vin.length == 0)) {
// hidden sender is sending to hidden recipient
// the sent and received values are not known in this case. only the fee paid is known and subtracted from the sender.
// process vout addresses
processVoutAddresses(['hidden_address'], 0, arr_vout, function(vout_array) {
// save updated array
arr_vout = vout_array;
// add a private send address with the known amount sent
module.exports.is_unique(arr_vin, 'hidden_address', 'addresses', function(unique, index) {
if (unique == true) {
module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
arr_vin.push({addresses: 'hidden_address', amount: amount_sat});
// move to next vout
loop.next();
});
} else {
module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
arr_vin[index].amount = arr_vin[index].amount + amount_sat;
// move to next vout
loop.next();
});
}
});
});
// loop through all hidden/anonymous outputs
async.eachSeries(vhidden, function(current_vhidden, loop) {
if (current_vhidden.vpub_old > 0) {
// process vout addresses and save the updated array
arr_vout = processVoutAddresses([hidden_address], parseFloat(current_vhidden.vpub_old), arr_vout);
// move to next vout
loop();
} else {
const amount_sat = module.exports.convert_to_satoshi(parseFloat(current_vhidden.vpub_new));
const index = module.exports.is_unique(arr_vin, hidden_address, 'addresses');
if ((!vout || vout.length == 0) && (!vin || vin.length == 0)) {
// hidden sender is sending to hidden recipient
// the sent and received values are not known in this case. only the fee paid is known and subtracted from the sender.
// process vout addresses and save the updated array
arr_vout = processVoutAddresses([hidden_address], 0, arr_vout);
// add a private send address with the known amount sent
if (index == -1) {
// add hidden address to array
arr_vin.push({addresses: hidden_address, amount: amount_sat});
} else {
module.exports.is_unique(arr_vin, 'hidden_address', 'addresses', function(unique, index) {
if (unique == true) {
module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
arr_vin.push({addresses: 'hidden_address', amount: amount_sat});
// move to next vout
loop.next();
});
} else {
module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
arr_vin[index].amount = arr_vin[index].amount + amount_sat;
// move to next vout
loop.next();
});
}
});
}
}
});
}
}
if (typeof vout[0] !== 'undefined' && vout[0].scriptPubKey.type == 'nonstandard') {
if (arr_vin.length > 0 && arr_vout.length > 0) {
if (arr_vin[0].addresses == arr_vout[0].addresses) {
//PoS
arr_vout[0].amount = arr_vout[0].amount - arr_vin[0].amount;
arr_vin.shift();
// check if any vin remains
if (arr_vin == null || arr_vin.length == 0) {
// empty vin should be linked to coinbase
arr_vin = [{coinbase: "coinbase"}];
var new_vout = [];
// loop through the arr_vout to create a copy of the data with coin amounts only for use with prepare_vin()
for (i = 0; i < arr_vout.length; i++) {
new_vout.push({
value: arr_vout[i].amount / 100000000
});
// update hidden address amount in array
arr_vin[index].amount = arr_vin[index].amount + amount_sat;
}
// call the prepare_vin again to populate the vin data correctly
module.exports.prepare_vin({txid: txid, vin: arr_vin, vout: new_vout}, function(return_vin, return_tx_type_vin) {
return cb(arr_vout, return_vin, return_tx_type_vin);
});
// move to next vout
loop();
} else {
return cb(arr_vout, arr_vin, tx_type);
// add a private send address with the known amount sent
if (index == -1) {
// add hidden address to array
arr_vin.push({addresses: hidden_address, amount: amount_sat});
} else {
// update hidden address amount in array
arr_vin[index].amount += amount_sat;
}
// move to next vout
loop();
}
} else
return cb(arr_vout, arr_vin, tx_type);
} else
return cb(arr_vout, arr_vin, tx_type);
} else
return cb(arr_vout, arr_vin, tx_type);
}
}, function() {
// finished updating hidden vout data
finalize_vout(vout[0], arr_vout, arr_vin, tx_type, txid, function(final_arr_vout, final_arr_vin, final_tx_type) {
return cb(final_arr_vout, final_arr_vin, final_tx_type);
});
});
} else {
// finalize vout data
finalize_vout(vout[0], arr_vout, arr_vin, tx_type, txid, function(final_arr_vout, final_arr_vin, final_tx_type) {
return cb(final_arr_vout, final_arr_vin, final_tx_type);
});
}
});
} catch(err) {
// check if a "Maximum call stack size exceeded" error occurred
@@ -1309,122 +1309,12 @@ module.exports = {
}
},
get_input_addresses: function(input, vout, cb) {
var addresses = [];
if (input.coinbase) {
var amount = 0;
module.exports.syncLoop(vout.length, function (loop) {
var i = loop.iteration();
amount = amount + parseFloat(vout[i].value);
loop.next();
}, function() {
addresses.push({hash: 'coinbase', amount: amount});
return cb(addresses, null);
});
} else {
module.exports.get_rawtransaction(input.txid, function(tx) {
if (tx) {
var tx_type = null;
module.exports.syncLoop(tx.vout.length, function (loop) {
var i = loop.iteration();
if (tx.vout[i].n == input.vout) {
if (tx.vout[i].scriptPubKey.addresses || tx.vout[i].scriptPubKey.address) {
var new_address = tx.vout[i].scriptPubKey.address || tx.vout[i].scriptPubKey.addresses[0];
// check if address is inside an array
if (Array.isArray(new_address)) {
// extract the address
new_address = new_address[0];
}
module.exports.is_unique(addresses, new_address, 'hash', function(unique, index) {
if (unique == true)
addresses.push({hash: new_address, amount: tx.vout[i].value});
else
addresses[index].amount = addresses[index].amount + tx.vout[i].value;
loop.break(true);
loop.next();
});
} else {
// no addresses defined
// check if bitcoin features are enabled
if (settings.blockchain_specific.bitcoin.enabled == true) {
// assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address
encodeP2PKaddress(tx.vout[i].scriptPubKey.asm, function(p2pkh_address) {
// check if the address was encoded properly
if (p2pkh_address != null) {
// mark this tx as p2pk
tx_type = 'p2pk';
// check if address is inside an array
if (Array.isArray(p2pkh_address)) {
// extract the address
p2pkh_address = p2pkh_address[0];
}
// save the P2PKH address
module.exports.is_unique(addresses, p2pkh_address, 'hash', function(unique, index) {
if (unique == true)
addresses.push({hash: p2pkh_address, amount: tx.vout[i].value});
else
addresses[index].amount = addresses[index].amount + tx.vout[i].value;
loop.break(true);
loop.next();
});
} else {
// could not decipher the address, save as unknown and move to next vin
console.log('Failed to find vin address from tx ' + input.txid);
module.exports.is_unique(addresses, 'unknown_address', 'hash', function(unique, index) {
if (unique == true)
addresses.push({hash: 'unknown_address', amount: tx.vout[i].value});
else
addresses[index].amount = addresses[index].amount + tx.vout[i].value;
loop.break(true);
loop.next();
});
}
});
} else {
// could not decipher the address, save as unknown and move to next vin
console.log('Failed to find vin address from tx ' + input.txid);
module.exports.is_unique(addresses, 'unknown_address', 'hash', function(unique, index) {
if (unique == true)
addresses.push({hash: 'unknown_address', amount: tx.vout[i].value});
else
addresses[index].amount = addresses[index].amount + tx.vout[i].value;
loop.break(true);
loop.next();
});
}
}
} else
loop.next();
}, function() {
return cb(addresses, tx_type);
});
} else
return cb();
});
}
},
prepare_vin: function(tx, cb) {
var arr_vin = [];
var tx_type = null;
let arr_vin = [];
let tx_type = null;
module.exports.syncLoop(tx.vin.length, function (loop) {
var i = loop.iteration();
module.exports.get_input_addresses(tx.vin[i], tx.vout, function(addresses, tx_type_vin) {
async.eachSeries(tx.vin, function(vin, loop) {
get_input_addresses(vin, tx.vout, function(addresses, tx_type_vin) {
// check if the tx type is set
if (tx_type_vin != null) {
// set the tx type return value
@@ -1432,28 +1322,25 @@ module.exports = {
}
if (addresses && addresses.length) {
module.exports.is_unique(arr_vin, addresses[0].hash, 'addresses', function(unique, index) {
if (unique == true) {
module.exports.convert_to_satoshi(parseFloat(addresses[0].amount), function(amount_sat) {
arr_vin.push({addresses: addresses[0].hash, amount: amount_sat});
loop.next();
});
} else {
module.exports.convert_to_satoshi(parseFloat(addresses[0].amount), function(amount_sat) {
arr_vin[index].amount = arr_vin[index].amount + amount_sat;
loop.next();
});
}
});
const amount_sat = module.exports.convert_to_satoshi(parseFloat(addresses[0].amount));
const index = module.exports.is_unique(arr_vin, addresses[0].hash, 'addresses');
if (index == -1)
arr_vin.push({addresses: addresses[0].hash, amount: amount_sat});
else
arr_vin[index].amount += amount_sat;
loop();
} else {
// could not decipher the address, save as unknown and move to next vin
console.log('Failed to find vin address from tx ' + tx.txid);
module.exports.is_unique(arr_vin, 'unknown_address', 'addresses', function(unique, index) {
if (unique == true)
arr_vin.push({addresses: 'unknown_address', amount: 0});
loop.next();
});
const index = module.exports.is_unique(arr_vin, 'unknown_address', 'addresses');
if (index == -1)
arr_vin.push({addresses: 'unknown_address', amount: 0});
loop();
}
});
}, function() {
lib/settings.js
+15 -1
View File
@@ -1412,10 +1412,24 @@ exports.sync = {
// BALANCES : get the supply by running a query on the addresses collection and summing up all positive balances (potentially a long running query for blockchains with tons of addresses)
// TXOUTSET : retrieved from gettxoutsetinfo rpc cmd
"supply": "GETINFO",
// batch_size: The maximum number of records before saving data to the database.
// This value is used for syncing transactions, addresses and address transactions.
// Each record type is processed within a single block so batching only happens if there are more than `batch_size` txes in a single block for example.
// If the number of txes is lower than `batch_size` then all txes are saved in one batch.
// A higher batch_size can save data faster than having to do smaller batches but only up to a certain point since it also requires more memory and resources for larger batches and the optimal number depends entirely on your server's resources.
// A lower batch size generally ensures there will be no memory limitations although it can also slow down the sync process.
// It is recommended to leave this value alone unless you know what you are doing although some experimentation with different batch sizes using the benchmark script can often help determine the optimal setting for your server.
"batch_size": 5000,
// elastic_stack_size: If a "RangeError: Maximum call stack size exceeded" error occurs during a block sync (which can happen when dealing with large transactions with many addresses), the sync script will automatically be reloaded using a larger stack size value which increases memory usage based on this value.
// NOTE: If the first reload of the sync script still doesn't have enough memory to handle processing of a large transaction, the sync is smart enough to continue increasing the stack size by this value again and again until it finishes processing all blocks and then returns back to the default amount of memory for future blocks.
// It is recommended to leave this value alone unless you know what you are doing.
"elastic_stack_size": 4096
"elastic_stack_size": 4096,
// wait_for_bulk_database_save: Determine whether to wait for all records to be saved or just send the records without waiting for save confirmation when saving bulk data
// This setting only controls how to treat records that are bulk saved to the database which include txes, addresstxes and addresses
// If set to true, bulk transactions to the database will wait for save confirmation which results in a slower save time but also returns information about which records failed to save
// If set to false, bulk transactions to the database will not wait for save confirmation which results in a faster save time but will not return any error message for records that failed to save
// NOTE: If you want to sync data as fast as possible and are sure that your blockchain doesn't contain any problematic or unsupported data types then you can set this value to "false" to maximize the speed of the block sync
"wait_for_bulk_database_save": true
};
// captcha: a collection of settings that pertain to the captcha security used by different elements of the explorer