From 0b0ef817f187584484e193ab115f6bdd7ed7ea4e Mon Sep 17 00:00:00 2001 From: Joe Uhren Date: Thu, 9 Jan 2025 20:00:37 -0700 Subject: [PATCH] "Maximum call stack size exceeded" error bug fixed -The "Maximum call stack size exceeded" error is now handled internally by the block sync script in a way which will capture the error and re-launch the sync using a larger stack size and have the sync resume from where it left off. If the re-launch still doesn't have enough memory it will continue re-launching with more and more memory until the sync can finish without errors and then it will return to sync with a lower memory footprint for future syncs -Added a new option for sync.elastic_stack_size which is used to determine how much memory should be used to increase the stack size for the block sync after encountering the "Maximum call stack size exceeded" error -Fixed an issue with the block sync when using more than 1 thread that could sometimes cause the flattened txes value in the coinstats database to be written incorrectly (Use `npm run reindex-txcount` to fix this issue without needing to reindex the entire database) -Updated the benchmark script so that it can also benefit from being able to capture the "Maximum call stack size exceeded" error even though the timing will be off so it outputs a new warning in that scenario which instructs to run the benchmark again with a higher stack size to properly capture the benchmark time -Removed the "Maximum call stack size exceeded" error notes from the "Known Issues" section of the README --- README.md | 24 --- lib/block_sync.js | 367 ++++++++++++++++++++++++++++++----------- lib/explorer.js | 277 ++++++++++++++++--------------- lib/settings.js | 6 +- scripts/benchmark.js | 141 +++++++++++++--- scripts/sync.js | 121 +++++++++----- settings.json.template | 6 +- 7 files changed, 614 insertions(+), 328 deletions(-) diff --git a/README.md b/README.md index 53a550d..73a724e 100644 --- a/README.md +++ b/README.md @@ -936,30 +936,6 @@ The benchmark script can be started with the following command: ### Known Issues -**exceeding stack size** - -``` -RangeError: Maximum call stack size exceeded -``` - -Nodes default stack size may be too small to index addresses with many tx's. If you experience the above error while running sync.js the stack size needs to be increased. - -To determine the default setting run: - -``` -node --v8-options | grep -B0 -A1 stack-size -``` - -To run a sync with a larger stack size launch with: - -``` -node --stack-size=[SIZE] scripts/sync.js index update -``` - -Where [SIZE] is an integer higher than the default. - -*note: SIZE will depend on which blockchain you are using, you may need to play around a bit to find an optimal setting* - **Error: bind EACCES ...** This error can appear when you try to run the explorer on a port number lower than 1024. There are a couple solutions to this problem which are explained in more detail in the [Run Express Webserver on Port 80](#run-express-webserver-on-port-80) section. diff --git a/lib/block_sync.js b/lib/block_sync.js index 0274b90..a7f9347 100644 --- a/lib/block_sync.js +++ b/lib/block_sync.js @@ -6,6 +6,7 @@ const lib = require('./explorer'); const settings = require('../lib/settings'); const async = require('async'); let stopSync = false; +let stackSizeErrorId = null; function check_delete_tx(tx, block_height, tx_count, timeout, cb) { // check if the tx object exists and does not match the current block height @@ -116,84 +117,119 @@ function update_address(hash, blockheight, txid, amount, type, cb) { }); } +function finalize_update_tx_db(coin, check_only, end, txes, cb) { + let statUpdateObject = {}; + + // check what stats data should be updated + if (stopSync || stackSizeErrorId || check_only == 2) { + // only update txes when fixing invalid and missing blocks or when a "normal" sync was stopped prematurely + statUpdateObject.txes = txes; + } else { + // update last and txes values for "normal" sync that finishes without being stopped prematurely + statUpdateObject = { + txes: txes, + last: end + }; + } + + // update local stats + Stats.updateOne({coin: coin}, statUpdateObject).then(() => { + return cb(); + }).catch((err) => { + console.log(err); + return cb(); + }); +} + module.exports = { save_tx: function(txid, blockheight, block, cb) { lib.get_rawtransaction(txid, function(tx) { if (tx && tx != `${settings.localization.ex_error}: ${settings.localization.check_console}`) { lib.prepare_vin(tx, function(vin, tx_type_vin) { lib.prepare_vout(tx.vout, txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) { - lib.syncLoop(nvin.length, function (loop) { - const i = loop.iteration(); - - // check if address is inside an array - if (Array.isArray(nvin[i].addresses)) { - // extract the address - nvin[i].addresses = nvin[i].addresses[0]; - } - - update_address(nvin[i].addresses, blockheight, txid, nvin[i].amount, 'vin', function() { - loop.next(); - }); - }, function() { - lib.syncLoop(vout.length, function (subloop) { - const t = subloop.iteration(); + // check if vout is null which indicates an error + if (vout != null) { + lib.syncLoop(nvin.length, function (loop) { + const i = loop.iteration(); // check if address is inside an array - if (Array.isArray(vout[t].addresses)) { + if (Array.isArray(nvin[i].addresses)) { // extract the address - vout[t].addresses = vout[t].addresses[0]; + nvin[i].addresses = nvin[i].addresses[0]; } - if (vout[t].addresses) { - update_address(vout[t].addresses, blockheight, txid, vout[t].amount, 'vout', function() { - subloop.next(); - }); - } else - subloop.next(); + update_address(nvin[i].addresses, blockheight, txid, nvin[i].amount, 'vin', function() { + loop.next(); + }); }, function() { - lib.calculate_total(vout, function(total) { - var op_return = null; - var algo = null; + lib.syncLoop(vout.length, function (subloop) { + const t = subloop.iteration(); - // check if the op_return value should be decoded and saved - if (settings.transaction_page.show_op_return) { - // loop through vout to find the op_return value - tx.vout.forEach(function (vout_data) { - // check if the op_return value exists - if (vout_data.scriptPubKey != null && vout_data.scriptPubKey.asm != null && vout_data.scriptPubKey.asm.indexOf('OP_RETURN') > -1) { - // decode the op_return value - op_return = hex_to_ascii(vout_data.scriptPubKey.asm.replace('OP_RETURN', '').trim()); - } + // check if address is inside an array + if (Array.isArray(vout[t].addresses)) { + // extract the address + vout[t].addresses = vout[t].addresses[0]; + } + + if (vout[t].addresses) { + update_address(vout[t].addresses, blockheight, txid, vout[t].amount, 'vout', function() { + subloop.next(); }); - } + } else + subloop.next(); + }, function() { + lib.calculate_total(vout, function(total) { + var op_return = null; + var algo = null; - // check if the algo value should be saved - if (settings.block_page.multi_algorithm.show_algo) { - // get the algo value - algo = block[settings.block_page.multi_algorithm.key_name]; - } + // check if the op_return value should be decoded and saved + if (settings.transaction_page.show_op_return) { + // loop through vout to find the op_return value + tx.vout.forEach(function (vout_data) { + // check if the op_return value exists + if (vout_data.scriptPubKey != null && vout_data.scriptPubKey.asm != null && vout_data.scriptPubKey.asm.indexOf('OP_RETURN') > -1) { + // decode the op_return value + op_return = hex_to_ascii(vout_data.scriptPubKey.asm.replace('OP_RETURN', '').trim()); + } + }); + } - const newTx = new Tx({ - txid: tx.txid, - vin: (vin == null || vin.length == 0 ? [] : nvin), - vout: vout, - total: total.toFixed(8), - timestamp: tx.time, - blockhash: tx.blockhash, - blockindex: blockheight, - tx_type: (tx_type_vout == null ? tx_type_vin : tx_type_vout), - op_return: op_return, - algo: algo - }); + // check if the algo value should be saved + if (settings.block_page.multi_algorithm.show_algo) { + // get the algo value + algo = block[settings.block_page.multi_algorithm.key_name]; + } - newTx.save().then(() => { - return cb(null, vout.length > 0); - }).catch((err) => { - return cb(err, false); + const newTx = new Tx({ + txid: tx.txid, + vin: (vin == null || vin.length == 0 ? [] : nvin), + vout: vout, + total: total.toFixed(8), + timestamp: tx.time, + blockhash: tx.blockhash, + blockindex: blockheight, + tx_type: (tx_type_vout == null ? tx_type_vin : tx_type_vout), + op_return: op_return, + algo: algo + }); + + newTx.save().then(() => { + return cb(null, vout.length > 0); + }).catch((err) => { + return cb(err, false); + }); }); }); }); - }); + } else { + // create a custom error that will be specifically checked for later (NOTE: tx_type_vout contains the error code in this special case) + const customError = new Error(tx_type_vout); + + customError.code = tx_type_vout; + + // return the custom error + return cb(customError, false); + } }); }); } else @@ -214,6 +250,9 @@ module.exports = { if (parallel_tasks < 1) parallel_tasks = 1; + let finished_tasks = 0; + let processed_last_block = false; + for (i = start; i < (end + 1); i++) blocks_to_scan.push(i); @@ -242,6 +281,12 @@ module.exports = { }); async.eachLimit(blocks_to_scan, parallel_tasks, function(block_height, next_block) { + // check if this is the last block to process + if (block_height == end) { + // ensure the process knows not to wait for more threads to stop after this + processed_last_block = true; + } + // add the current block height to a queue and wait for it to be next in queue before starting to sync the block block_queue.push( { @@ -283,14 +328,21 @@ module.exports = { if (blockhash) { lib.get_block(blockhash, function(block) { if (block) { + let tx_counter = 0; + + // loop through all txes in this block async.eachLimit(block.tx, parallel_tasks, function(txid, next_tx) { + // increment tx counter + tx_counter++; + Tx.findOne({txid: txid}).then((tx) => { if (tx && check_only != 2) { setTimeout(function() { tx = null; + tx_counter--; // check if the script is stopping - if (stopSync && check_only != 2) { + if ((stopSync && check_only != 2) || stackSizeErrorId) { // stop the loop next_tx({}); } else @@ -307,10 +359,14 @@ module.exports = { // save the transaction to local database module.exports.save_tx(txid, block_height, block, function(err, tx_has_vout) { if (err) { - // output a nicer error msg for the 11000 error code "duplicate key error collection" which can happen in some blockchains with non-standard txids being reused - if (err.code === 11000) + // check the error code + if (err.code == 'StackSizeError') { + // ensure the process halts after stopping all sync threads + stackSizeErrorId = txid; + } else if (err.code === 11000) { + // output a nicer error msg for the 11000 error code "duplicate key error collection" which can happen in some blockchains with non-standard txids being reused console.log(`${settings.localization.ex_warning}: ${block_height}: ${txid} already exists`); - else + } else console.log(err); } else @@ -321,9 +377,10 @@ module.exports = { setTimeout(function() { tx = null; + tx_counter--; // check if the script is stopping - if (stopSync && check_only != 2) { + if ((stopSync && check_only != 2) || stackSizeErrorId) { // stop the loop next_tx({}); } else @@ -334,9 +391,10 @@ module.exports = { // skip adding the current tx setTimeout(function() { tx = null; + tx_counter--; // check if the script is stopping - if (stopSync && check_only != 2) { + if ((stopSync && check_only != 2) || stackSizeErrorId) { // stop the loop next_tx({}); } else @@ -350,9 +408,10 @@ module.exports = { setTimeout(function() { tx = null; + tx_counter--; // check if the script is stopping - if (stopSync && check_only != 2) { + if ((stopSync && check_only != 2) || stackSizeErrorId) { // stop the loop next_tx({}); } else @@ -360,19 +419,41 @@ module.exports = { }, timeout); }); }, function() { - setTimeout(function() { - blockhash = null; - block = null; + // set the retry limit to a value that will be reached in ~10 seconds based on the + // timeout value which should be more than enough time for all threads to finish + // processing their last tx in case of error or cancel/kill script + const retryLimit = (10000 / timeout); + let retryAttempts = 0; - // reset the slot in the block array back to 0 - block_numbers[slotIndex] = 0; + // wait for all threads to finish before continuing + const handle = setInterval(() => { + // check if all threads have properly finished or else the retry limit has been reached + // NOTE: the retry limit should never need to be used but is put in place to prevent an + // infinite loop just in case something goes very wrong + if (tx_counter === 0 || retryAttempts >= retryLimit) { + // stop waiting for all threads to finish + clearInterval(handle); - // check if the script is stopping - if (stopSync && check_only != 2) { - // stop the loop - next_block({}); - } else - next_block(); + blockhash = null; + block = null; + + // reset the slot in the block array back to 0 + block_numbers[slotIndex] = 0; + + // check if the script is stopping + if ((stopSync && check_only != 2) || stackSizeErrorId) { + // stop the loop + finished_tasks++; + next_block({}); + } else { + // check if the last block is finished or in process and increment the finished counter + if (processed_last_block) + finished_tasks++; + + // proceed to next block + next_block(); + } + } }, timeout); }); } else { @@ -383,11 +464,18 @@ module.exports = { block_numbers[slotIndex] = 0; // check if the script is stopping - if (stopSync && check_only != 2) { + if ((stopSync && check_only != 2) || stackSizeErrorId) { // stop the loop + finished_tasks++; next_block({}); - } else + } else { + // check if the last block is finished or in process and increment the finished counter + if (processed_last_block) + finished_tasks++; + + // proceed to next block next_block(); + } }, timeout); } }); @@ -397,11 +485,18 @@ module.exports = { block_numbers[slotIndex] = 0; // check if the script is stopping - if (stopSync && check_only != 2) { + if ((stopSync && check_only != 2) || stackSizeErrorId) { // stop the loop + finished_tasks++; next_block({}); - } else + } else { + // check if the last block is finished or in process and increment the finished counter + if (processed_last_block) + finished_tasks++; + + // proceed to next block next_block(); + } }, timeout); } }); @@ -410,27 +505,37 @@ module.exports = { () => {} ); }, function() { - var statUpdateObject = {}; + // set the retry limit to a value that will be reached in ~10 seconds based on the + // timeout value which should be more than enough time for all threads to finish + // processing their last tx in case of error or cancel/kill script + const retryLimit = (10000 / timeout); + let retryAttempts = 0; - // check what stats data should be updated - if (stopSync || check_only == 2) { - // only update txes when fixing invalid and missing blocks or when a "normal" sync was stopped prematurely - statUpdateObject.txes = txes; - } else { - // update last and txes values for "normal" sync that finishes without being stopped prematurely - statUpdateObject = { - txes: txes, - last: end - }; - } + // wait for all threads to finish before continuing + const handle = setInterval(() => { + // check if all threads have properly finished or else the retry limit has been reached + // NOTE: the retry limit should never need to be used but is put in place to prevent an + // infinite loop just in case something goes very wrong + if (finished_tasks === parallel_tasks || retryAttempts >= retryLimit) { + // stop waiting for all threads to finish + clearInterval(handle); - // update local stats - Stats.updateOne({coin: coin}, statUpdateObject).then(() => { - return cb(txes); - }).catch((err) => { - console.log(err); - return cb(txes); - }); + // finish the update + finalize_update_tx_db(coin, check_only, end, txes, function() { + // check if the script should continue or respawn a new process + if (!stackSizeErrorId) { + // continue to end of process + return cb(txes); + } else { + // reload the sync process + module.exports.respawnSync(); + } + }); + } else { + // still waiting for threads to finish so increment the retry counter + retryAttempts++; + } + }, timeout); }); }, @@ -664,5 +769,69 @@ module.exports = { getStopSync: function() { return stopSync; + }, + + setStackSizeErrorId: function(value) { + stackSizeErrorId = value; + }, + + getStackSizeErrorId: function() { + return stackSizeErrorId; + }, + + respawnSync: function() { + let extraArgument = ''; + + // check if this is the benchmark script which must be handled slightly differently than a normal sync + if (process.argv[1].endsWith('benchmark.js')) { + // add the extra argument for benchmark syncing + extraArgument = '1'; + } + + const stackSizeArg = process.execArgv.find(arg => arg.startsWith('--stack-size=')); + let stackSize = 4096; + + // check if the script was called with a stack size argument + if (stackSizeArg) { + // set the default stack size to the value that is currently being used + stackSize = parseInt(stackSizeArg.split('=')[1]); + } + + // increase stack size by the elastic amount + stackSize += settings.sync.elastic_stack_size; + + // show an error msg + console.log(`${settings.localization.ex_error}: Maximum call stack size exceeded while processing txid ${stackSizeErrorId}`); + console.log(`Restarting sync process with increased stack size of ${stackSize}. ${settings.localization.please_wait}..`); + + // filter out any existing --stack-size from execArgv + const execArgvWithoutStackSize = process.execArgv.filter(arg => !arg.startsWith('--stack-size=')); + + // populate child process arguments + const args = [ + ...execArgvWithoutStackSize, + `--stack-size=${stackSize}`, + ...process.argv.slice(1) // includes the path to the sync script and any user args + ]; + + // add the extra argument to resume the benchmark sync and skip the unlock step + if (extraArgument != '') + args.push(extraArgument); + else { + // remove lock + lib.remove_lock(process.argv[2] == null || process.argv[2] == '' ? 'index' : process.argv[2]); + } + + const { spawn } = require('child_process'); + + // reload the sync process + const child = spawn(process.execPath, args, { + stdio: 'inherit' + }); + + // when the child process ends, exit this parent process with the same code + child.on('exit', code => { + process.exit(code ?? 1); + }); } }; \ No newline at end of file diff --git a/lib/explorer.js b/lib/explorer.js index df6838b..8dbccdf 100644 --- a/lib/explorer.js +++ b/lib/explorer.js @@ -1117,51 +1117,61 @@ module.exports = { var arr_vin = vin; var tx_type = null; - module.exports.syncLoop(vout.length, function (loop) { - var i = loop.iteration(); - // make sure vout has an address - if (vout[i].scriptPubKey.type != 'nonstandard' && vout[i].scriptPubKey.type != 'nulldata') { - // check if this is a zerocoin tx - if (vout[i].scriptPubKey.type != 'zerocoinmint') { - var address_list = vout[i].scriptPubKey.addresses; - // check if there are one or more addresses in the vout - if (address_list == null || address_list.length == 0) { - // no addresses defined - // check if there is a single address defined - if (vout[i].scriptPubKey.address == null) { - // no single address defined - // check if bitcoin features are enabled - if (settings.blockchain_specific.bitcoin.enabled == true) { - // assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address - encodeP2PKaddress(vout[i].scriptPubKey.asm, function(p2pkh_address) { - // check if the address was encoded properly - if (p2pkh_address != null) { - // mark this tx as p2pk - tx_type = 'p2pk'; - // process vout addresses - processVoutAddresses(p2pkh_address, vout[i].value, arr_vout, function(vout_array) { - // save updated array - arr_vout = vout_array; - // move to next vout - loop.next(); - }); - } else { - // could not decipher the address, save as unknown and move to next vout - console.log('Failed to find vout address from tx ' + txid); - // process vout addresses - processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) { - // save updated array - arr_vout = vout_array; - // move to next vout - loop.next(); - }); - } - }); + try { + module.exports.syncLoop(vout.length, function (loop) { + var i = loop.iteration(); + // make sure vout has an address + if (vout[i].scriptPubKey.type != 'nonstandard' && vout[i].scriptPubKey.type != 'nulldata') { + // check if this is a zerocoin tx + if (vout[i].scriptPubKey.type != 'zerocoinmint') { + var address_list = vout[i].scriptPubKey.addresses; + // check if there are one or more addresses in the vout + if (address_list == null || address_list.length == 0) { + // no addresses defined + // check if there is a single address defined + if (vout[i].scriptPubKey.address == null) { + // no single address defined + // check if bitcoin features are enabled + if (settings.blockchain_specific.bitcoin.enabled == true) { + // assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address + encodeP2PKaddress(vout[i].scriptPubKey.asm, function(p2pkh_address) { + // check if the address was encoded properly + if (p2pkh_address != null) { + // mark this tx as p2pk + tx_type = 'p2pk'; + // process vout addresses + processVoutAddresses(p2pkh_address, vout[i].value, arr_vout, function(vout_array) { + // save updated array + arr_vout = vout_array; + // move to next vout + loop.next(); + }); + } else { + // could not decipher the address, save as unknown and move to next vout + console.log('Failed to find vout address from tx ' + txid); + // process vout addresses + processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) { + // save updated array + arr_vout = vout_array; + // move to next vout + loop.next(); + }); + } + }); + } else { + // could not decipher the address, save as unknown and move to next vout + console.log('Failed to find vout address from tx ' + txid); + // process vout addresses + processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) { + // save updated array + arr_vout = vout_array; + // move to next vout + loop.next(); + }); + } } else { - // could not decipher the address, save as unknown and move to next vout - console.log('Failed to find vout address from tx ' + txid); - // process vout addresses - processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) { + // process vout address + processVoutAddresses([vout[i].scriptPubKey.address], vout[i].value, arr_vout, function(vout_array) { // save updated array arr_vout = vout_array; // move to next vout @@ -1169,8 +1179,8 @@ module.exports = { }); } } else { - // process vout address - processVoutAddresses([vout[i].scriptPubKey.address], vout[i].value, arr_vout, function(vout_array) { + // process vout addresses + processVoutAddresses(address_list, vout[i].value, arr_vout, function(vout_array) { // save updated array arr_vout = vout_array; // move to next vout @@ -1178,51 +1188,59 @@ module.exports = { }); } } else { - // process vout addresses - processVoutAddresses(address_list, vout[i].value, arr_vout, function(vout_array) { - // save updated array - arr_vout = vout_array; - // move to next vout - loop.next(); - }); + // TODO: add support for zerocoin transactions + console.log('Zerocoin tx found. skipping for now as it is unsupported'); + tx_type = "zerocoin"; + loop.next(); } } else { - // TODO: add support for zerocoin transactions - console.log('Zerocoin tx found. skipping for now as it is unsupported'); - tx_type = "zerocoin"; + // no address, move to next vout loop.next(); } - } else { - // no address, move to next vout - loop.next(); - } - }, function() { - // check if zksnarks is enabled - if (settings.blockchain_specific.zksnarks.enabled == true) { - // check for hidden/anonymous outputs - if (vhidden != null && vhidden.length > 0) { - tx_type = "zksnarks"; - // loop through all hidden/anonymous outputs - module.exports.syncLoop(vhidden.length, function (loop) { - var i = loop.iteration(); + }, function() { + // check if zksnarks is enabled + if (settings.blockchain_specific.zksnarks.enabled == true) { + // check for hidden/anonymous outputs + if (vhidden != null && vhidden.length > 0) { + tx_type = "zksnarks"; + // loop through all hidden/anonymous outputs + module.exports.syncLoop(vhidden.length, function (loop) { + var i = loop.iteration(); - if (vhidden[i].vpub_old > 0) { - // process vout addresses - processVoutAddresses(['hidden_address'], parseFloat(vhidden[i].vpub_old), arr_vout, function(vout_array) { - // save updated array - arr_vout = vout_array; - // move to next vout - loop.next(); - }); - } else { - if ((!vout || vout.length == 0) && (!vin || vin.length == 0)) { - // hidden sender is sending to hidden recipient - // the sent and received values are not known in this case. only the fee paid is known and subtracted from the sender. + if (vhidden[i].vpub_old > 0) { // process vout addresses - processVoutAddresses(['hidden_address'], 0, arr_vout, function(vout_array) { + processVoutAddresses(['hidden_address'], parseFloat(vhidden[i].vpub_old), arr_vout, function(vout_array) { // save updated array arr_vout = vout_array; - // add a private send address with the known amount sent + // move to next vout + loop.next(); + }); + } else { + if ((!vout || vout.length == 0) && (!vin || vin.length == 0)) { + // hidden sender is sending to hidden recipient + // the sent and received values are not known in this case. only the fee paid is known and subtracted from the sender. + // process vout addresses + processVoutAddresses(['hidden_address'], 0, arr_vout, function(vout_array) { + // save updated array + arr_vout = vout_array; + // add a private send address with the known amount sent + module.exports.is_unique(arr_vin, 'hidden_address', 'addresses', function(unique, index) { + if (unique == true) { + module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) { + arr_vin.push({addresses: 'hidden_address', amount: amount_sat}); + // move to next vout + loop.next(); + }); + } else { + module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) { + arr_vin[index].amount = arr_vin[index].amount + amount_sat; + // move to next vout + loop.next(); + }); + } + }); + }); + } else { module.exports.is_unique(arr_vin, 'hidden_address', 'addresses', function(unique, index) { if (unique == true) { module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) { @@ -1238,64 +1256,57 @@ module.exports = { }); } }); + } + } + }); + } + } + + if (typeof vout[0] !== 'undefined' && vout[0].scriptPubKey.type == 'nonstandard') { + if (arr_vin.length > 0 && arr_vout.length > 0) { + if (arr_vin[0].addresses == arr_vout[0].addresses) { + //PoS + arr_vout[0].amount = arr_vout[0].amount - arr_vin[0].amount; + arr_vin.shift(); + + // check if any vin remains + if (arr_vin == null || arr_vin.length == 0) { + // empty vin should be linked to coinbase + arr_vin = [{coinbase: "coinbase"}]; + + var new_vout = []; + + // loop through the arr_vout to create a copy of the data with coin amounts only for use with prepare_vin() + for (i = 0; i < arr_vout.length; i++) { + new_vout.push({ + value: arr_vout[i].amount / 100000000 + }); + } + + // call the prepare_vin again to populate the vin data correctly + module.exports.prepare_vin({txid: txid, vin: arr_vin, vout: new_vout}, function(return_vin, return_tx_type_vin) { + return cb(arr_vout, return_vin, return_tx_type_vin); }); } else { - module.exports.is_unique(arr_vin, 'hidden_address', 'addresses', function(unique, index) { - if (unique == true) { - module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) { - arr_vin.push({addresses: 'hidden_address', amount: amount_sat}); - // move to next vout - loop.next(); - }); - } else { - module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) { - arr_vin[index].amount = arr_vin[index].amount + amount_sat; - // move to next vout - loop.next(); - }); - } - }); + return cb(arr_vout, arr_vin, tx_type); } - } - }); - } - } - - if (typeof vout[0] !== 'undefined' && vout[0].scriptPubKey.type == 'nonstandard') { - if (arr_vin.length > 0 && arr_vout.length > 0) { - if (arr_vin[0].addresses == arr_vout[0].addresses) { - //PoS - arr_vout[0].amount = arr_vout[0].amount - arr_vin[0].amount; - arr_vin.shift(); - - // check if any vin remains - if (arr_vin == null || arr_vin.length == 0) { - // empty vin should be linked to coinbase - arr_vin = [{coinbase: "coinbase"}]; - - var new_vout = []; - - // loop through the arr_vout to create a copy of the data with coin amounts only for use with prepare_vin() - for (i = 0; i < arr_vout.length; i++) { - new_vout.push({ - value: arr_vout[i].amount / 100000000 - }); - } - - // call the prepare_vin again to populate the vin data correctly - module.exports.prepare_vin({txid: txid, vin: arr_vin, vout: new_vout}, function(return_vin, return_tx_type_vin) { - return cb(arr_vout, return_vin, return_tx_type_vin); - }); - } else { + } else return cb(arr_vout, arr_vin, tx_type); - } } else return cb(arr_vout, arr_vin, tx_type); } else return cb(arr_vout, arr_vin, tx_type); - } else - return cb(arr_vout, arr_vin, tx_type); - }); + }); + } catch(err) { + // check if a "Maximum call stack size exceeded" error occurred + if (err instanceof RangeError && /Maximum call stack size exceeded/i.test(err.message)) { + // return invalid results with error msg + return cb(null, null, 'StackSizeError'); + } else { + // any other error should be output normally + throw err; + } + } }, get_input_addresses: function(input, vout, cb) { diff --git a/lib/settings.js b/lib/settings.js index fbfdaec..aecef5e 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -1411,7 +1411,11 @@ exports.sync = { // HEAVY: retrieved from getsupply rpc cmd (The "blockchain_specific.heavycoin.enabled" setting must be set to true and the "blockchain_specific.heavycoin.api_cmds.getsupply" setting must be set up correctly for this option to work properly) // BALANCES : get the supply by running a query on the addresses collection and summing up all positive balances (potentially a long running query for blockchains with tons of addresses) // TXOUTSET : retrieved from gettxoutsetinfo rpc cmd - "supply": "GETINFO" + "supply": "GETINFO", + // elastic_stack_size: If a "RangeError: Maximum call stack size exceeded" error occurs during a block sync (which can happen when dealing with large transactions with many addresses), the sync script will automatically be reloaded using a larger stack size value which increases memory usage based on this value. + // NOTE: If the first reload of the sync script still doesn't have enough memory to handle processing of a large transaction, the sync is smart enough to continue increasing the stack size by this value again and again until it finishes processing all blocks and then returns back to the default amount of memory for future blocks. + // It is recommended to leave this value alone unless you know what you are doing. + "elastic_stack_size": 4096 }; // captcha: a collection of settings that pertain to the captcha security used by different elements of the explorer diff --git a/scripts/benchmark.js b/scripts/benchmark.js index acc01d0..a267a60 100644 --- a/scripts/benchmark.js +++ b/scripts/benchmark.js @@ -1,18 +1,25 @@ const mongoose = require('mongoose'); +const Address = require('../models/address'); +const Tx = require('../models/tx'); const blkSync = require('../lib/block_sync'); const settings = require('../lib/settings'); +const resumeSync = process.argv[2] == '1'; let dbString = `mongodb://${settings.benchmark.address}:${settings.benchmark.port}/admin` // prevent stopping of the sync script to be able to gracefully shut down process.on('SIGINT', () => { - console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + if (!blkSync.getStackSizeErrorId()) + console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + blkSync.setStopSync(true); }); // prevent killing of the sync script to be able to gracefully shut down process.on('SIGTERM', () => { - console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + if (!blkSync.getStackSizeErrorId()) + console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + blkSync.setStopSync(true); }); @@ -89,6 +96,74 @@ function check_create_user(cb) { return cb(); } +function initialize_data_startup(cb) { + console.log(`${settings.localization.initializing_database}.. ${settings.localization.please_wait}..`); + + const db = require('../lib/database'); + + // check if stats collection is initialized + db.check_stats(settings.coin.name, function(stats_exists) { + let skip = true; + + // determine if stats collection already exists + if (stats_exists == false) { + console.log(`${settings.localization.creating_initial_entry.replace('{1}', 'stats')}.. ${settings.localization.please_wait}..`); + skip = false; + } + + // initialize the stats collection + db.create_stats(settings.coin.name, skip, function() { + // get the stats object from the database + db.get_stats(settings.coin.name, function(stats) { + // finished initializing startup data + console.log('Database initialization complete'); + return cb(stats); + }); + }); + }); +} + +function delete_txes(cb) { + // check if the benchmark sync is being resumed + if (resumeSync) { + // do not delete the list of txes for a resume sync + return cb(); + } else { + // delete all previous transaction records from the benchmark database + Tx.deleteMany({}).then(() => { + return cb(); + }); + } +} + +function delete_addresses(cb) { + // check if the benchmark sync is being resumed + if (resumeSync) { + // do not delete the list of addresses for a resume sync + return cb(); + } else { + // delete all previous address records from the benchmark database + Address.deleteMany({}).then(() => { + return cb(); + }); + } +} + +function delete_stats(cb) { + // check if the benchmark sync is being resumed + if (resumeSync) { + // do not delete the database stats for a resume sync + return cb(); + } else { + const Stats = require('../models/stats'); + + // delete all previous stat records from the benchmark database + Stats.deleteMany({}).then(() => { + return cb(); + }); + } +} + console.log(`${settings.localization.script_launched}: ${process.pid}`); mongoose.set('strictQuery', true); @@ -104,37 +179,49 @@ check_create_user(function() { // connect to the benchmark database mongoose.connect(dbString).then(() => { - const Tx = require('../models/tx'); - // delete all previous transaction records from the benchmark database - Tx.deleteMany({}).then(() => { - const Address = require('../models/address'); - + delete_txes(function() { // delete all previous address records from the benchmark database - Address.deleteMany({}).then(() => { - // get starting timestamp - const s_timer = new Date().getTime(); + delete_addresses(function() { + // delete all previous stat records from the benchmark database + delete_stats(function() { + // initialize the benchmark database + initialize_data_startup(function(stats) { + // get the last synced block index value + const last = (stats.last ? stats.last : 0); - // start the block sync - blkSync.update_tx_db(settings.coin.name, 1, settings.benchmark.block_to_sync, 0, settings.sync.update_timeout, false, function() { - // get ending timestamp - const e_timer = new Date().getTime(); + // get starting timestamp + const s_timer = new Date().getTime(); - // get count of transactions - Tx.countDocuments({}).then((txcount) => { - // get count of addresses - Address.countDocuments({}).then((acount) => { - // check if the script stopped prematurely - if (blkSync.getStopSync()) - console.log('Block sync was stopped prematurely'); + // start the block sync + blkSync.update_tx_db(settings.coin.name, last, settings.benchmark.block_to_sync, stats.txes, settings.sync.update_timeout, false, function() { + // get ending timestamp + const e_timer = new Date().getTime(); - // output final benchmark stats - console.log({ - tx_count: txcount, - address_count: acount, - seconds: (e_timer - s_timer) / 1000, + // get count of transactions + Tx.countDocuments({}).then((txcount) => { + // get count of addresses + Address.countDocuments({}).then((acount) => { + // check if the script stopped prematurely + if (blkSync.getStopSync()) + console.log('Block sync was stopped prematurely'); + + // output final benchmark stats + console.log({ + tx_count: txcount, + address_count: acount, + seconds: (e_timer - s_timer) / 1000, + }); + + // check if the sync needed to be resumed + if (resumeSync) { + // output a warning msg + console.log(`\n${settings.localization.ex_warning}: The sync ran out of memory during processing and therefore the run time was affected. It is recommended to re-run the benchmark again using a larger stack size such as 25000 or higher with the cmd "node --stack-size=25000 scripts/benchmark.js" to help ensure an accurate benchmark time.`); + } + + exit(0); + }); }); - exit(0); }); }); }); diff --git a/scripts/sync.js b/scripts/sync.js index b975463..ab219fa 100644 --- a/scripts/sync.js +++ b/scripts/sync.js @@ -18,14 +18,18 @@ var stopSync = false; // prevent stopping of the sync script to be able to gracefully shut down process.on('SIGINT', () => { - console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + if (!blkSync.getStackSizeErrorId()) + console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + blkSync.setStopSync(true); stopSync = true; }); // prevent killing of the sync script to be able to gracefully shut down process.on('SIGTERM', () => { - console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + if (!blkSync.getStackSizeErrorId()) + console.log(`${settings.localization.stopping_sync_process}.. ${settings.localization.please_wait}..`); + blkSync.setStopSync(true); stopSync = true; }); @@ -202,6 +206,12 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout, tx_count = updated_tx_count2; setTimeout(function() { + // check if there was a memory error + if (blkSync.getStackSizeErrorId() != null) { + // stop the loop + tx_loop.break(true); + } + // move to the next tx record tx_loop.next(); }, timeout); @@ -209,6 +219,12 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout, }); }, function() { setTimeout(function() { + // check if there was a memory error + if (blkSync.getStackSizeErrorId() != null) { + // stop the loop + block_loop.break(true); + } + // move to the next block record block_loop.next(); }, timeout); @@ -216,49 +232,57 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout, }); }); }, function() { - // get the most recent stats - Stats.findOne({coin: settings.coin.name}).then((stats) => { - // add missing txes for the current block - blkSync.update_tx_db(settings.coin.name, current_block, current_block, (stats.txes + tx_count), timeout, 2, function(updated_tx_count) { - // update the stats collection by removing the orphaned txes in this block from the tx count - // and setting the orphan_index and orphan_current values in case the sync is interrupted before finishing - Stats.updateOne({coin: settings.coin.name}, { - orphan_index: current_block, - orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0]) - }).then(() => { - // clear the saved block hash data - correct_block_data = null; + // check if there was a memory error + if (!blkSync.getStackSizeErrorId()) { + // get the most recent stats + Stats.findOne({coin: settings.coin.name}).then((stats) => { + // add missing txes for the current block + blkSync.update_tx_db(settings.coin.name, current_block, current_block, (stats.txes + tx_count), timeout, 2, function(updated_tx_count) { + // update the stats collection by removing the orphaned txes in this block from the tx count + // and setting the orphan_index and orphan_current values in case the sync is interrupted before finishing + Stats.updateOne({coin: settings.coin.name}, { + orphan_index: current_block, + orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0]) + }).then(() => { + // clear the saved block hash data + correct_block_data = null; - // move to the next block - current_block++; + // move to the next block + current_block++; - setTimeout(function() { - // process next block - next(null); - }, timeout); - }).catch((err) => { - console.log(err); + setTimeout(function() { + // process next block + next(null); + }, timeout); + }).catch((err) => { + console.log(err); - // clear the saved block hash data - correct_block_data = null; + // clear the saved block hash data + correct_block_data = null; - // move to the next block - current_block++; + // move to the next block + current_block++; - setTimeout(function() { - // process next block - next(null); - }, timeout); + setTimeout(function() { + // process next block + next(null); + }, timeout); + }); }); - }); - }).catch((err) => { - console.log(err); + }).catch((err) => { + console.log(err); + setTimeout(function() { + // process next block + next(null); + }, timeout); + }); + } else { setTimeout(function() { - // process next block - next(null); + // stop the loop + next('StackSizeError'); }, timeout); - }); + } }); }); } else { @@ -283,11 +307,17 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout, function(err) { // check if there is a msg to display if (err != '' && err != 'stop') { - // display the msg - console.log(err); + // check if this is the StackSizeError error + if (err == 'StackSizeError') { + // reload the sync process + blkSync.respawnSync(); + } else { + // display the msg + console.log(err); - // stop fixing orphaned block data - return cb(); + // stop fixing orphaned block data + return cb(); + } } else { // check if the script is stopping if (!stopSync) @@ -514,9 +544,14 @@ function check_add_tx(txid, blockhash, tx_count, cb) { // save the tx to the local database blkSync.save_tx(txid, block.height, block, function(save_tx_err, tx_has_vout) { // check if there were any save errors - if (save_tx_err) - console.log(save_tx_err); - else + if (save_tx_err) { + // check the error code + if (save_tx_err.code == 'StackSizeError') { + // ensure the process halts after stopping all sync threads + blkSync.setStackSizeErrorId(txid); + } else + console.log(save_tx_err); + } else console.log('%s: %s', block.height, txid); // check if the tx was saved correctly diff --git a/settings.json.template b/settings.json.template index 71b9cf1..29993be 100644 --- a/settings.json.template +++ b/settings.json.template @@ -1498,7 +1498,11 @@ // HEAVY: retrieved from getsupply rpc cmd (The "blockchain_specific.heavycoin.enabled" setting must be set to true and the "blockchain_specific.heavycoin.api_cmds.getsupply" setting must be set up correctly for this option to work properly) // BALANCES : get the supply by running a query on the addresses collection and summing up all positive balances (potentially a long running query for blockchains with tons of addresses) // TXOUTSET : retrieved from gettxoutsetinfo rpc cmd - "supply": "GETINFO" + "supply": "GETINFO", + // elastic_stack_size: If a "RangeError: Maximum call stack size exceeded" error occurs during a block sync (which can happen when dealing with large transactions with many addresses), the sync script will automatically be reloaded using a larger stack size value which increases memory usage based on this value. + // NOTE: If the first reload of the sync script still doesn't have enough memory to handle processing of a large transaction, the sync is smart enough to continue increasing the stack size by this value again and again until it finishes processing all blocks and then returns back to the default amount of memory for future blocks. + // It is recommended to leave this value alone unless you know what you are doing. + "elastic_stack_size": 4096 }, // captcha: a collection of settings that pertain to the captcha security used by different elements of the explorer