Improved block sync speed

-A number of functions have been rewritten to be more optimized and faster: calculate_total, is_unique, convert_to_satoshi, get_input_addresses, processVoutAddresses, prepare_vout, prepare_vin
-Txes are now written to database via bulk writes which helps improve the sync speed and also controls memory usage with batching to write data once a certain threshold is reached
-update_address function changed to update_addresses since it now bulk writes the addresses in batches to improve sync speed and also controls memory usage with batching to write data once a certain threshold is reached
-The syncLoop function has been completely removed from the project and replaced with async library loops or even normal "for" loops in some cases which greatly improves sync speeds over large batches of data
-Fixed an issue with the flattened count of txes that is saved to the coinstats collection which could save incorrectly when using more than 1 thread
-Fixed an issue with the block sync which caused an unwanted delay when syncing less blocks than the amount of threads used to sync the data
-Fixed an issue with vout data processing that could sometimes populate data out of order
-Added a new sync.batch_size setting used to determine how many records (txes, addresses, addresstxes) should be saved in a single database transaction
-Added a new wait_for_bulk_database_save setting used to increase the block sync speed at the cost of not returning any error msgs for data that failed to save
-get_input_addresses function no longer returns in the exports section of the explorer.js file since it is only referenced in that file
-Updated explorerspec tests to use the newest function changes for any tests that needed to be updated

Special thanks to Karzo from Pepecoin for help with the bulkwrite code changes!

README.md

@@ -967,6 +967,7 @@ You can support us via one of the following options:
 - **[Alan Rudolf (aka suprnurd)](https://github.com/suprnurd):** for the custom changes found in the [Ciquidus explorer](https://github.com/suprnurd/ciquidus)
 - **[Tim Garrity (aka uaktags)](https://github.com/uaktags):** for his many contributions to the Iquidus explorer and custom features from the [uaktags explorer](https://github.com/uaktags/explorer)
 - **[TheHolyRoger](https://github.com/TheHolyRoger):** for his continued work and contributions to the Iquidus explorer
+- **[Karzo](https://github.com/KarzoGitHub):** for helping with the bulkwrite block sync code changes
 - All the rest of the Iquidus contributors who helped shape the Iquidus explorer in some way
 
 ### License

app.js

@@ -501,7 +501,8 @@ app.use('/ext/gettx/:txid', function(req, res) {
           if (rtx && rtx.txid) {
             lib.prepare_vin(rtx, function(vin, tx_type_vin) {
               lib.prepare_vout(rtx.vout, rtx.txid, vin, ((typeof rtx.vjoinsplit === 'undefined' || rtx.vjoinsplit == null) ? [] : rtx.vjoinsplit), function(rvout, rvin, tx_type_vout) {
-                lib.calculate_total(rvout, function(total) {
+                const total = lib.calculate_total(rvout);
+
                 if (!rtx.confirmations > 0) {
                   var utx = {
                     txid: rtx.txid,
@@ -531,7 +532,6 @@ app.use('/ext/gettx/:txid', function(req, res) {
                 }
               });
             });
-            });
           } else
             res.send({ error: 'tx not found.', hash: txid});
         });

lib/block_sync.js

@@ -8,17 +8,17 @@ const async = require('async');
 let stopSync = false;
 let stackSizeErrorId = null;
 
-function check_delete_tx(tx, block_height, tx_count, timeout, cb) {
+function check_delete_tx(tx, block_height, timeout, cb) {
   // check if the tx object exists and does not match the current block height
   if (tx && tx.blockindex != block_height) {
     // the transaction exists but does not match the correct block height, therefore it should be deleted
-    module.exports.delete_and_cleanup_tx(tx.txid, tx.blockindex, tx_count, timeout, function(updated_tx_count) {
+    module.exports.delete_and_cleanup_tx(tx.txid, tx.blockindex, timeout, function(updated_tx_count) {
       // finished removing the transaction
       return cb(updated_tx_count, true);
     });
   } else {
     // tx dosn't exist or block heights match so nothing to do
-    return cb(tx_count, false);
+    return cb(0, false);
   }
 }
 
@@ -71,50 +71,112 @@ function hex_to_ascii(hex) {
   return str;
 }
 
-function update_address(hash, blockheight, txid, amount, type, cb) {
-  let addr_inc = {}
-
-  if (hash == 'coinbase')
-    addr_inc.sent = amount;
-  else {
-    if (type == 'vin') {
-      addr_inc.sent = amount;
-      addr_inc.balance = -amount;
-    } else {
-      addr_inc.received = amount;
-      addr_inc.balance = amount;
-    }
-  }
-
-  Address.findOneAndUpdate({a_id: hash}, {
-    $inc: addr_inc
-  }, {
-    new: true,
-    upsert: true
-  }).then((address) => {
-    if (hash != 'coinbase') {
-      AddressTx.findOneAndUpdate({a_id: hash, txid: txid}, {
-        $inc: {
-          amount: addr_inc.balance
+function update_addresses(addresses, blockheight, txid, cb) {
+  const bulkAddresses = addresses.map((address) => {
+    return {
+      updateOne: {
+        filter: { a_id: address.hash },
+        update: {
+          $setOnInsert: { a_id: address.hash },
+          $inc: { sent: address.sent, balance: address.balance, received: address.received },
         },
-        $set: {
-          a_id: hash,
-          blockindex: blockheight,
-          txid: txid
-        }
-      }, {
-        new: true,
         upsert: true
-      }).then((addresstx) => {
-        return cb();
-      }).catch((err) => {
-        return cb(err);
+      }
+    };
   });
-    } else
-      return cb();
-  }).catch((err) => {
-    return cb(err);
+
+  const bulkAddressTxes = addresses
+    .filter((address) => address.hash !== 'coinbase')
+    .map((address) => {
+      return {
+        updateOne: {
+          filter: { a_id: address.hash, txid: txid },
+          update: {
+            $setOnInsert: { a_id: address.hash, blockindex: blockheight, txid: txid },
+            $inc: { amount: (address.type == 'vin' ? -address.amount : address.amount) }
+          },
+          upsert: true
+        }
+      };
     });
+
+  try {
+    let completed = 0;
+    let errorCalled = false;
+
+    // callback to run when one of the 2 table writes is complete
+    function onDone(err) {
+      // check if the previous process already returned an error
+      if (errorCalled) {
+        // stop if there was already an error
+        return;
+      } else {
+        // check if there was an error
+        if (err) {
+          // ensure the next data set knows there was already an error
+          errorCalled = true;
+
+          // return the error
+          return cb(err);
+        }
+
+        // increment the completed counter
+        completed += 1;
+
+        // check if both data sets completed
+        if (completed === 2) {
+          // finished updating address data
+          return cb();
+        }
+      }
+    }
+
+    // start processing both sets of data in parallel
+    processInBatches(Address, bulkAddresses, onDone);
+    processInBatches(AddressTx, bulkAddressTxes, onDone);
+  } catch (err) {
+    return cb(err);
+  }
+}
+
+function processInBatches(collection, data, cb) {
+  const batch_size = settings.sync.batch_size;
+  let index = 0;
+
+  function processNextBatch() {
+    // check if all records were saved
+    if (index >= data.length) {
+      // all records were saved
+      return cb();
+    }
+
+    // get the next batch of data
+    const batch = data.slice(index, index + batch_size);
+
+    // increment the index by the batch size
+    index += batch_size;
+
+    try {
+      // asynchronously write data to the collection database
+      collection.bulkWrite(batch, { ordered: false, writeConcern: { w: (settings.sync.wait_for_bulk_database_save ? 1 : 0) } }).then((result) => {
+        // process the next batch of records
+        processNextBatch();
+      }).catch((err) => {
+        console.log(err);
+
+        // process the next batch of records
+        processNextBatch();
+      });
+    } catch(err) {
+      console.log(err);
+
+      // process the next batch of records
+      processNextBatch();
+    }
+  }
+
+  // start processing records
+  processNextBatch();
 }
 
 function finalize_update_tx_db(coin, check_only, end, txes, cb) {
@@ -143,49 +205,103 @@ function finalize_update_tx_db(coin, check_only, end, txes, cb) {
 
 module.exports = {
   save_tx: function(txid, blockheight, block, cb) {
+    try {
       lib.get_rawtransaction(txid, function(tx) {
         if (tx && tx != `${settings.localization.ex_error}: ${settings.localization.check_console}`) {
           lib.prepare_vin(tx, function(vin, tx_type_vin) {
             lib.prepare_vout(tx.vout, txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) {
               // check if vout is null which indicates an error
               if (vout != null) {
-              lib.syncLoop(nvin.length, function (loop) {
-                const i = loop.iteration();
+                let addressBatch = [];
 
+                // add all vin addresses to the batch array
+                nvin.forEach(function(input) {
                   // check if address is inside an array
-                if (Array.isArray(nvin[i].addresses)) {
+                  if (Array.isArray(input.addresses)) {
                     // extract the address
-                  nvin[i].addresses = nvin[i].addresses[0];
+                    input.addresses = input.addresses[0];
                   }
 
-                update_address(nvin[i].addresses, blockheight, txid, nvin[i].amount, 'vin', function() {
-                  loop.next();
-                });
-              }, function() {
-                lib.syncLoop(vout.length, function (subloop) {
-                  const t = subloop.iteration();
+                  // check if a vin address exists
+                  if (input.addresses) {
+                    const index = lib.is_unique(addressBatch, input.addresses, 'hash');
+                    let sent = 0;
+                    let balance = 0;
 
-                  // check if address is inside an array
-                  if (Array.isArray(vout[t].addresses)) {
-                    // extract the address
-                    vout[t].addresses = vout[t].addresses[0];
+                    if (input.addresses == 'coinbase')
+                      sent = input.amount;
+                    else {
+                      sent = input.amount;
+                      balance = -input.amount;
                     }
 
-                  if (vout[t].addresses) {
-                    update_address(vout[t].addresses, blockheight, txid, vout[t].amount, 'vout', function() {
-                      subloop.next();
+                    // check if the address already exists in the array
+                    if (index == -1) {
+                      // unique address
+                      addressBatch.push({
+                        hash: input.addresses,
+                        amount: input.amount,
+                        type: 'vin',
+                        sent: sent,
+                        balance: balance,
+                        received: 0
                       });
-                  } else
-                    subloop.next();
-                }, function() {
-                  lib.calculate_total(vout, function(total) {
-                    var op_return = null;
-                    var algo = null;
+                    } else {
+                      // address already exists
+                      addressBatch[index].amount += input.amount;
+                      addressBatch[index].sent += sent;
+                      addressBatch[index].balance += balance;
+                    }
+                  }
+                });
+
+                // add all vout addresses to the batch array
+                vout.forEach(function(output) {
+                  // check if address is inside an array
+                  if (Array.isArray(output.addresses)) {
+                    // extract the address
+                    output.addresses = output.addresses[0];
+                  }
+
+                  // check if a vout address exists
+                  if (output.addresses) {
+                    const index = lib.is_unique(addressBatch, output.addresses, 'hash');
+                    const balance = output.amount;
+                    const received = output.amount;
+
+                    // check if the address already exists in the array
+                    if (index == -1) {
+                      // unique address
+                      addressBatch.push({
+                        hash: output.addresses,
+                        amount: output.amount,
+                        type: 'vout',
+                        sent: 0,
+                        balance: balance,
+                        received: received
+                      });
+                    } else {
+                      // address already exists
+                      addressBatch[index].amount -= output.amount;
+                      addressBatch[index].balance += balance;
+                      addressBatch[index].received += received;
+                    }
+                  }
+                });
+
+                // save the addresses to the database
+                update_addresses(addressBatch, blockheight, txid, function(err) {
+                  if (err)
+                    return cb(err, false, null);
+                  else {
+                    const total = lib.calculate_total(vout);
+                    let op_return = null;
+                    let algo = null;
 
                     // check if the op_return value should be decoded and saved
                     if (settings.transaction_page.show_op_return) {
                       // loop through vout to find the op_return value
-                      tx.vout.forEach(function (vout_data) {
+                      tx.vout.forEach(function(vout_data) {
                         // check if the op_return value exists
                         if (vout_data.scriptPubKey != null && vout_data.scriptPubKey.asm != null && vout_data.scriptPubKey.asm.indexOf('OP_RETURN') > -1) {
                           // decode the op_return value
@@ -200,7 +316,8 @@ module.exports = {
                       algo = block[settings.block_page.multi_algorithm.key_name];
                     }
 
-                    const newTx = new Tx({
+                    // return the transaction data
+                    return cb(null, vout.length > 0, new Tx({
                       txid: tx.txid,
                       vin: (vin == null || vin.length == 0 ? [] : nvin),
                       vout: vout,
@@ -211,30 +328,27 @@ module.exports = {
                       tx_type: (tx_type_vout == null ? tx_type_vin : tx_type_vout),
                       op_return: op_return,
                       algo: algo
-                    });
-
-                    newTx.save().then(() => {
-                      return cb(null, vout.length > 0);
-                    }).catch((err) => {
-                      return cb(err, false);
-                    });
-                  });
-                });
+                    }));
+                  }
                 });
               } else {
-              // create a custom error that will be specifically checked for later (NOTE: tx_type_vout contains the error code in this special case)
+                // create a custom error that will be specifically checked for later
+                // NOTE: tx_type_vout contains the error code in this special case
                 const customError = new Error(tx_type_vout);
 
                 customError.code = tx_type_vout;
 
                 // return the custom error
-              return cb(customError, false);
+                return cb(customError, false, null);
               }
             });
           });
         } else
-        return cb('tx not found: ' + txid, false);
+          return cb('tx not found: ' + txid, false, null);
       });
+    } catch(err) {
+      return cb(`Error querying database for txid ${txid}: ${err}`, false, null);
+    }
   },
 
   // updates tx & address balances
@@ -329,6 +443,84 @@ module.exports = {
                 lib.get_block(blockhash, function(block) {
                   if (block) {
                     let tx_counter = 0;
+                    let txBatch = [];
+
+                    // create a queue for batching txes for this block
+                    const txBatchQueue = async.queue(function(tx, done) {
+                      // add the tx to an array
+                      txBatch.push(tx);
+
+                      // check if the batch of txes should be saved
+                      if (txBatch.length >= settings.sync.batch_size) {
+                        // save the current batch of txes to the database now
+                        flushTxBatch(done);
+                      } else {
+                        // continue without saving txes
+                        done();
+                      }
+                    }, 1);
+
+                    // add a function used to bulkWrite txes to the database
+                    function flushTxBatch(txBatchCallback) {
+                      // copy current batch of txes to a local variable
+                      const localTxBatch = txBatch;
+
+                      // clear the global array of batched txes
+                      txBatch = [];
+
+                      // check if there are actually any txes to save
+                      if (!localTxBatch || localTxBatch.length === 0) {
+                        // no txes to save
+                        return txBatchCallback();
+                      } else {
+                        // get the transaction batch ready to bulk update
+                        const bulkTxes = localTxBatch.map(tx => {
+                          // convert tx to plain JS object
+                          const plainTx = tx.toObject();
+
+                          // remove the _id field to prevent issues with some blockchains that can reuse non-standard txids
+                          delete plainTx._id;
+
+                          return {
+                            updateOne: {
+                              filter: { txid: plainTx.txid },
+                              update: [
+                                {
+                                  $replaceWith: {
+                                    $cond: {
+                                      if: { $gt: [ plainTx.blockindex, { $ifNull: ["$blockindex", -1] } ] },
+                                      then: {
+                                        $mergeObjects: [
+                                          // if a doc exists, keep that _id
+                                          { _id: "$_id" },
+                                          // overwrite everything else with plainTx
+                                          plainTx
+                                        ]
+                                      },
+                                      else: "$$ROOT"
+                                    }
+                                  }
+                                }
+                              ],
+                              upsert: true
+                            }
+                          };
+                        });
+
+                        try {
+                          // write the transactions to the database
+                          Tx.bulkWrite(bulkTxes, { ordered: false, writeConcern: { w: (settings.sync.wait_for_bulk_database_save ? 1 : 0) } }).then(() => {
+                            return txBatchCallback();
+                          }).catch((err) => {
+                            console.log(err);
+                            return txBatchCallback();
+                          });
+                        } catch(err) {
+                          console.log(err);
+                          return txBatchCallback();
+                        }
+                      }
+                    }
 
                     // loop through all txes in this block
                     async.eachLimit(block.tx, parallel_tasks, function(txid, next_tx) {
@@ -350,30 +542,21 @@ module.exports = {
                           }, timeout);
                         } else {
                           // check if the transaction exists but doesn't match the current block height
-                          check_delete_tx(tx, block_height, txes, timeout, function(updated_txes, tx_deleted) {
+                          check_delete_tx(tx, block_height, timeout, function(updated_txes, tx_deleted) {
                             // update the running tx count
-                            txes = updated_txes;
+                            txes += updated_txes;
 
                             // check if this tx should be added to the local database
                             if (tx_deleted || !tx) {
                               // save the transaction to local database
-                              module.exports.save_tx(txid, block_height, block, function(err, tx_has_vout) {
+                              module.exports.save_tx(txid, block_height, block, function(err, tx_has_vout, newTx) {
                                 if (err) {
                                   // check the error code
                                   if (err.code == 'StackSizeError') {
                                     // ensure the process halts after stopping all sync threads
                                     stackSizeErrorId = txid;
-                                  } else if (err.code === 11000) {
-                                    // output a nicer error msg for the 11000 error code "duplicate key error collection" which can happen in some blockchains with non-standard txids being reused
-                                    console.log(`${settings.localization.ex_warning}: ${block_height}: ${txid} already exists`);
                                   } else
                                     console.log(err);
-                                }
-                                else
-                                  console.log('%s: %s', block_height, txid);
-
-                                if (tx_has_vout)
-                                  txes++;
 
                                   setTimeout(function() {
                                     tx = null;
@@ -386,6 +569,27 @@ module.exports = {
                                     } else
                                       next_tx();
                                   }, timeout);
+                                } else {
+                                  console.log('%s: %s', block_height, txid);
+
+                                  if (tx_has_vout)
+                                    txes++;
+
+                                  // add the tx to a queue
+                                  txBatchQueue.push(newTx, function(queue_err) {
+                                    setTimeout(function() {
+                                      tx = null;
+                                      tx_counter--;
+
+                                      // check if the script is stopping
+                                      if ((stopSync && check_only != 2) || stackSizeErrorId) {
+                                        // stop the loop
+                                        next_tx({});
+                                      } else
+                                        next_tx();
+                                    }, timeout);
+                                  });
+                                }
                               });
                             } else {
                               // skip adding the current tx
@@ -437,6 +641,11 @@ module.exports = {
                           blockhash = null;
                           block = null;
 
+                          // save the remaining txes for this block
+                          flushTxBatch(function(batch_err) {
+                            if (batch_err)
+                              console.error(batch_err);
+
                             // reset the slot in the block array back to 0
                             block_numbers[slotIndex] = 0;
 
@@ -453,6 +662,7 @@ module.exports = {
                               // proceed to next block
                               next_block();
                             }
+                          });
                         }
                       }, timeout);
                     });
@@ -516,7 +726,7 @@ module.exports = {
         // check if all threads have properly finished or else the retry limit has been reached
         // NOTE: the retry limit should never need to be used but is put in place to prevent an
         //       infinite loop just in case something goes very wrong
-        if (finished_tasks === parallel_tasks || retryAttempts >= retryLimit) {
+        if (finished_tasks === ((end - start + 1) >= parallel_tasks ? parallel_tasks : (end - start + 1)) || retryAttempts >= retryLimit) {
           // stop waiting for all threads to finish
           clearInterval(handle);
 
@@ -539,7 +749,9 @@ module.exports = {
     });
   },
 
-  delete_and_cleanup_tx: function(txid, block_height, tx_count, timeout, cb) {
+  delete_and_cleanup_tx: function(txid, block_height, timeout, cb) {
+    let tx_count = 0;
+
     // lookup all address tx records associated with the current tx
     AddressTx.find({txid: txid}).exec().then((address_txes) => {
       if (address_txes.length == 0) {
@@ -704,14 +916,12 @@ module.exports = {
           }
 
           // loop through the address txes
-          lib.syncLoop(addressTxArray.length, function(address_loop) {
-            var a = address_loop.iteration();
-
+          async.eachSeries(addressTxArray, function(addressTx, address_loop) {
              // fix the balance, sent and received data for the current address
-            fix_address_data(addressTxArray[a], function() {
+            fix_address_data(addressTx, function() {
               setTimeout(function() {
                 // move to the next address record
-                address_loop.next();
+                address_loop();
               }, timeout);
             });
           }, function() {

lib/database.js

@@ -1,19 +1,20 @@
-var mongoose = require('mongoose'),
-    Stats = require('../models/stats'),
-    Markets = require('../models/markets'),
-    Masternode = require('../models/masternode'),
-    Address = require('../models/address'),
-    AddressTx = require('../models/addresstx'),
-    Tx = require('../models/tx'),
-    Orphans = require('../models/orphans'),
-    Richlist = require('../models/richlist'),
-    Peers = require('../models/peers'),
-    Heavy = require('../models/heavy'),
-    NetworkHistory = require('../models/networkhistory'),
-    ClaimAddress = require('../models/claimaddress'),
-    lib = require('./explorer'),
-    settings = require('./settings'),
-    fs = require('fs');
+const mongoose = require('mongoose');
+const Stats = require('../models/stats');
+const Markets = require('../models/markets');
+const Masternode = require('../models/masternode');
+const Address = require('../models/address');
+const AddressTx = require('../models/addresstx');
+const Tx = require('../models/tx');
+const Orphans = require('../models/orphans');
+const Richlist = require('../models/richlist');
+const Peers = require('../models/peers');
+const Heavy = require('../models/heavy');
+const NetworkHistory = require('../models/networkhistory');
+const ClaimAddress = require('../models/claimaddress');
+const lib = require('./explorer');
+const settings = require('./settings');
+const fs = require('fs');
+const async = require('async');
 
 function find_address(hash, caseSensitive, cb) {
   if (caseSensitive) {
@@ -250,24 +251,22 @@ function remove_inactive_markets(installed_markets, cb) {
     // check if the database has any markets installed
     if (db_markets != null && db_markets.length > 0) {
       // loop through the list of markets in the database
-      lib.syncLoop(db_markets.length, function(market_loop) {
-        let m = market_loop.iteration();
-
+      async.eachSeries(db_markets, function(current_market, market_loop) {
         // check if this market is installed
-        if (installed_markets.findIndex(x => x.market.toUpperCase() == db_markets[m].market.toUpperCase() && x.coin_symbol.toUpperCase() == db_markets[m].coin_symbol.toUpperCase() && x.pair_symbol.toUpperCase() == db_markets[m].pair_symbol.toUpperCase()) == -1) {
+        if (installed_markets.findIndex(x => x.market.toUpperCase() == current_market.market.toUpperCase() && x.coin_symbol.toUpperCase() == current_market.coin_symbol.toUpperCase() && x.pair_symbol.toUpperCase() == current_market.pair_symbol.toUpperCase()) == -1) {
           // remove this market from the database because it is not installed or active
-          Markets.deleteOne({market: db_markets[m].market, coin_symbol: db_markets[m].coin_symbol, pair_symbol: db_markets[m].pair_symbol}).then(() => {
+          Markets.deleteOne({market: current_market.market, coin_symbol: current_market.coin_symbol, pair_symbol: current_market.pair_symbol}).then(() => {
             // move to the next market record
-            market_loop.next();
+            market_loop();
           }).catch((err) => {
             console.log(err);
 
             // move to the next market record
-            market_loop.next();
+            market_loop();
           });
         } else {
           // move to the next market record
-          market_loop.next();
+          market_loop();
         }
       }, function() {
         // finished removing inactive markets
@@ -299,35 +298,30 @@ function init_heavy(cb) {
 function init_claimaddress(coin, cb) {
   // first, get the stats data
   Stats.findOne({coin: coin}).then((stats) => {
-    var newer_claim_address = false;
+    let newer_claim_address = false;
 
-    // check if stats were found
-    if (stats) {
-      // check if the claim address data was already moved to the new collection
-      if (stats.newer_claim_address != null && stats.newer_claim_address == true)
+    // check if stats were found and the claim address data was already moved to the new collection
+    if (stats && stats.newer_claim_address != null && stats.newer_claim_address == true)
       newer_claim_address = true;
-    }
 
     // check if the claim address data should be moved to a new collection
     if (!newer_claim_address) {
       // find all addresses with a custom claim address name
-      Address.find({$and: [{"name": {$ne: ""}}, {"name": {$ne: null}}]}).exec().then((addresses) => {
+      Address.find({$and: [{'name': {$ne: ''}}, {'name': {$ne: null}}]}).exec().then((addresses) => {
         // loop through the claimed addresses
-        lib.syncLoop(addresses.length, function(address_loop) {
-          var a = address_loop.iteration();
-
+        async.eachSeries(addresses, function(current_address, address_loop) {
           // create a new claimaddress record
-          var claim_address = new ClaimAddress({
-            a_id: addresses[a].a_id,
-            claim_name: addresses[a].name
+          const claim_address = new ClaimAddress({
+            a_id: current_address.a_id,
+            claim_name: current_address.name
           });
 
           // add new claim address to collection
           claim_address.save().then(() => {
-            address_loop.next();
+            address_loop();
           }).catch((err) => {
             console.log(err);
-            address_loop.next();
+            address_loop();
           });
         }, function() {
           // finished moving all claimed address data to the new collection
@@ -864,17 +858,17 @@ module.exports = {
   },
 
   get_txs: function(block, cb) {
-    var txs = [];
+    let txs = [];
 
-    lib.syncLoop(block.tx.length, function (loop) {
-      var i = loop.iteration();
-
-      find_tx(block.tx[i], function(tx) {
+    async.eachSeries(block.tx, function(current_tx, loop) {
+      find_tx(current_tx[i], function(tx) {
         if (tx) {
+          // add tx to the list
           txs.push(tx);
-          loop.next();
-        } else
-          loop.next();
+        }
+
+        // move to next tx
+        loop();
       });
     }, function() {
       return cb(txs);
@@ -951,14 +945,10 @@ module.exports = {
   },
 
   get_address_txs_ajax: function(hash, start, length, cb) {
-    var totalCount = 0;
-
-    AddressTx.find({a_id: hash}).countDocuments().then((count) => {
-      totalCount = count;
-
+    AddressTx.find({a_id: hash}).countDocuments().then((totalCount) => {
       AddressTx.aggregate([
         { $match: { a_id: hash } },
-        { $sort: {blockindex: -1} },
+        { $sort: { blockindex: -1 } },
         { $skip: Number(start) },
         {
           $group: {
@@ -972,29 +962,27 @@ module.exports = {
             balance: '$balance'
           }
         },
-        { $sort: {blockindex: -1} }
+        { $sort: { blockindex: -1 } }
       ]).then((balance_sum) => {
         AddressTx.find({a_id: hash}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec().then((address_tx) => {
-          var txs = [];
-          var count = address_tx.length;
-          var running_balance = balance_sum.length > 0 ? balance_sum[0].balance : 0;
-          var txs = [];
+          let txs = [];
+          let running_balance = (balance_sum.length > 0 ? balance_sum[0].balance : 0);
 
-          lib.syncLoop(count, function (loop) {
-            var i = loop.iteration();
-
-            find_tx(address_tx[i].txid, function (tx) {
-              if (tx && !txs.includes(tx)) {
+          async.eachSeries(address_tx, function(current_addresstx, loop) {
+            find_tx(current_addresstx.txid, function(tx) {
+              if (tx) {
+                // set the balance for this tx
                 tx.balance = running_balance;
-                txs.push(tx);
-                loop.next();
-              } else if (!txs.includes(tx)) {
-                txs.push("1. Not found");
-                loop.next();
-              } else
-                loop.next();
 
-              running_balance = running_balance - address_tx[i].amount;
+                // add tx to list of txes
+                txs.push(tx);
+
+                // subtract from the running balance
+                running_balance -= current_addresstx.amount;
+              }
+
+              // move to next address tx
+              loop();
             });
           }, function () {
             return cb(txs, totalCount);
@@ -1133,7 +1121,7 @@ module.exports = {
   },
 
   get_distribution: function(richlist, stats, cb) {
-    var distribution = {
+    const distribution = {
       supply: stats.supply,
       t_1_25: {percent: 0, total: 0 },
       t_26_50: {percent: 0, total: 0 },
@@ -1142,32 +1130,30 @@ module.exports = {
       t_101plus: {percent: 0, total: 0 }
     };
 
-    lib.syncLoop(richlist.balance.length, function (loop) {
-      var i = loop.iteration();
-      var count = i + 1;
-      var percentage = ((richlist.balance[i].balance / 100000000) / stats.supply) * 100;
+    async.timesSeries(richlist.balance.length, function(i, loop) {
+      const percentage = ((richlist.balance[i].balance / 100000000) / stats.supply) * 100;
 
-      if (count <= 25 ) {
+      if ((i + 1) <= 25) {
         distribution.t_1_25.percent = distribution.t_1_25.percent + percentage;
         distribution.t_1_25.total = distribution.t_1_25.total + (richlist.balance[i].balance / 100000000);
       }
 
-      if (count <= 50 && count > 25) {
+      if ((i + 1) <= 50 && (i + 1) > 25) {
         distribution.t_26_50.percent = distribution.t_26_50.percent + percentage;
         distribution.t_26_50.total = distribution.t_26_50.total + (richlist.balance[i].balance / 100000000);
       }
 
-      if (count <= 75 && count > 50) {
+      if ((i + 1) <= 75 && (i + 1) > 50) {
         distribution.t_51_75.percent = distribution.t_51_75.percent + percentage;
         distribution.t_51_75.total = distribution.t_51_75.total + (richlist.balance[i].balance / 100000000);
       }
 
-      if (count <= 100 && count > 75) {
+      if ((i + 1) <= 100 && (i + 1) > 75) {
         distribution.t_76_100.percent = distribution.t_76_100.percent + percentage;
         distribution.t_76_100.total = distribution.t_76_100.total + (richlist.balance[i].balance / 100000000);
       }
 
-      loop.next();
+      loop();
     }, function() {
       distribution.t_101plus.percent = parseFloat(100 - distribution.t_76_100.percent - distribution.t_51_75.percent - distribution.t_26_50.percent - distribution.t_1_25.percent - (settings.richlist_page.burned_coins.include_burned_coins_in_distribution == true && richlist.burned > 0 ? ((richlist.burned / 100000000) / stats.supply) * 100 : 0)).toFixed(2);
       distribution.t_101plus.total = parseFloat(distribution.supply - distribution.t_76_100.total - distribution.t_51_75.total - distribution.t_26_50.total - distribution.t_1_25.total - (settings.richlist_page.burned_coins.include_burned_coins_in_distribution == true && richlist.burned > 0 ? (richlist.burned / 100000000) : 0)).toFixed(8);
@@ -1187,23 +1173,21 @@ module.exports = {
   // updates heavycoin stats
   // height: current block height, count: amount of votes to store
   update_heavy: function(coin, height, count, cb) {
-    var newVotes = [];
+    lib.get_maxmoney(function(maxmoney) {
+      lib.get_maxvote(function(maxvote) {
+        lib.get_vote(function(vote) {
+          lib.get_phase(function(phase) {
+            lib.get_reward(function(reward) {
+              module.exports.get_stats(settings.coin.name, function(stats) {
+                lib.get_estnext(function(estnext) {
+                  lib.get_nextin(function(nextin) {
+                    let newVotes = [];
 
-    lib.get_maxmoney( function (maxmoney) {
-      lib.get_maxvote( function (maxvote) {
-        lib.get_vote( function (vote) {
-          lib.get_phase( function (phase) {
-            lib.get_reward( function (reward) {
-              module.exports.get_stats(settings.coin.name, function (stats) {
-                lib.get_estnext( function (estnext) {
-                  lib.get_nextin( function (nextin) {
-                    lib.syncLoop(count, function (loop) {
-                      var i = loop.iteration();
-
-                      lib.get_blockhash(height - i, function (hash) {
-                        lib.get_block(hash, function (block) {
+                    async.timesSeries(count, function(i, loop) {
+                      lib.get_blockhash(height - i, function(hash) {
+                        lib.get_block(hash, function(block) {
                           newVotes.push({ count: height - i, reward: block.reward, vote: (block && block.vote ? block.vote : 0) });
-                          loop.next();
+                          loop();
                         });
                       });
                     }, function() {
@@ -1219,7 +1203,7 @@ module.exports = {
                         votes: newVotes
                       }).then(() => {
                         // update reward_last_updated value
-                        module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (new_cb) {
+                        module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (update_success) {
                           console.log('Heavycoin update complete');
                           return cb();
                         });
@@ -1830,10 +1814,10 @@ module.exports = {
   },
 
   populate_claim_address_names: function(tx, cb) {
-    var addresses = [];
+    const addresses = [];
 
     // loop through vin addresses
-    tx.vin.forEach(function (vin) {
+    tx.vin.forEach(function(vin) {
       // check if this address already exists
       if (addresses.indexOf(vin.addresses) == -1) {
         // add address to array
@@ -1842,7 +1826,7 @@ module.exports = {
     });
 
     // loop through vout addresses
-    tx.vout.forEach(function (vout) {
+    tx.vout.forEach(function(vout) {
       // check if this address already exists
       if (addresses.indexOf(vout.addresses) == -1) {
         // add address to array
@@ -1851,31 +1835,29 @@ module.exports = {
     });
 
     // loop through address array
-    lib.syncLoop(addresses.length, function(loop) {
-      var a = loop.iteration();
-
-      module.exports.get_claim_name(addresses[a], function(claim_name) {
+    async.eachSeries(addresses, function(current_address, loop) {
+      module.exports.get_claim_name(current_address, function(claim_name) {
         if (claim_name != null && claim_name != '') {
           // look for address in vin
-          for (v = 0; v < tx.vin.length; v++) {
+          for (let v = 0; v < tx.vin.length; v++) {
             // check if this is the correct address
-            if (tx.vin[v].addresses == addresses[a]) {
+            if (tx.vin[v].addresses == current_address) {
               // add claim name to array
               tx.vin[v]['claim_name'] = claim_name;
             }
           }
 
           // look for address in vout
-          for (v = 0; v < tx.vout.length; v++) {
+          for (let v = 0; v < tx.vout.length; v++) {
             // check if this is the correct address
-            if (tx.vout[v].addresses == addresses[a]) {
+            if (tx.vout[v].addresses == current_address) {
               // add claim name to array
               tx.vout[v]['claim_name'] = claim_name;
             }
           }
         }
 
-        loop.next();
+        loop();
       });
     }, function() {
       // return modified tx object

lib/explorer.js

@@ -1,10 +1,9 @@
-var request = require('postman-request'),
-    async = require('async'),
-    settings = require('./settings'),
-    Address = require('../models/address');
-
-var base_server = 'http://127.0.0.1:' + settings.webserver.port + "/";
-var base_url = base_server + 'api/';
+const request = require('postman-request');
+const async = require('async');
+const settings = require('./settings');
+const Address = require('../models/address');
+const base_server = 'http://127.0.0.1:' + settings.webserver.port + '/';
+const base_url = base_server + 'api/';
 const onode = require('./node');
 const client = new onode.Client(settings.wallet);
 
@@ -76,7 +75,7 @@ function convertHashUnits(hashes) {
   }
 }
 
-function processVoutAddresses(address_list, vout_value, arr_vout, cb) {
+function processVoutAddresses(address_list, vout_value, arr_vout) {
   // check if there are any addresses to process
   if (address_list != null && address_list.length > 0) {
     // check if vout address is inside an array
@@ -85,28 +84,21 @@ function processVoutAddresses(address_list, vout_value, arr_vout, cb) {
       address_list[0] = address_list[0][0];
     }
 
-    // check if vout address is unique, if so add to array, if not add its amount to existing index
-    module.exports.is_unique(arr_vout, address_list[0], 'addresses', function(unique, index) {
-      if (unique == true) {
-        // unique vout
-        module.exports.convert_to_satoshi(parseFloat(vout_value), function(amount_sat) {
-          arr_vout.push({addresses: address_list[0], amount: amount_sat});
+    const amount_sat = module.exports.convert_to_satoshi(parseFloat(vout_value));
+    const index = module.exports.is_unique(arr_vout, address_list[0], 'addresses');
 
-          return cb(arr_vout);
-        });
+    // check if vout address is unique, if so add to array, if not add its amount to existing index
+    if (index == -1) {
+      // unique vout
+      arr_vout.push({addresses: address_list[0], amount: amount_sat});
     } else {
       // already exists
-        module.exports.convert_to_satoshi(parseFloat(vout_value), function(amount_sat) {
-          arr_vout[index].amount = arr_vout[index].amount + amount_sat;
+      arr_vout[index].amount += amount_sat;
+    }
+  }
 
-          return cb(arr_vout);
-        });
-      }
-    });
-  } else {
-    // no address, move to next vout
-    return cb(arr_vout);
-  }
+  // return the list of addresses
+  return arr_vout;
 }
 
 function encodeP2PKaddress(p2pk_descriptor, cb) {
@@ -182,12 +174,146 @@ function normalizeMasternodeCount(raw_count) {
   }
 }
 
+function get_input_addresses(input, vout, cb) {
+  let addresses = [];
+
+  if (input.coinbase) {
+    let amount = 0;
+
+    for (let i = 0; i < vout.length; i++)
+      amount += parseFloat(vout[i].value);
+
+    addresses.push({ hash: 'coinbase', amount: amount });
+    return cb(addresses, null);
+  } else {
+    module.exports.get_rawtransaction(input.txid, function(tx) {
+      if (tx) {
+        let tx_type = null;
+
+        async.eachSeries(tx.vout, function(current_vout, loop) {
+          if (current_vout.n == input.vout) {
+            if (current_vout.scriptPubKey.addresses || current_vout.scriptPubKey.address) {
+              let new_address = current_vout.scriptPubKey.address || current_vout.scriptPubKey.addresses[0];
+
+              // check if address is inside an array
+              if (Array.isArray(new_address)) {
+                // extract the address
+                new_address = new_address[0];
+              }
+
+              const index = module.exports.is_unique(addresses, new_address, 'hash');
+
+              if (index == -1)
+                addresses.push({hash: new_address, amount: current_vout.value});
+              else
+                addresses[index].amount += current_vout.value;
+
+              loop();
+            } else {
+              // no addresses defined
+              // check if bitcoin features are enabled
+              if (settings.blockchain_specific.bitcoin.enabled == true) {
+                // assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address
+                encodeP2PKaddress(current_vout.scriptPubKey.asm, function(p2pkh_address) {
+                  // check if the address was encoded properly
+                  if (p2pkh_address != null) {
+                    // mark this tx as p2pk
+                    tx_type = 'p2pk';
+
+                    // check if address is inside an array
+                    if (Array.isArray(p2pkh_address)) {
+                      // extract the address
+                      p2pkh_address = p2pkh_address[0];
+                    }
+
+                    // save the P2PKH address
+                    const index = module.exports.is_unique(addresses, p2pkh_address, 'hash');
+
+                    if (index == -1)
+                      addresses.push({hash: p2pkh_address, amount: current_vout.value});
+                    else
+                      addresses[index].amount += current_vout.value;
+
+                    loop();
+                  } else {
+                    // could not decipher the address, save as unknown and move to next vin
+                    console.log('Failed to find vin address from tx ' + input.txid);
+
+                    const index = module.exports.is_unique(addresses, 'unknown_address', 'hash');
+
+                    if (index == -1)
+                      addresses.push({hash: 'unknown_address', amount: current_vout.value});
+                    else
+                      addresses[index].amount += current_vout.value;
+
+                    loop();
+                  }
+                });
+              } else {
+                // could not decipher the address, save as unknown and move to next vin
+                console.log('Failed to find vin address from tx ' + input.txid);
+
+                const index = module.exports.is_unique(addresses, 'unknown_address', 'hash');
+
+                if (index == -1)
+                  addresses.push({hash: 'unknown_address', amount: current_vout.value});
+                else
+                  addresses[index].amount += current_vout.value;
+
+                loop();
+              }
+            }
+          } else
+            loop();
+        }, function() {
+          return cb(addresses, tx_type);
+        });
+      } else
+        return cb();
+    });
+  }
+}
+
+function finalize_vout(first_vout, arr_vout, arr_vin, tx_type, txid, cb) {
+  if (typeof first_vout !== 'undefined' && first_vout.scriptPubKey.type == 'nonstandard') {
+    if (arr_vin.length > 0 && arr_vout.length > 0) {
+      if (arr_vin[0].addresses == arr_vout[0].addresses) {
+        // pos
+        arr_vout[0].amount -= arr_vin[0].amount;
+        arr_vin.shift();
+
+        // check if any vin remains
+        if (arr_vin == null || arr_vin.length == 0) {
+          // empty vin should be linked to coinbase
+          arr_vin = [{coinbase: 'coinbase'}];
+
+          let new_vout = [];
+
+          // loop through the arr_vout to create a copy of the data with coin amounts only for use with prepare_vin()
+          for (i = 0; i < arr_vout.length; i++)
+            new_vout.push({ value: arr_vout[i].amount / 100000000 });
+
+          // call the prepare_vin again to populate the vin data correctly
+          module.exports.prepare_vin({ txid: txid, vin: arr_vin, vout: new_vout}, function(return_vin, return_tx_type_vin) {
+            return cb(arr_vout, return_vin, return_tx_type_vin);
+          });
+        } else
+          return cb(arr_vout, arr_vin, tx_type);
+      } else
+        return cb(arr_vout, arr_vin, tx_type);
+    } else
+      return cb(arr_vout, arr_vin, tx_type);
+  } else
+    return cb(arr_vout, arr_vin, tx_type);
+}
+
 module.exports = {
-  convert_to_satoshi: function(amount, cb) {
-    // fix to 8dp & convert to string
-    var fixed = amount.toFixed(8).toString();
+  convert_to_satoshi: function(amount) {
+    // fix to 8 decimals and convert to string
+    const fixed = amount.toFixed(8).toString();
+
     // remove decimal (.) and return integer
-    return cb(parseInt(fixed.replace('.', '')));
+    return parseInt(fixed.replace('.', ''));
   },
 
   get_hashrate: function(cb) {
@@ -810,77 +936,13 @@ module.exports = {
     }
   },
 
-  // synchonous loop used to interate through an array,
-  // avoid use unless absolutely neccessary
-  syncLoop: function(iterations, process, exit) {
-    var index = 0,
-        done = false,
-        shouldExit = false;
-
-    var loop = {
-      next: function() {
-        if (done) {
-          if (shouldExit && exit) {
-            // exit if we're done
-            exit();
-          }
-
-          // stop the loop if we're done
-          return;
-        }
-
-        // if we're not finished
-        if (index < iterations) {
-          // increment our index
-          index++;
-
-          if (index % 100 === 0) {
-            // clear stack
-            setTimeout(function() {
-              // run our process, pass in the loop
-              process(loop);
-            }, 1);
-          } else {
-            // run our process, pass in the loop
-            process(loop);
-          }
-        } else {
-          // otherwise we're done
-          // make sure we say we're done
-          done = true;
-
-          if (exit) {
-            // call the callback on exit
-            exit();
-          }
-        }
-      },
-      iteration: function() {
-        // return the loop number we're on
-        return index - 1;
-      },
-      break: function(end) {
-        // end the loop
-        done = true;
-        // passing end as true means we still call the exit callback
-        shouldExit = end;
-      }
-    };
-
-    loop.next();
-
-    return loop;
-  },
-
   balance_supply: function(cb) {
     Address.find({}, 'balance').where('balance').gt(0).exec().then((docs) => {
-      var count = 0;
+      let count = 0;
 
-      module.exports.syncLoop(docs.length, function (loop) {
-        var i = loop.iteration();
-
-        count = count + docs[i].balance;
-        loop.next();
+      async.eachSeries(docs, function(current_doc, loop) {
+        count += current_doc.balance;
+        loop();
       }, function() {
         return cb(count);
       });
@@ -1080,223 +1142,161 @@ module.exports = {
     });
   },
 
-  is_unique: function(array, object, key_name, cb) {
-    var unique = true;
-    var index = null;
+  is_unique: function(array, object, key_name) {
+    for (let i = 0; i < array.length; i++) {
+      if (array[i][key_name] === object)
+        return i;
+    }
 
-    module.exports.syncLoop(array.length, function (loop) {
-      var i = loop.iteration();
-
-      if (array[i][key_name] == object) {
-        unique = false;
-        index = i;
-        loop.break(true);
-        loop.next();
-      } else
-        loop.next();
-    }, function() {
-      return cb(unique, index);
-    });
+    return -1;
   },
 
-  calculate_total: function(vout, cb) {
-    var total = 0;
+  calculate_total: function(vout) {
+    let total = 0;
 
-    module.exports.syncLoop(vout.length, function (loop) {
-      var i = loop.iteration();
+    for (let i = 0; i < vout.length; i++)
+      total += vout[i].amount;
 
-      total = total + vout[i].amount;
-      loop.next();
-    }, function() {
-      return cb(total);
-    });
+    return total;
   },
 
   prepare_vout: function(vout, txid, vin, vhidden, cb) {
-    var arr_vout = [];
-    var arr_vin = vin;
-    var tx_type = null;
+    let arr_vout = [];
+    let arr_vin = vin;
+    let tx_type = null;
 
     try {
-      module.exports.syncLoop(vout.length, function (loop) {
-        var i = loop.iteration();
+      async.eachSeries(vout, function(current_vout, loop) {
         // make sure vout has an address
-        if (vout[i].scriptPubKey.type != 'nonstandard' && vout[i].scriptPubKey.type != 'nulldata') {
+        if (current_vout.scriptPubKey.type != 'nonstandard' && current_vout.scriptPubKey.type != 'nulldata') {
           // check if this is a zerocoin tx
-          if (vout[i].scriptPubKey.type != 'zerocoinmint') {
-            var address_list = vout[i].scriptPubKey.addresses;
+          if (current_vout.scriptPubKey.type != 'zerocoinmint') {
+            const address_list = current_vout.scriptPubKey.addresses;
+
             // check if there are one or more addresses in the vout
             if (address_list == null || address_list.length == 0) {
               // no addresses defined
               // check if there is a single address defined
-              if (vout[i].scriptPubKey.address == null) {
+              if (current_vout.scriptPubKey.address == null) {
                 // no single address defined
                 // check if bitcoin features are enabled
                 if (settings.blockchain_specific.bitcoin.enabled == true) {
                   // assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address
-                  encodeP2PKaddress(vout[i].scriptPubKey.asm, function(p2pkh_address) {
+                  encodeP2PKaddress(current_vout.scriptPubKey.asm, function(p2pkh_address) {
                     // check if the address was encoded properly
                     if (p2pkh_address != null) {
                       // mark this tx as p2pk
                       tx_type = 'p2pk';
-                      // process vout addresses
-                      processVoutAddresses(p2pkh_address, vout[i].value, arr_vout, function(vout_array) {
-                        // save updated array
-                        arr_vout = vout_array;
+
+                      // process vout addresses and save the updated array
+                      arr_vout = processVoutAddresses(p2pkh_address, current_vout.value, arr_vout);
+                    } else {
+                      // could not decipher the address, save as unknown
+                      console.log('Failed to find vout address from tx ' + txid);
+
+                      // process vout addresses and save the updated array
+                      arr_vout = processVoutAddresses(['unknown_address'], current_vout.value, arr_vout);
+                    }
+
                     // move to next vout
-                        loop.next();
+                    loop();
                   });
                 } else {
                   // could not decipher the address, save as unknown and move to next vout
                   console.log('Failed to find vout address from tx ' + txid);
-                      // process vout addresses
-                      processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) {
-                        // save updated array
-                        arr_vout = vout_array;
+
+                  // process vout addresses and save the updated array
+                  arr_vout = processVoutAddresses(['unknown_address'], current_vout.value, arr_vout);
+
                   // move to next vout
-                        loop.next();
-                      });
-                    }
-                  });
-                } else {
-                  // could not decipher the address, save as unknown and move to next vout
-                  console.log('Failed to find vout address from tx ' + txid);
-                  // process vout addresses
-                  processVoutAddresses(['unknown_address'], vout[i].value, arr_vout, function(vout_array) {
-                    // save updated array
-                    arr_vout = vout_array;
-                    // move to next vout
-                    loop.next();
-                  });
+                  loop();
                 }
               } else {
-                // process vout address
-                processVoutAddresses([vout[i].scriptPubKey.address], vout[i].value, arr_vout, function(vout_array) {
-                  // save updated array
-                  arr_vout = vout_array;
+                // process vout addresses and save the updated array
+                arr_vout = processVoutAddresses([current_vout.scriptPubKey.address], current_vout.value, arr_vout);
+
                 // move to next vout
-                  loop.next();
-                });
+                loop();
               }
             } else {
-              // process vout addresses
-              processVoutAddresses(address_list, vout[i].value, arr_vout, function(vout_array) {
-                // save updated array
-                arr_vout = vout_array;
+              // process vout addresses and save the updated array
+              arr_vout = processVoutAddresses(address_list, current_vout.value, arr_vout);
+
               // move to next vout
-                loop.next();
-              });
+              loop();
             }
           } else {
             // TODO: add support for zerocoin transactions
             console.log('Zerocoin tx found. skipping for now as it is unsupported');
             tx_type = "zerocoin";
-            loop.next();
+            loop();
           }
         } else {
           // no address, move to next vout
-          loop.next();
+          loop();
         }
       }, function() {
-        // check if zksnarks is enabled
-        if (settings.blockchain_specific.zksnarks.enabled == true) {
-          // check for hidden/anonymous outputs
-          if (vhidden != null && vhidden.length > 0) {
-            tx_type = "zksnarks";
-            // loop through all hidden/anonymous outputs
-            module.exports.syncLoop(vhidden.length, function (loop) {
-              var i = loop.iteration();
+        // check if zksnarks is enabled and there are any hidden/anonymous outputs
+        if (settings.blockchain_specific.zksnarks.enabled == true && vhidden != null && vhidden.length > 0) {
+          const hidden_address = 'hidden_address';
+          tx_type = 'zksnarks';
+
+          // loop through all hidden/anonymous outputs
+          async.eachSeries(vhidden, function(current_vhidden, loop) {
+            if (current_vhidden.vpub_old > 0) {
+              // process vout addresses and save the updated array
+              arr_vout = processVoutAddresses([hidden_address], parseFloat(current_vhidden.vpub_old), arr_vout);
 
-              if (vhidden[i].vpub_old > 0) {
-                // process vout addresses
-                processVoutAddresses(['hidden_address'], parseFloat(vhidden[i].vpub_old), arr_vout, function(vout_array) {
-                  // save updated array
-                  arr_vout = vout_array;
               // move to next vout
-                  loop.next();
-                });
+              loop();
             } else {
+              const amount_sat = module.exports.convert_to_satoshi(parseFloat(current_vhidden.vpub_new));
+              const index = module.exports.is_unique(arr_vin, hidden_address, 'addresses');
+
               if ((!vout || vout.length == 0) && (!vin || vin.length == 0)) {
                 // hidden sender is sending to hidden recipient
                 // the sent and received values are not known in this case. only the fee paid is known and subtracted from the sender.
-                  // process vout addresses
-                  processVoutAddresses(['hidden_address'], 0, arr_vout, function(vout_array) {
-                    // save updated array
-                    arr_vout = vout_array;
+                // process vout addresses and save the updated array
+                arr_vout = processVoutAddresses([hidden_address], 0, arr_vout);
+
                 // add a private send address with the known amount sent
-                    module.exports.is_unique(arr_vin, 'hidden_address', 'addresses', function(unique, index) {
-                      if (unique == true) {
-                        module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
-                          arr_vin.push({addresses: 'hidden_address', amount: amount_sat});
-                          // move to next vout
-                          loop.next();
-                        });
+                if (index == -1) {
+                  // add hidden address to array
+                  arr_vin.push({addresses: hidden_address, amount: amount_sat});
                 } else {
-                        module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
+                  // update hidden address amount in array
                   arr_vin[index].amount = arr_vin[index].amount + amount_sat;
-                          // move to next vout
-                          loop.next();
-                        });
                 }
+
+                // move to next vout
+                loop();
+              } else {
+                // add a private send address with the known amount sent
+                if (index == -1) {
+                  // add hidden address to array
+                  arr_vin.push({addresses: hidden_address, amount: amount_sat});
+                } else {
+                  // update hidden address amount in array
+                  arr_vin[index].amount += amount_sat;
+                }
+
+                // move to next vout
+                loop();
+              }
+            }
+          }, function() {
+            // finished updating hidden vout data
+            finalize_vout(vout[0], arr_vout, arr_vin, tx_type, txid, function(final_arr_vout, final_arr_vin, final_tx_type) {
+              return cb(final_arr_vout, final_arr_vin, final_tx_type);
             });
           });
         } else {
-                  module.exports.is_unique(arr_vin, 'hidden_address', 'addresses', function(unique, index) {
-                    if (unique == true) {
-                      module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
-                        arr_vin.push({addresses: 'hidden_address', amount: amount_sat});
-                        // move to next vout
-                        loop.next();
-                      });
-                    } else {
-                      module.exports.convert_to_satoshi(parseFloat(vhidden[i].vpub_new), function(amount_sat) {
-                        arr_vin[index].amount = arr_vin[index].amount + amount_sat;
-                        // move to next vout
-                        loop.next();
+          // finalize vout data
+          finalize_vout(vout[0], arr_vout, arr_vin, tx_type, txid, function(final_arr_vout, final_arr_vin, final_tx_type) {
+            return cb(final_arr_vout, final_arr_vin, final_tx_type);
           });
         }
       });
-                }
-              }
-            });
-          }
-        }
-
-        if (typeof vout[0] !== 'undefined' && vout[0].scriptPubKey.type == 'nonstandard') {
-          if (arr_vin.length > 0 && arr_vout.length > 0) {
-            if (arr_vin[0].addresses == arr_vout[0].addresses) {
-              //PoS
-              arr_vout[0].amount = arr_vout[0].amount - arr_vin[0].amount;
-              arr_vin.shift();
-
-              // check if any vin remains
-              if (arr_vin == null || arr_vin.length == 0) {
-                // empty vin should be linked to coinbase
-                arr_vin = [{coinbase: "coinbase"}];
-
-                var new_vout = [];
-
-                // loop through the arr_vout to create a copy of the data with coin amounts only for use with prepare_vin()
-                for (i = 0; i < arr_vout.length; i++) {
-                  new_vout.push({
-                    value: arr_vout[i].amount / 100000000
-                  });
-                }
-
-                // call the prepare_vin again to populate the vin data correctly
-                module.exports.prepare_vin({txid: txid, vin: arr_vin, vout: new_vout}, function(return_vin, return_tx_type_vin) {
-                  return cb(arr_vout, return_vin, return_tx_type_vin);
-                });
-              } else {
-                return cb(arr_vout, arr_vin, tx_type);
-              }
-            } else
-              return cb(arr_vout, arr_vin, tx_type);
-          } else
-            return cb(arr_vout, arr_vin, tx_type);
-        } else
-          return cb(arr_vout, arr_vin, tx_type);
-      });
     } catch(err) {
       // check if a "Maximum call stack size exceeded" error occurred
       if (err instanceof RangeError && /Maximum call stack size exceeded/i.test(err.message)) {
@@ -1309,122 +1309,12 @@ module.exports = {
     }
   },
 
-  get_input_addresses: function(input, vout, cb) {
-    var addresses = [];
-
-    if (input.coinbase) {
-      var amount = 0;
-
-      module.exports.syncLoop(vout.length, function (loop) {
-        var i = loop.iteration();
-
-        amount = amount + parseFloat(vout[i].value);
-        loop.next();
-      }, function() {
-        addresses.push({hash: 'coinbase', amount: amount});
-        return cb(addresses, null);
-      });
-    } else {
-      module.exports.get_rawtransaction(input.txid, function(tx) {
-        if (tx) {
-          var tx_type = null;
-
-          module.exports.syncLoop(tx.vout.length, function (loop) {
-            var i = loop.iteration();
-
-            if (tx.vout[i].n == input.vout) {
-              if (tx.vout[i].scriptPubKey.addresses || tx.vout[i].scriptPubKey.address) {
-                var new_address = tx.vout[i].scriptPubKey.address || tx.vout[i].scriptPubKey.addresses[0];
-
-                // check if address is inside an array
-                if (Array.isArray(new_address)) {
-                  // extract the address
-                  new_address = new_address[0];
-                }
-
-                module.exports.is_unique(addresses, new_address, 'hash', function(unique, index) {
-                  if (unique == true)
-                    addresses.push({hash: new_address, amount: tx.vout[i].value});
-                  else
-                    addresses[index].amount = addresses[index].amount + tx.vout[i].value;
-
-                  loop.break(true);
-                  loop.next();
-                });
-              } else {
-                // no addresses defined
-                // check if bitcoin features are enabled
-                if (settings.blockchain_specific.bitcoin.enabled == true) {
-                  // assume the asm value is a P2PK (Pay To Pubkey) public key that should be encoded as a P2PKH (Pay To Pubkey Hash) address
-                  encodeP2PKaddress(tx.vout[i].scriptPubKey.asm, function(p2pkh_address) {
-                    // check if the address was encoded properly
-                    if (p2pkh_address != null) {
-                      // mark this tx as p2pk
-                      tx_type = 'p2pk';
-
-                      // check if address is inside an array
-                      if (Array.isArray(p2pkh_address)) {
-                        // extract the address
-                        p2pkh_address = p2pkh_address[0];
-                      }
-
-                      // save the P2PKH address
-                      module.exports.is_unique(addresses, p2pkh_address, 'hash', function(unique, index) {
-                        if (unique == true)
-                          addresses.push({hash: p2pkh_address, amount: tx.vout[i].value});
-                        else
-                          addresses[index].amount = addresses[index].amount + tx.vout[i].value;
-
-                        loop.break(true);
-                        loop.next();
-                      });
-                    } else {
-                      // could not decipher the address, save as unknown and move to next vin
-                      console.log('Failed to find vin address from tx ' + input.txid);
-                      module.exports.is_unique(addresses, 'unknown_address', 'hash', function(unique, index) {
-                        if (unique == true)
-                          addresses.push({hash: 'unknown_address', amount: tx.vout[i].value});
-                        else
-                          addresses[index].amount = addresses[index].amount + tx.vout[i].value;
-
-                        loop.break(true);
-                        loop.next();
-                      });
-                    }
-                  });
-                } else {
-                  // could not decipher the address, save as unknown and move to next vin
-                  console.log('Failed to find vin address from tx ' + input.txid);
-                  module.exports.is_unique(addresses, 'unknown_address', 'hash', function(unique, index) {
-                    if (unique == true)
-                      addresses.push({hash: 'unknown_address', amount: tx.vout[i].value});
-                    else
-                      addresses[index].amount = addresses[index].amount + tx.vout[i].value;
-
-                    loop.break(true);
-                    loop.next();
-                  });
-                }
-              }
-            } else
-              loop.next();
-          }, function() {
-            return cb(addresses, tx_type);
-          });
-        } else
-          return cb();
-      });
-    }
-  },
-
   prepare_vin: function(tx, cb) {
-    var arr_vin = [];
-    var tx_type = null;
+    let arr_vin = [];
+    let tx_type = null;
 
-    module.exports.syncLoop(tx.vin.length, function (loop) {
-      var i = loop.iteration();
-
-      module.exports.get_input_addresses(tx.vin[i], tx.vout, function(addresses, tx_type_vin) {
+    async.eachSeries(tx.vin, function(vin, loop) {
+      get_input_addresses(vin, tx.vout, function(addresses, tx_type_vin) {
         // check if the tx type is set
         if (tx_type_vin != null) {
           // set the tx type return value
@@ -1432,28 +1322,25 @@ module.exports = {
         }
 
         if (addresses && addresses.length) {
-          module.exports.is_unique(arr_vin, addresses[0].hash, 'addresses', function(unique, index) {
-            if (unique == true) {
-              module.exports.convert_to_satoshi(parseFloat(addresses[0].amount), function(amount_sat) {
+          const amount_sat = module.exports.convert_to_satoshi(parseFloat(addresses[0].amount));
+          const index = module.exports.is_unique(arr_vin, addresses[0].hash, 'addresses');
+
+          if (index == -1)
             arr_vin.push({addresses: addresses[0].hash, amount: amount_sat});
-                loop.next();
-              });
-            } else {
-              module.exports.convert_to_satoshi(parseFloat(addresses[0].amount), function(amount_sat) {
-                arr_vin[index].amount = arr_vin[index].amount + amount_sat;
-                loop.next();
-              });
-            }
-          });
+          else
+            arr_vin[index].amount += amount_sat;
+
+          loop();
         } else {
           // could not decipher the address, save as unknown and move to next vin
           console.log('Failed to find vin address from tx ' + tx.txid);
-          module.exports.is_unique(arr_vin, 'unknown_address', 'addresses', function(unique, index) {
-            if (unique == true)
+
+          const index = module.exports.is_unique(arr_vin, 'unknown_address', 'addresses');
+
+          if (index == -1)
             arr_vin.push({addresses: 'unknown_address', amount: 0});
 
-            loop.next();
-          });
+          loop();
         }
       });
     }, function() {

lib/settings.js

@@ -1412,10 +1412,24 @@ exports.sync = {
   //         BALANCES : get the supply by running a query on the addresses collection and summing up all positive balances (potentially a long running query for blockchains with tons of addresses)
   //         TXOUTSET : retrieved from gettxoutsetinfo rpc cmd
   "supply": "GETINFO",
+  // batch_size: The maximum number of records before saving data to the database.
+  //             This value is used for syncing transactions, addresses and address transactions.
+  //             Each record type is processed within a single block so batching only happens if there are more than `batch_size` txes in a single block for example.
+  //             If the number of txes is lower than `batch_size` then all txes are saved in one batch.
+  //             A higher batch_size can save data faster than having to do smaller batches but only up to a certain point since it also requires more memory and resources for larger batches and the optimal number depends entirely on your server's resources.
+  //             A lower batch size generally ensures there will be no memory limitations although it can also slow down the sync process.
+  //             It is recommended to leave this value alone unless you know what you are doing although some experimentation with different batch sizes using the benchmark script can often help determine the optimal setting for your server.
+  "batch_size": 5000,
   // elastic_stack_size: If a "RangeError: Maximum call stack size exceeded" error occurs during a block sync (which can happen when dealing with large transactions with many addresses), the sync script will automatically be reloaded using a larger stack size value which increases memory usage based on this value.
   //                     NOTE: If the first reload of the sync script still doesn't have enough memory to handle processing of a large transaction, the sync is smart enough to continue increasing the stack size by this value again and again until it finishes processing all blocks and then returns back to the default amount of memory for future blocks.
   //                           It is recommended to leave this value alone unless you know what you are doing.
-  "elastic_stack_size": 4096
+  "elastic_stack_size": 4096,
+  // wait_for_bulk_database_save: Determine whether to wait for all records to be saved or just send the records without waiting for save confirmation when saving bulk data
+  //                              This setting only controls how to treat records that are bulk saved to the database which include txes, addresstxes and addresses
+  //                              If set to true, bulk transactions to the database will wait for save confirmation which results in a slower save time but also returns information about which records failed to save
+  //                              If set to false, bulk transactions to the database will not wait for save confirmation which results in a faster save time but will not return any error message for records that failed to save
+  //                              NOTE: If you want to sync data as fast as possible and are sure that your blockchain doesn't contain any problematic or unsupported data types then you can set this value to "false" to maximize the speed of the block sync
+  "wait_for_bulk_database_save": true
 };
 
 // captcha: a collection of settings that pertain to the captcha security used by different elements of the explorer

routes/index.js

@@ -1,8 +1,9 @@
-var express = require('express'),
-    router = express.Router(),
-    settings = require('../lib/settings'),
-    db = require('../lib/database'),
-    lib = require('../lib/explorer');
+const express = require('express');
+const router = express.Router();
+const settings = require('../lib/settings');
+const db = require('../lib/database');
+const lib = require('../lib/explorer');
+const async = require('async');
 
 function send_block_data(res, block, txs, title_text, orphan) {
   res.render(
@@ -95,26 +96,24 @@ function get_last_updated_date(show_last_updated, last_updated_field, cb) {
 function get_block_data_from_wallet(block, res, orphan) {
   var ntxs = [];
 
-  lib.syncLoop(block.tx.length, function (loop) {
-    var i = loop.iteration();
-
-    lib.get_rawtransaction(block.tx[i], function(tx) {
+  async.eachSeries(block.tx, function(block_tx, loop) {
+    lib.get_rawtransaction(block_tx, function(tx) {
       if (tx && tx != `${settings.localization.ex_error}: ${settings.localization.check_console}`) {
         lib.prepare_vin(tx, function(vin, tx_type_vin) {
-          lib.prepare_vout(tx.vout, block.tx[i], vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) {
-            lib.calculate_total(vout, function(total) {
+          lib.prepare_vout(tx.vout, block_tx, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) {
+            const total = lib.calculate_total(vout);
+
             ntxs.push({
-                txid: block.tx[i],
+              txid: block_tx,
               vout: vout,
               total: total.toFixed(8)
             });
 
-              loop.next();
-            });
+            loop();
           });
         });
       } else
-        loop.next();
+        loop();
     });
   }, function() {
     send_block_data(res, block, ntxs, 'Block ' + block.height, orphan);
@@ -188,7 +187,8 @@ function route_get_tx(res, txid) {
           if (rtx && rtx.txid) {
             lib.prepare_vin(rtx, function(vin, tx_type_vin) {
               lib.prepare_vout(rtx.vout, rtx.txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof rtx.vjoinsplit === 'undefined' || rtx.vjoinsplit == null) ? [] : rtx.vjoinsplit), function(rvout, rvin, tx_type_vout) {
-                lib.calculate_total(rvout, function(total) {
+                const total = lib.calculate_total(rvout);
+
                 if (!rtx.confirmations > 0) {
                   lib.get_block(rtx.blockhash, function(block) {
                     if (block && block != `${settings.localization.ex_error}: ${settings.localization.check_console}`) {
@@ -267,7 +267,6 @@ function route_get_tx(res, txid) {
                 }
               });
             });
-            });
           } else
             route_get_txlist(res, null);
         });

scripts/sync.js

@@ -1,20 +1,20 @@
-var mongoose = require('mongoose'),
-    lib = require('../lib/explorer'),
-    blkSync = require('../lib/block_sync'),
-    db = require('../lib/database'),
-    Tx = require('../models/tx'),
-    Address = require('../models/address'),
-    AddressTx = require('../models/addresstx'),
-    Orphans = require('../models/orphans'),
-    Richlist = require('../models/richlist'),
-    Stats = require('../models/stats'),
-    settings = require('../lib/settings'),
-    async = require('async');
-var mode = 'update';
-var database = 'index';
-var block_start = 1;
-var lockCreated = false;
-var stopSync = false;
+const mongoose = require('mongoose');
+const lib = require('../lib/explorer');
+const blkSync = require('../lib/block_sync');
+const db = require('../lib/database');
+const Tx = require('../models/tx');
+const Address = require('../models/address');
+const AddressTx = require('../models/addresstx');
+const Orphans = require('../models/orphans');
+const Richlist = require('../models/richlist');
+const Stats = require('../models/stats');
+const settings = require('../lib/settings');
+const async = require('async');
+let mode = 'update';
+let database = 'index';
+let block_start = 1;
+let lockCreated = false;
+let stopSync = false;
 
 // prevent stopping of the sync script to be able to gracefully shut down
 process.on('SIGINT', () => {
@@ -172,7 +172,7 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout,
                 // this fork needs to be resolved
                 // lookup the current block data from the wallet
                 lib.get_block(correct_block_data.prev_hash, function (block_data) {
-                  var tx_count = 0;
+                  let tx_count = 0;
 
                   // check if the good block hash is in the list of blockhashes
                   if (blockhashes.indexOf(correct_block_data.prev_hash) > -1) {
@@ -181,9 +181,7 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout,
                   }
 
                   // loop through the remaining orphaned block hashes
-                  lib.syncLoop(blockhashes.length, function(block_loop) {
-                    var i = block_loop.iteration();
-
+                  async.timesSeries(blockhashes.length, function(i, block_loop) {
                     console.log('Resolving orphaned block [' + (i + 1).toString() + '/' + blockhashes.length.toString() + ']: ' + blockhashes[i]);
 
                     // find all orphaned txid's from the current orphan block hash
@@ -191,17 +189,15 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout,
                       // save the orphan block data to the orphan collection
                       create_orphan(current_block, blockhashes[i], correct_block_data.prev_hash, block_data.previousblockhash, correct_block_data.next_hash, function() {
                         // loop through the remaining orphaned block hashes
-                        lib.syncLoop(txids.length, function(tx_loop) {
-                          var t = tx_loop.iteration();
-
+                        async.eachSeries(txids, function(current_txid, tx_loop) {
                           // remove the orphaned tx and cleanup all associated data
-                          blkSync.delete_and_cleanup_tx(txids[t], current_block, tx_count, timeout, function(updated_tx_count) {
+                          blkSync.delete_and_cleanup_tx(current_txid, current_block, timeout, function(updated_tx_count) {
                             // update the running tx count
-                            tx_count = updated_tx_count;
+                            tx_count += updated_tx_count;
 
                             // some blockchains will reuse the same orphaned transaction ids
                             // lookup the transaction that was just deleted to ensure it doesn't belong to another block
-                            check_add_tx(txids[t], blockhashes[i], tx_count, function(updated_tx_count2) {
+                            check_add_tx(current_txid, blockhashes[i], tx_count, function(updated_tx_count2) {
                               // update the running tx count
                               tx_count = updated_tx_count2;
 
@@ -209,11 +205,11 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout,
                                 // check if there was a memory error
                                 if (blkSync.getStackSizeErrorId() != null) {
                                   // stop the loop
-                                  tx_loop.break(true);
-                                }
-
+                                  tx_loop({});
+                                } else {
                                   // move to the next tx record
-                                tx_loop.next();
+                                  tx_loop();
+                                }
                               }, timeout);
                             });
                           });
@@ -222,11 +218,11 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout,
                             // check if there was a memory error
                             if (blkSync.getStackSizeErrorId() != null) {
                               // stop the loop
-                              block_loop.break(true);
-                            }
-
+                              block_loop({});
+                            } else {
                               // move to the next block record
-                            block_loop.next();
+                              block_loop();
+                            }
                           }, timeout);
                         });
                       });
@@ -326,33 +322,31 @@ function update_orphans(orphan_index, orphan_current, last_blockindex, timeout,
           // get the list of orphans with a null next_blockhash
           Orphans.find({next_blockhash: null}).exec().then((orphans) => {
             // loop through the list of orphans
-            lib.syncLoop(orphans.length, function(orphan_loop) {
-              var o = orphan_loop.iteration();
-
+            async.eachSeries(orphans, function(current_orphan, orphan_loop) {
               // lookup the block data from the wallet
-              lib.get_block(orphans[o].good_blockhash, function (good_block_data) {
+              lib.get_block(current_orphan.good_blockhash, function (good_block_data) {
                 // check if the next block hash is known
                 if (good_block_data.nextblockhash != null) {
                   // update the next blockhash for this orphan record
-                  Orphans.updateOne({blockindex: orphans[o].blockindex, orphan_blockhash: orphans[o].orphan_blockhash}, {
+                  Orphans.updateOne({blockindex: current_orphan.blockindex, orphan_blockhash: current_orphan.orphan_blockhash}, {
                     next_blockhash: good_block_data.nextblockhash
                   }).then(() => {
                     setTimeout(function() {
                       // move to the next orphan record
-                      orphan_loop.next();
+                      orphan_loop();
                     }, timeout);
                   }).catch((err) => {
                     console.log(err);
 
                     setTimeout(function() {
                       // move to the next orphan record
-                      orphan_loop.next();
+                      orphan_loop();
                     }, timeout);
                   });
                 } else {
                   setTimeout(function() {
                     // move to the next orphan record
-                    orphan_loop.next();
+                    orphan_loop();
                   }, timeout);
                 }
               });
@@ -542,8 +536,8 @@ function check_add_tx(txid, blockhash, tx_count, cb) {
         // check if the block was found
         if (block) {
           // save the tx to the local database
-          blkSync.save_tx(txid, block.height, block, function(save_tx_err, tx_has_vout) {
-            // check if there were any save errors
+          blkSync.save_tx(txid, block.height, block, function(save_tx_err, tx_has_vout, newTx) {
+            // check for errors
             if (save_tx_err) {
               // check the error code
               if (save_tx_err.code == 'StackSizeError') {
@@ -551,16 +545,25 @@ function check_add_tx(txid, blockhash, tx_count, cb) {
                 blkSync.setStackSizeErrorId(txid);
               } else
                 console.log(save_tx_err);
-            } else
+
+              return cb(tx_count);
+            } else {
+              // save the tx
+              newTx.save().then(() => {
                 console.log('%s: %s', block.height, txid);
 
-            // check if the tx was saved correctly
+                // check if the tx has vouts
                 if (tx_has_vout) {
                   // keep a running total of txes that were added
                   tx_count++;
                 }
 
                 return cb(tx_count);
+              }).catch((err) => {
+                console.log(err);
+                return cb(tx_count);
+              });
+            }
           });
         } else {
           // block not found so there is nothing to fix
@@ -842,10 +845,11 @@ function occurrences(string, subString, allowOverlapping) {
 }
 
 function block_sync(reindex, stats) {
-  // Get the last synced block index value
-  var last = (stats.last ? stats.last : 0);
-  // Get the total number of blocks
-  var count = (stats.count ? stats.count : 0);
+  // get the last synced block index value
+  let last = (stats.last ? stats.last : 0);
+
+  // get the total number of blocks
+  let count = (stats.count ? stats.count : 0);
 
   // Check if the sync msg should be shown
   check_show_sync_message(count - last);
@@ -1148,20 +1152,19 @@ if (lib.is_locked([database]) == false) {
       } else if (database == 'peers') {
         lib.get_peerinfo(function(body) {
           if (body != null) {
-            lib.syncLoop(body.length, function(loop) {
-              var i = loop.iteration();
-              var address = body[i].addr;
-              var port = null;
+            async.timesSeries(body.length, function(i, loop) {
+              let address = body[i].addr;
+              let port = null;
 
               if (occurrences(address, ':') == 1 || occurrences(address, ']:') == 1) {
-                // Separate the port # from the IP address
-                address = address.substring(0, address.lastIndexOf(":")).replace("[", "").replace("]", "");
-                port = body[i].addr.substring(body[i].addr.lastIndexOf(":") + 1);
+                // separate the port # from the IP address
+                address = address.substring(0, address.lastIndexOf(':')).replace('[', '').replace(']', '');
+                port = body[i].addr.substring(body[i].addr.lastIndexOf(':') + 1);
               }
 
-              if (address.indexOf("]") > -1) {
-                // Remove [] characters from IPv6 addresses
-                address = address.replace("[", "").replace("]", "");
+              if (address.indexOf(']') > -1) {
+                // remove [] characters from IPv6 addresses
+                address = address.replace('[', '').replace(']', '');
               }
 
               db.find_peer(address, port, function(peer) {
@@ -1190,10 +1193,11 @@ if (lib.is_locked([database]) == false) {
                       // check if the script is stopping
                       if (stopSync) {
                         // stop the loop
-                        loop.break(true);
+                        loop({});
+                      } else {
+                        // move to next peer
+                        loop();
                       }
-
-                      loop.next();
                     });
                   });
                 } else {
@@ -1224,10 +1228,11 @@ if (lib.is_locked([database]) == false) {
                           // check if the script is stopping
                           if (stopSync) {
                             // stop the loop
-                            loop.break(true);
+                            loop({});
+                          } else {
+                            // move to next peer
+                            loop();
                           }
-
-                          loop.next();
                         });
                       }
                     });
@@ -1255,36 +1260,35 @@ if (lib.is_locked([database]) == false) {
       } else if (database == 'masternodes') {
         lib.get_masternodelist(function(body) {
           if (body != null) {
-            var isObject = false;
-            var objectKeys = null;
+            let isObject = false;
+            let objectKeys = null;
 
-            // Check if the masternode data is an array or an object
+            // check if the masternode data is an array or an object
             if (body.length == null) {
-              // Process data as an object
+              // process data as an object
               objectKeys = Object.keys(body);
               isObject = true;
             }
 
-            lib.syncLoop((isObject ? objectKeys : body).length, function(loop) {
-              var i = loop.iteration();
-
+            async.timesSeries((isObject ? objectKeys : body).length, function(i, loop) {
               db.save_masternode((isObject ? body[objectKeys[i]] : body[i]), (isObject ? objectKeys[i] : null), function(success) {
                 if (success) {
                   // check if the script is stopping
                   if (stopSync) {
                     // stop the loop
-                    loop.break(true);
+                    loop({});
+                  } else {
+                    // move to next masternode
+                    loop();
                   }
-
-                  loop.next();
                 } else {
                   console.log('Error: Cannot save masternode %s.', (isObject ? (body[objectKeys[i]].payee ? body[objectKeys[i]].payee : 'UNKNOWN') : (body[i].addr ? body[i].addr : 'UNKNOWN')));
                   exit(1);
                 }
               });
             }, function() {
-              db.remove_old_masternodes(function(cb) {
-                db.update_last_updated_stats(settings.coin.name, { masternodes_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
+              db.remove_old_masternodes(function() {
+                db.update_last_updated_stats(settings.coin.name, { masternodes_last_updated: Math.floor(new Date() / 1000) }, function(update_success) {
                   // check if the script stopped prematurely
                   if (stopSync) {
                     console.log('Masternode sync was stopped prematurely');

settings.json.template

@@ -1499,10 +1499,24 @@
     //         BALANCES : get the supply by running a query on the addresses collection and summing up all positive balances (potentially a long running query for blockchains with tons of addresses)
     //         TXOUTSET : retrieved from gettxoutsetinfo rpc cmd
     "supply": "GETINFO",
+    // batch_size: The maximum number of records before saving data to the database.
+    //             This value is used for syncing transactions, addresses and address transactions.
+    //             Each record type is processed within a single block so batching only happens if there are more than `batch_size` txes in a single block for example.
+    //             If the number of txes is lower than `batch_size` then all txes are saved in one batch.
+    //             A higher batch_size can save data faster than having to do smaller batches but only up to a certain point since it also requires more memory and resources for larger batches and the optimal number depends entirely on your server's resources.
+    //             A lower batch size generally ensures there will be no memory limitations although it can also slow down the sync process.
+    //             It is recommended to leave this value alone unless you know what you are doing although some experimentation with different batch sizes using the benchmark script can often help determine the optimal setting for your server.
+    "batch_size": 5000,
     // elastic_stack_size: If a "RangeError: Maximum call stack size exceeded" error occurs during a block sync (which can happen when dealing with large transactions with many addresses), the sync script will automatically be reloaded using a larger stack size value which increases memory usage based on this value.
     //                     NOTE: If the first reload of the sync script still doesn't have enough memory to handle processing of a large transaction, the sync is smart enough to continue increasing the stack size by this value again and again until it finishes processing all blocks and then returns back to the default amount of memory for future blocks.
     //                           It is recommended to leave this value alone unless you know what you are doing.
-    "elastic_stack_size": 4096
+    "elastic_stack_size": 4096,
+    // wait_for_bulk_database_save: Determine whether to wait for all records to be saved or just send the records without waiting for save confirmation when saving bulk data
+    //                              This setting only controls how to treat records that are bulk saved to the database which include txes, addresstxes and addresses
+    //                              If set to true, bulk transactions to the database will wait for save confirmation which results in a slower save time but also returns information about which records failed to save
+    //                              If set to false, bulk transactions to the database will not wait for save confirmation which results in a faster save time but will not return any error message for records that failed to save
+    //                              NOTE: If you want to sync data as fast as possible and are sure that your blockchain doesn't contain any problematic or unsupported data types then you can set this value to "false" to maximize the speed of the block sync
+    "wait_for_bulk_database_save": true
   },
 
   // captcha: a collection of settings that pertain to the captcha security used by different elements of the explorer

test/explorerSpec.js

@@ -4,23 +4,20 @@ describe('explorer', function() {
 
   describe('convert_to_satoshi', function() {
     it('should be able to convert round numbers', function() {
-      lib.convert_to_satoshi(500, function(amount_sat) {
+      const amount_sat = lib.convert_to_satoshi(500);
       expect(amount_sat).toEqual(50000000000);
     });
-    });
 
     it('should be able to convert decimals above 1', function() {
-      lib.convert_to_satoshi(500.12564, function(amount_sat) {
+      const amount_sat = lib.convert_to_satoshi(500.12564);
       expect(amount_sat).toEqual(50012564000);
     });
-    });
 
     it('should be able to convert decimals below 1', function() {
-      lib.convert_to_satoshi(0.0005, function(amount_sat) {
+      const amount_sat = lib.convert_to_satoshi(0.0005);
       expect(amount_sat).toEqual(50000);
     });
   });
-  });
 
   describe('is_unique', function() {
     var arrayStrMap = [
@@ -38,31 +35,23 @@ describe('explorer', function() {
     ];
 
     it('should return index of matching string object', function() {
-      lib.is_unique(arrayStrMap, arrayStrMap[2].addresses, 'addresses', function(unique, index) {
+      const index = lib.is_unique(arrayStrMap, arrayStrMap[2].addresses, 'addresses');
       expect(index).toEqual(2);
-        expect(unique).toEqual(false);
-      });
     });
 
     it('should return index of matching array object', function() {
-      lib.is_unique(arrayArrMap, arrayArrMap[2].addresses, 'addresses', function(unique, index) {
+      const index = lib.is_unique(arrayArrMap, arrayArrMap[2].addresses, 'addresses');
       expect(index).toEqual(2);
-        expect(unique).toEqual(false);
-      });
     });
 
-    it('should return true if no matching string object', function() {
-      lib.is_unique(arrayStrMap, 'unique', 'addresses', function(unique, index) {
-        expect(index).toEqual(null);
-        expect(unique).toEqual(true);
-      });
+    it('should return index of -1 if no matching string object', function() {
+      const index = lib.is_unique(arrayStrMap, 'unique', 'addresses');
+      expect(index).toEqual(-1);
     });
 
-    it('should return true if no matching array object', function() {
-      lib.is_unique(arrayArrMap, ['unique'], 'addresses', function(unique, index) {
-        expect(index).toEqual(null);
-        expect(unique).toEqual(true);
-      });
+    it('should return index of -1 if no matching array object', function() {
+      const index = lib.is_unique(arrayArrMap, ['unique'], 'addresses');
+      expect(index).toEqual(-1);
     });
   });
 
@@ -104,12 +93,11 @@ describe('explorer', function() {
 
     it('should calculate correct total', function(done) {
       lib.prepare_vout(data.txA().vout, data.txA().txid, data.txA().vin, ((typeof data.txA().vjoinsplit === 'undefined' || data.txA().vjoinsplit == null) ? [] : data.txA().vjoinsplit), function(prepared) {
-        lib.calculate_total(prepared, function(total) {
+        const total = lib.calculate_total(prepared)
         expect(total).toEqual(19499989908960);
         done();
       });
     });
-    });
 
     afterEach(function() {
       jasmine.DEFAULT_TIMEOUT_INTERVAL = originalTimeout;