purple-explorer/scripts/sync.js

var mongoose = require('mongoose'),
    lib = require('../lib/explorer'),
    db = require('../lib/database'),
    Tx = require('../models/tx'),
    Address = require('../models/address'),
    AddressTx = require('../models/addresstx'),
    Orphans = require('../models/orphans'),
    Richlist = require('../models/richlist'),
    Stats = require('../models/stats'),
    settings = require('../lib/settings'),
    async = require('async');
var mode = 'update';
var database = 'index';
var block_start = 1;
var lockCreated = false;
var stopSync = false;

// prevent stopping of the sync script to be able to gracefully shut down
process.on('SIGINT', () => {
  console.log('Stopping sync process.. Please wait..');
  stopSync = true;
});

// prevent killing of the sync script to be able to gracefully shut down
process.on('SIGTERM', () => {
  console.log('Stopping sync process.. Please wait..');
  stopSync = true;
});

// displays usage and exits
function usage() {
  console.log('Usage: /path/to/node scripts/sync.js [mode]');
  console.log('');
  console.log('Mode: (required)');
  console.log('update           Updates index from last sync to current block');
  console.log('check            Checks index for (and adds) any missing transactions/addresses');
  console.log('                 Optional parameter: block number to start checking from');
  console.log('reindex          Clears index then resyncs from genesis to current block');
  console.log('reindex-rich     Clears and recreates the richlist data');
  console.log('reindex-txcount  Rescan and flatten the tx count value for faster access');
  console.log('reindex-last     Rescan and flatten the last blockindex value for faster access');
  console.log('market           Updates market summaries, orderbooks, trade history + charts');
  console.log('peers            Updates peer info based on local wallet connections');
  console.log('masternodes      Updates the list of active masternodes on the network');
  console.log('');
  console.log('Notes:');
  console.log('- \'current block\' is the latest created block when script is executed.');
  console.log('- The market + peers databases only support (& defaults to) reindex mode.');
  console.log('- If check mode finds missing data (other than new data since last sync),');
  console.log('  this likely means that sync.update_timeout in settings.json is set too low.');
  console.log('');
  process.exit(100);
}

// exit function used to cleanup before finishing script
function exit(exitCode) {
  // always disconnect mongo connection
  mongoose.disconnect();

  // only remove sync lock if it was created in this session
  if (!lockCreated || lib.remove_lock(database) == true) {
    // clean exit with previous exit code
    process.exit(exitCode);
  } else {
    // error removing lock
    process.exit(1);
  }
}

// updates tx & address balances
function update_tx_db(coin, start, end, txes, timeout, check_only, cb) {
  var complete = false;
  var blocks_to_scan = [];
  var task_limit_blocks = settings.sync.block_parallel_tasks;
  var task_limit_txs = 1;

  // fix for invalid block height (skip genesis block as it should not have valid txs)
  if (typeof start === 'undefined' || start < 1)
    start = 1;

  if (task_limit_blocks < 1)
    task_limit_blocks = 1;

  for (i = start; i < (end + 1); i++)
    blocks_to_scan.push(i);

  async.eachLimit(blocks_to_scan, task_limit_blocks, function(block_height, next_block) {
    if (check_only == 0 && block_height % settings.sync.save_stats_after_sync_blocks === 0) {
      Stats.updateOne({coin: coin}, {
        last: block_height - 1,
        txes: txes
      }).then(() => {});
    } else if (check_only == 1) {
      console.log('Checking block ' + block_height + '...');
    }

    lib.get_blockhash(block_height, function(blockhash) {
      if (blockhash) {
        lib.get_block(blockhash, function(block) {
          if (block) {
            async.eachLimit(block.tx, task_limit_txs, function(txid, next_tx) {
              Tx.findOne({txid: txid}).then((tx) => {
                if (tx && check_only != 2) {
                  setTimeout(function() {
                    tx = null;

                    // check if the script is stopping
                    if (stopSync && check_only != 2) {
                      // stop the loop
                      next_tx({});
                    } else
                      next_tx();
                  }, timeout);
                } else {
                  // check if the transaction exists but doesn't match the current block height
                  check_delete_tx(tx, block_height, txes, timeout, function(updated_txes, tx_deleted) {
                    // update the running tx count
                    txes = updated_txes;

                    // check if this tx should be added to the local database
                    if (tx_deleted || !tx) {
                      // save the transaction to local database
                      db.save_tx(txid, block_height, function(err, tx_has_vout) {
                        if (err)
                          console.log(err);
                        else
                          console.log('%s: %s', block_height, txid);

                        if (tx_has_vout)
                          txes++;

                        setTimeout(function() {
                          tx = null;

                          // check if the script is stopping
                          if (stopSync && check_only != 2) {
                            // stop the loop
                            next_tx({});
                          } else
                            next_tx();
                        }, timeout);
                      });
                    } else {
                      // skip adding the current tx
                      setTimeout(function() {
                        tx = null;

                        // check if the script is stopping
                        if (stopSync && check_only != 2) {
                          // stop the loop
                          next_tx({});
                        } else
                          next_tx();
                      }, timeout);
                    }
                  });
                }
              }).catch((err) => {
                console.log(err);

                setTimeout(function() {
                  tx = null;

                  // check if the script is stopping
                  if (stopSync && check_only != 2) {
                    // stop the loop
                    next_tx({});
                  } else
                    next_tx();
                }, timeout);
              });
            }, function() {
              setTimeout(function() {
                blockhash = null;
                block = null;

                // check if the script is stopping
                if (stopSync && check_only != 2) {
                  // stop the loop
                  next_block({});
                } else
                  next_block();
              }, timeout);
            });
          } else {
            console.log('Block not found: %s', blockhash);

            setTimeout(function() {
              // check if the script is stopping
              if (stopSync && check_only != 2) {
                // stop the loop
                next_block({});
              } else
                next_block();
            }, timeout);
          }
        });
      } else {
        setTimeout(function() {
          // check if the script is stopping
          if (stopSync && check_only != 2) {
            // stop the loop
            next_block({});
          } else
            next_block();
        }, timeout);
      }
    });
  }, function() {
    var statUpdateObject = {};

    // check what stats data should be updated
    if (stopSync || check_only == 2) {
      // only update txes when fixing invalid and missing blocks or when a "normal" sync was stopped prematurely
      statUpdateObject.txes = txes;
    } else {
      // update last and txes values for "normal" sync that finishes without being stopped prematurely
      statUpdateObject = {
        txes: txes,
        last: end
      };
    }

    // update local stats
    Stats.updateOne({coin: coin}, statUpdateObject).then(() => {
      return cb(txes);
    }).catch((err) => {
      console.log(err);
      return cb(txes);
    });
  });
}

// fixes data belonging to orphaned blocks
function update_orphans(orphan_index, orphan_current, last_blockindex, timeout, cb) {
  // lookup the earliest orphaned block if this is the first time that orphans are being checked
  get_earliest_orphan_block(orphan_index, orphan_current, last_blockindex, function(orphan_data, err) {
    if (err != null) {
      console.log(err);
      return cb();
    } else {
      // re-populate the orphan data in case it has changed
      orphan_index = orphan_data.orphan_index;
      orphan_current = orphan_data.orphan_current;

      // Check if the sync msg should be shown
      check_show_sync_message(last_blockindex - orphan_index);

      // start from the current orphan (if exists) or else use the last orphan index
      let current_block = (orphan_current == 0 ? orphan_index : orphan_current);
      let unresolved_forks = [];
      let correct_block_data = null;

      // loop infinitely until finished iterating through all known blocks
      async.forever(function(next) {
        // check if the script is stopping or if the last block has been reached
        if (stopSync || current_block >= last_blockindex) {
          // stop the main loop
          next('stop');
        } else {
          if (unresolved_forks.length == 0 && current_block % settings.sync.save_stats_after_sync_blocks === 0) {
            Stats.updateOne({coin: settings.coin.name}, {
              orphan_index: current_block - 1,
              orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0])
            }).then(() => {}).catch((staterr) => {
              console.log(staterr);
            });
          }

          // do not show the 'checking...' msg if this block is about to be resolved
          if (correct_block_data == null)
            console.log(current_block.toString() + ': Checking for forks...');

          // check if there is a fork in this block
          check_block_height_for_fork(current_block, function(blockhashes, fork_err) {
            if (fork_err != null) {
              // an error occurred
              // stop looking for orphans
              next(fork_err);
            } else if (blockhashes == null) {
              // no forks found in this block
              // check if there are previously unresolved forks
              if (unresolved_forks.length == 0) {
                // move to the next block
                current_block++;

                setTimeout(function() {
                  // process next block
                  next(null);
                }, timeout);
              } else {
                // one or more forks still need to be resolved
                // set the current block to the block after the next orphaned block
                current_block = unresolved_forks[unresolved_forks.length - 1] + 1;

                // lookup the block hash using the block height
                lib.get_blockhash(current_block, function(block_hash) {
                  // check if the block hash was found
                  if (block_hash) {
                    // lookup the current block data from the wallet
                    lib.get_block(block_hash, function (block_data) {
                      // remember the correct block hashes
                      correct_block_data = {
                        prev_hash: block_data.previousblockhash,
                        next_hash: block_data.hash
                      };

                      console.log('Good ' + current_block.toString() + ' block found. Returning to fix block ' + unresolved_forks[unresolved_forks.length -1].toString());

                      // go back to the last unresolved fork
                      current_block = unresolved_forks.pop();

                      setTimeout(function() {
                        // process next block
                        next(null);
                      }, timeout);
                    });
                  } else {
                    // an error occurred
                    // stop looking for orphans
                    next('cannot find block ' + current_block.toString());
                  }
                });
              }
            } else {
              // there is at least one fork in this block height
              // check if the correct block hash is known
              if (correct_block_data != null) {
                // this fork needs to be resolved
                // lookup the current block data from the wallet
                lib.get_block(correct_block_data.prev_hash, function (block_data) {
                  var tx_count = 0;

                  // check if the good block hash is in the list of blockhashes
                  if (blockhashes.indexOf(correct_block_data.prev_hash) > -1) {
                    // remove the good block hash
                    blockhashes.splice(blockhashes.indexOf(correct_block_data.prev_hash), 1);
                  }

                  // loop through the remaining orphaned block hashes
                  lib.syncLoop(blockhashes.length, function(block_loop) {
                    var i = block_loop.iteration();

                    console.log('Resolving orphaned block [' + (i + 1).toString() + '/' + blockhashes.length.toString() + ']: ' + blockhashes[i]);

                    // find all orphaned txid's from the current orphan block hash
                    get_orphaned_txids(blockhashes[i], function(txids) {
                      // save the orphan block data to the orphan collection
                      create_orphan(current_block, blockhashes[i], correct_block_data.prev_hash, block_data.previousblockhash, correct_block_data.next_hash, function() {
                        // loop through the remaining orphaned block hashes
                        lib.syncLoop(txids.length, function(tx_loop) {
                          var t = tx_loop.iteration();

                          // remove the orphaned tx and cleanup all associated data
                          delete_and_cleanup_tx(txids[t], current_block, tx_count, timeout, function(updated_tx_count) {
                            // update the running tx count
                            tx_count = updated_tx_count;

                            // some blockchains will reuse the same orphaned transaction ids
                            // lookup the transaction that was just deleted to ensure it doesn't belong to another block
                            check_add_tx(txids[t], blockhashes[i], tx_count, function(updated_tx_count2) {
                              // update the running tx count
                              tx_count = updated_tx_count2;

                              setTimeout(function() {
                                // move to the next tx record
                                tx_loop.next();
                              }, timeout);
                            });
                          });
                        }, function() {
                          setTimeout(function() {
                            // move to the next block record
                            block_loop.next();
                          }, timeout);
                        });
                      });
                    });
                  }, function() {
                    // get the most recent stats
                    Stats.findOne({coin: settings.coin.name}).then((stats) => {
                      // add missing txes for the current block
                      update_tx_db(settings.coin.name, current_block, current_block, (stats.txes + tx_count), timeout, 2, function(updated_tx_count) {
                        // update the stats collection by removing the orphaned txes in this block from the tx count
                        // and setting the orphan_index and orphan_current values in case the sync is interrupted before finishing
                        Stats.updateOne({coin: settings.coin.name}, {
                          orphan_index: current_block,
                          orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0])
                        }).then(() => {
                          // clear the saved block hash data
                          correct_block_data = null;

                          // move to the next block
                          current_block++;

                          setTimeout(function() {
                            // process next block
                            next(null);
                          }, timeout);
                        }).catch((err) => {
                          console.log(err);

                          // clear the saved block hash data
                          correct_block_data = null;

                          // move to the next block
                          current_block++;

                          setTimeout(function() {
                            // process next block
                            next(null);
                          }, timeout);
                        });
                      });
                    }).catch((err) => {
                      console.log(err);

                      setTimeout(function() {
                        // process next block
                        next(null);
                      }, timeout);
                    });
                  });
                });
              } else {
                // correct block hash is unknown
                console.log('Block ' + current_block.toString() + ' has ' + (blockhashes.length - 1).toString() + ' unresolved fork' + (blockhashes.length - 1 == 1 ? '' : 's'));

                // add this block height to the list of unresolved forks
                unresolved_forks.push(current_block);

                // move to the next block
                current_block++;

                setTimeout(function() {
                  // process next block
                  next(null);
                }, timeout);
              }
            }
          });
        }
      },
      function(err) {
        // check if there is a msg to display
        if (err != '' && err != 'stop') {
          // display the msg
          console.log(err);

          // stop fixing orphaned block data
          return cb();
        } else {
          // check if the script is stopping
          if (!stopSync)
            console.log('Finished looking for forks');

          // get the list of orphans with a null next_blockhash
          Orphans.find({next_blockhash: null}).exec().then((orphans) => {
            // loop through the list of orphans
            lib.syncLoop(orphans.length, function(orphan_loop) {
              var o = orphan_loop.iteration();

              // lookup the block data from the wallet
              lib.get_block(orphans[o].good_blockhash, function (good_block_data) {
                // check if the next block hash is known
                if (good_block_data.nextblockhash != null) {
                  // update the next blockhash for this orphan record
                  Orphans.updateOne({blockindex: orphans[o].blockindex, orphan_blockhash: orphans[o].orphan_blockhash}, {
                    next_blockhash: good_block_data.nextblockhash
                  }).then(() => {
                    setTimeout(function() {
                      // move to the next orphan record
                      orphan_loop.next();
                    }, timeout);
                  }).catch((err) => {
                    console.log(err);

                    setTimeout(function() {
                      // move to the next orphan record
                      orphan_loop.next();
                    }, timeout);
                  });
                } else {
                  setTimeout(function() {
                    // move to the next orphan record
                    orphan_loop.next();
                  }, timeout);
                }
              });
            }, function() {
              // update the orphan stats
              Stats.updateOne({coin: settings.coin.name}, {
                orphan_index: current_block - 1,
                orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0])
              }).then(() => {
                // finished fixing orphaned block data
                return cb();
              }).catch((err) => {
                console.log(err);
                return cb();
              });
            });
          }).catch((err) => {
            console.log(err);
            return cb();
          });
        }
      });
    }
  });
}

function get_earliest_orphan_block(orphan_index, orphan_current, last_blockindex, cb) {
  // check if it is necessary to search for orphan data
  if (orphan_index == null || orphan_index == 0) {
    console.log('Finding the earliest orphaned blockindex.. Please wait..');

    Tx.aggregate([
      { $match: {
        "vout": { $exists: true, $ne: [] }
      }},
      { $group: {
        _id: "$blockindex",
        blockhashes: { $addToSet: "$blockhash" }
      }},
      { $match: {
        "blockhashes.1": { "$exists": true }
      }},
      { $sort: {
        "_id": 1
      }},
      { $limit: 1 },
      { $project: {
        "_id": 1
      }}
    ]).option({ allowDiskUse: true }).then((data) => {
      if (data.length > 0) {
        // found the first unprocessed orphaned block
        orphan_current = data[0]._id;
        orphan_index = (orphan_current - 1);
        console.log('Found orphan block index: ' + orphan_current.toString());

        Stats.updateOne({coin: settings.coin.name}, {
          orphan_current: orphan_current,
          orphan_index: orphan_index
        }).then(() => {
          return cb({orphan_index: orphan_index, orphan_current: orphan_current}, null);
        });
      } else {
        // no unprocessed orphaned blocks found
        orphan_current = 0;
        orphan_index = last_blockindex;
        console.log('No orphaned blocks found');

        Stats.updateOne({coin: settings.coin.name}, {
          orphan_current: orphan_current,
          orphan_index: orphan_index
        }).then(() => {
          return cb({orphan_index: orphan_index, orphan_current: orphan_current}, null);
        });
      }
    }).catch((err) => {
      return cb(null, err);
    });
  } else
    return cb({orphan_index: orphan_index, orphan_current: orphan_current}, null);
}

function check_block_height_for_fork(block_height, cb) {
  // find all unique blockhashes in the txes collections for this block height
  Tx.aggregate([
    { $match: {
      $and: [
        {"blockindex": { $eq: block_height }},
        {"vout": { $exists: true, $ne: [] }}
      ]
    }},
    { $group: {
      _id: "$blockindex",
      blockhashes: { $addToSet: "$blockhash" }
    }}
  ]).then((data) => {
    if (data.length > 0) {
      // lookup the "good" block hash using the block height
      lib.get_blockhash(block_height, function(block_hash) {
        // check if there is more than 1 block hash
        if (data[0].blockhashes.length == 1) {
          // 1 block found
          // check if the found block is the good block
          if (data[0].blockhashes[0] == block_hash) {
            // no forks found
            return cb(null, null);
          } else {
            // add the good block to the list of blockhashes
            data[0].blockhashes.push(block_hash);

            // return the block hashes
            return cb(data[0].blockhashes, null);
          }
        } else {
          // more than 1 block found
          // check if the good block is already in the list
          if (data[0].blockhashes.indexOf(block_hash) == -1) {
            // add the good block to the list of blockhashes
            data[0].blockhashes.push(block_hash);
          }

          // return the block hashes
          return cb(data[0].blockhashes, null);
        }
      });
    } else {
      // no blocks found
      return cb(null, null);
    }
  }).catch((err) => {
    // an error was returned
    return cb(null, err);
  });
}

function create_orphan(blockindex, orphan_blockhash, good_blockhash, prev_blockhash, next_blockhash, cb) {
  var newOrphan = new Orphans({
    blockindex: blockindex,
    orphan_blockhash: orphan_blockhash,
    good_blockhash: good_blockhash,
    prev_blockhash: prev_blockhash,
    next_blockhash: next_blockhash
  });

  // create a new orphan record in the local database
  newOrphan.save().then(() => {
    // new orphan record saved successfully
    return cb();
  }).catch((err) => {
    // check if this is a duplicate key error which can be ignored
    if (!(err.toString().indexOf('E11000') > -1 || err.toString().indexOf('duplicate key error') > -1))
      console.log(err);

    return cb();
  });
}

function get_orphaned_txids(block_hash, cb) {
  // get all transactions by block hash
  Tx.find({blockhash: block_hash}).exec().then((txes) => {
    if (txes.length > 0) {
      // found at least one orphaned transaction
      var txids = [];

      // populate an array of txids without the object data
      for (t = 0; t < txes.length; t++)
        txids.push(txes[t].txid);

      return cb(txids, null);
    } else {
      // no txes found
      return cb(null, null);
    }
  }).catch((err) => {
    // an error was returned
    return cb(null, err);
  });
}

function check_add_tx(txid, blockhash, tx_count, cb) {
  // lookup the transaction to ensure it doesn't belong to another block
  lib.get_rawtransaction(txid, function(tx) {
    // check if this txid belongs to the main blockchain
    if (tx && tx.txid && tx.blockhash != blockhash && tx.confirmations > 0) {
      // lookup the correct block index in case it is not the same as the current block
      lib.get_block(tx.blockhash, function(block) {
        // check if the block was found
        if (block) {
          // save the tx to the local database
          db.save_tx(txid, block.height, function(save_tx_err, tx_has_vout) {
            // check if there were any save errors
            if (save_tx_err)
              console.log(save_tx_err);
            else
              console.log('%s: %s', block.height, txid);

            // check if the tx was saved correctly
            if (tx_has_vout) {
              // keep a running total of txes that were added
              tx_count++;
            }

            return cb(tx_count);
          });
        } else {
          // block not found so there is nothing to fix
          return cb(tx_count);
        }
      });
    } else {
      // block does not belong to main blockchain so there is nothing to fix
      return cb(tx_count);
    }
  });
}

function delete_and_cleanup_tx(txid, block_height, tx_count, timeout, cb) {
  // lookup all address tx records associated with the current tx
  AddressTx.find({txid: txid}).exec().then((address_txes) => {
    if (address_txes.length == 0) {
      // no vouts for this tx, so just delete the tx without cleaning up addresses
      delete_tx(txid, block_height, function(tx_err, tx_result) {
        if (tx_err) {
          console.log(tx_err);
          return cb(tx_count);
        } else {
          // NOTE: do not subtract from the tx_count here because only txes with vouts are counted
          return cb(tx_count);
        }
      });
    } else {
      // lookup the current tx in the local database
      Tx.findOne({txid: txid}).then((tx) => {
        var addressTxArray = [];
        var has_vouts = (tx.vout != null && tx.vout.length > 0);

        // check if this is a coinbase tx
        if (tx.vin == null || tx.vin.length == 0) {
          // add a coinbase tx into the addressTxArray array
          addressTxArray.push({
            txid: txid,
            a_id: 'coinbase',
            amount: tx.total
          });
        }

        // check if there are any vin addresses
        if (tx.vin != null && tx.vin.length > 0) {
          // loop through the vin data
          for (var vin_tx_counter = tx.vin.length - 1; vin_tx_counter >= 0; vin_tx_counter--) {
            // loop through the addresstxe data
            for (var vin_addresstx_counter = address_txes.length - 1; vin_addresstx_counter >= 0; vin_addresstx_counter--) {
              // check if there is a tx record that exactly matches to the addresstx
              if (tx.vin[vin_tx_counter].addresses == address_txes[vin_addresstx_counter].a_id && tx.vin[vin_tx_counter].amount == -address_txes[vin_addresstx_counter].amount) {
                // add the address into the addressTxArray array
                addressTxArray.push({
                  txid: txid,
                  a_id: tx.vin[vin_tx_counter].addresses,
                  amount: address_txes[vin_addresstx_counter].amount
                });

                // remove the found records from both arrays
                tx.vin.splice(vin_tx_counter, 1);
                address_txes.splice(vin_addresstx_counter, 1);

                break;
              }
            }
          }
        }

        // check if there are any vout addresses
        if (tx.vout != null && tx.vout.length > 0) {
          // loop through the vout data
          for (var vout_tx_counter = tx.vout.length - 1; vout_tx_counter >= 0; vout_tx_counter--) {
            // loop through the addresstxe data
            for (var vout_addresstx_counter = address_txes.length - 1; vout_addresstx_counter >= 0; vout_addresstx_counter--) {
              // check if there is a tx record that exactly matches to the addresstx
              if (tx.vout[vout_tx_counter].addresses == address_txes[vout_addresstx_counter].a_id && tx.vout[vout_tx_counter].amount == address_txes[vout_addresstx_counter].amount) {
                // add the address into the addressTxArray array
                addressTxArray.push({
                  txid: txid,
                  a_id: tx.vout[vout_tx_counter].addresses,
                  amount: address_txes[vout_addresstx_counter].amount
                });

                // remove the found records from both arrays
                tx.vout.splice(vout_tx_counter, 1);
                address_txes.splice(vout_addresstx_counter, 1);

                break;
              }
            }
          }
        }

        // check if there are still more vin/vout records to process
        if (tx.vin.length > 0 || tx.vout.length > 0 || address_txes.length > 0) {
          // get all unique remaining addresses
          var address_list = [];

          // get unique addresses from the tx vin
          tx.vin.forEach(function(vin) {
            if (address_list.indexOf(vin.addresses) == -1)
              address_list.push(vin.addresses);
          });

          // get unique addresses from the tx vout
          tx.vout.forEach(function(vout) {
            if (address_list.indexOf(vout.addresses) == -1)
              address_list.push(vout.addresses);
          });

          // get unique addresses from the addresstxes
          address_txes.forEach(function(address_tx) {
            if (address_list.indexOf(address_tx.a_id) == -1)
              address_list.push(address_tx.a_id);
          });

          // loop through each unique address
          address_list.forEach(function(address) {
            var vin_total = 0;
            var vout_total = 0;
            var address_tx_total = 0;

            // add up all the vin amounts for this address
            tx.vin.forEach(function(vin) {
              // check if this is the correct address
              if (vin.addresses == address)
                vin_total += vin.amount;
            });

            // add up all the vout amounts for this address
            tx.vout.forEach(function(vout) {
              // check if this is the correct address
              if (vout.addresses == address)
                vout_total += vout.amount;
            });

            // add up all the addresstx amounts for this address
            address_txes.forEach(function(address_tx) {
              // check if this is the correct address
              if (address_tx.a_id == address)
                address_tx_total += address_tx.amount;
            });

            // check if the tx and addresstx totals match
            if ((vout_total - vin_total) == address_tx_total) {
              // the values match (this indicates that this address sent coins to themselves)
              // add a vin record for this address into the addressTxArray array
              addressTxArray.push({
                txid: txid,
                a_id: address,
                amount: -vin_total
              });

              // add a vout record for this address into the addressTxArray array
              addressTxArray.push({
                txid: txid,
                a_id: address,
                amount: vout_total
              });
            } else {
              // the values do not match (this indicates there was a problem saving the data)
              // output the data for this address as-is, using the addresstx values
              address_txes.forEach(function(address_tx) {
                // check if this is the correct address
                if (address_tx.a_id == address) {
                  // add a record for this address into the addressTxArray array
                  addressTxArray.push({
                    txid: txid,
                    a_id: address,
                    amount: address_tx.amount
                  });
                }
              });
            }
          });
        }

        // loop through the address txes
        lib.syncLoop(addressTxArray.length, function(address_loop) {
          var a = address_loop.iteration();

           // fix the balance, sent and received data for the current address
          fix_address_data(addressTxArray[a], function() {
            setTimeout(function() {
              // move to the next address record
              address_loop.next();
            }, timeout);
          });
        }, function() {
          // delete all AddressTx records from the local collection for this tx
          AddressTx.deleteMany({txid: txid}).then((address_tx_result) => {
            // delete the tx from the local database
            delete_tx(txid, block_height, function(tx_err, tx_result) {
              if (tx_err) {
                console.log(tx_err);
                return cb(tx_count);
              } else {
                // check if the deleted tx had vouts
                if (has_vouts) {
                  // keep a running total of txes that were removed
                  tx_count -= tx_result.deletedCount;
                }

                return cb(tx_count);
              }
            });
          }).catch((err) => {
            console.log(err);

            // delete the tx from the local database
            delete_tx(txid, block_height, function(tx_err, tx_result) {
              if (tx_err) {
                console.log(tx_err);
                return cb(tx_count);
              } else {
                // check if the deleted tx had vouts
                if (has_vouts) {
                  // keep a running total of txes that were removed
                  tx_count -= tx_result.deletedCount;
                }

                return cb(tx_count);
              }
            });
          });
        });
      }).catch((err) => {
        console.log(err);
        return cb(tx_count);
      });
    }
  }).catch((err) => {
    console.log(err);
    return cb(tx_count);
  });
}

function fix_address_data(address_data, cb) {
  var addr_inc = {};
  var amount = address_data.amount;

  // determine how to fix the address balances
  if (address_data.a_id == 'coinbase')
    addr_inc.sent = -amount;
  else if (amount < 0) {
    // vin
    addr_inc.sent = amount;
    addr_inc.balance = -amount;
  } else {
    // vout
    addr_inc.received = -amount;
    addr_inc.balance = -amount;
  }

  // reverse the amount from the running totals in the Address collection for the current address
  Address.findOneAndUpdate({a_id: address_data.a_id}, {
    $inc: addr_inc
  }, {
    upsert: false
  }).then((return_address) => {
    // finished fixing the address balance data 
    return cb();
  }).catch((err) => {
    console.log(err);
    return cb();
  });
}

function delete_tx(txid, block_height, cb) {
  // delete the tx from the local database
  Tx.deleteOne({txid: txid, blockindex: block_height}).then((tx_result) => {
    return cb(null, tx_result);
  }).catch((err) => {
    return cb(err, null);
  });
}

function check_delete_tx(tx, block_height, tx_count, timeout, cb) {
  // check if the tx object exists and does not match the current block height
  if (tx && tx.blockindex != block_height) {
    // the transaction exists but does not match the correct block height, therefore it should be deleted
    delete_and_cleanup_tx(tx.txid, tx.blockindex, tx_count, timeout, function(updated_tx_count) {
      // finished removing the transaction
      return cb(updated_tx_count, true);
    });
  } else {
    // tx dosn't exist or block heights match so nothing to do
    return cb(tx_count, false);
  }
}

function update_heavy(coin, height, count, heavycoin_enabled, cb) {
  if (heavycoin_enabled == true) {
    db.update_heavy(coin, height, count, function() {
      return cb(true);
    });
  } else
    return cb(false);
}

function update_network_history(height, network_history_enabled, cb) {
  if (network_history_enabled == true) {
    db.update_network_history(height, function() {
      return cb(true);
    });
  } else
    return cb(false);
}

function check_show_sync_message(blocks_to_sync) {
  var retVal = false;
  var filePath = './tmp/show_sync_message.tmp';
  // Check if the sync msg should be shown
  if (blocks_to_sync > settings.sync.show_sync_msg_when_syncing_more_than_blocks) {
    // Check if the show sync stub file already exists
    if (!db.fs.existsSync(filePath)) {
      // File doesn't exist, so create it now
      db.fs.writeFileSync(filePath, '');
    }

    retVal = true;
  }

  return retVal;
}

function get_market_price(market_array) {
  // check how the market price should be updated
  if (settings.markets_page.market_price == 'COINGECKO') {
    // find the coingecko id
    find_coingecko_id(settings.coin.symbol, function(coingecko_id) {
      // check if the coingecko_id was found
      if (coingecko_id != null && coingecko_id != '') {
        const coingecko = require('../lib/apis/coingecko');
        const currency = lib.get_market_currency_code();

        console.log('Calculating market price.. Please wait..');

        // get the market price from coingecko api
        coingecko.get_market_prices(coingecko_id, currency, function (err, last_price, last_usd_price) {
          // check for errors
          if (err == null) {
            // get current stats
            Stats.findOne({coin: settings.coin.name}).then((stats) => {
              // update market stat prices
              Stats.updateOne({coin: settings.coin.name}, {
                last_price: (last_price == null ? 0 : last_price),
                last_usd_price: (last_usd_price == null ? 0 : last_usd_price)
              }).then(() => {
                // market prices updated successfully
                finish_market_sync();
              }).catch((err) => {
                // error saving stats
                console.log(err);
                exit(1);
              });
            }).catch((err) => {
              // error getting stats
              console.log(err);
              exit(1);
            });
          } else {
            // coingecko api returned an error
            console.log(err);
            exit(1);
          }
        });
      } else {
        // coingecko_id is not set which should have already thrown an error, so just exit
        exit(1);
      }
    });
  } else {
    console.log('Calculating market price.. Please wait..');

    // get the list of coins from coingecko
    coingecko_coin_list_api(market_array, function (coin_err, coin_list) {
      // check for errors
      if (coin_err == null) {
        let api_ids = '';

        // loop through all unique currencies in the market_array
        for (let m = 0; m < market_array.length; m++) {
          const index = coin_list.findIndex(p => p.symbol.toLowerCase() == market_array[m].currency.toLowerCase());

          // check if the market currency is found in the coin list
          if (index > -1) {
            // add to the list of api_ids
            api_ids += (api_ids == '' ? '' : ',') + coin_list[index].id;

            // add the coingecko id back to the market_array
            market_array[m].coingecko_id = coin_list[index].id;
          } else {
            // coin symbol not found in the api
            console.log('Error: Cannot find symbol "' + market_array[m].currency + '" in the coingecko api');
          }
        }

        // check if any api_ids were found
        if (api_ids != '') {
          const coingecko = require('../lib/apis/coingecko');
          const currency = lib.get_market_currency_code();

          // get the market price from coingecko api
          coingecko.get_avg_market_prices(api_ids, currency, market_array, function (mkt_err, last_price, last_usd) {   
            // check for errors
            if (mkt_err == null) {
              // update the last usd price
              Stats.updateOne({coin: settings.coin.name}, {
                last_price: last_price,
                last_usd_price: last_usd
              }).then(() => {
                // market price updated successfully
                finish_market_sync();
              }).catch((err) => {
                // error saving stat data
                console.log(err);
                exit(1);
              });
            } else {
              // coingecko api returned an error
              console.log(mkt_err);
              exit(1);
            }
          });
        } else {
          // no api_ids found so cannot continue to getting the usd price and error msgs were already thrown, so just exit
          exit(1);
        }
      } else {
        // coingecko api returned an error
        console.log(coin_err);
        exit(1);
      }
    });
  }
}

function finish_market_sync() {
  // update markets_last_updated value
  db.update_last_updated_stats(settings.coin.name, { markets_last_updated: Math.floor(new Date() / 1000) }, function() {
    // check if the script stopped prematurely
    if (stopSync) {
      console.log('Market sync was stopped prematurely');
      exit(1);
    } else {
      console.log('Market sync complete');
      exit(0);
    }
  });
}

function coingecko_coin_list_api(market_symbols, cb) {
  let coin_array = [];
  let call_coin_list_api = false;

  // check if market_symbols is an array
  if (!Array.isArray(market_symbols)) {
    // add this symbol to an array
    market_symbols = [{currency: market_symbols}];
  }

  // loop through all symbols
  for (var symbol of market_symbols) {
    // check if this symbol has a default coingecko id in the settings
    const index = settings.default_coingecko_ids.findIndex(p => p.symbol.toLowerCase() == symbol.currency.toLowerCase());

    // check if the coin symbol is found in settings
    if (index > -1) {
      // add this symbol and id to a new array
      coin_array.push({
        id: settings.default_coingecko_ids[index].id.toLowerCase(),
        symbol: symbol.currency.toLowerCase()
      });
    } else {
      // missing at least 1 symbol, so the coingecko api must be called
      call_coin_list_api = true;
      break;
    }
  }

  // check if the coin list api needs to be called
  if (call_coin_list_api) {
    const coingecko = require('../lib/apis/coingecko');

    // get the list of coins from coingecko
    coingecko.get_coin_data(function (err, coin_list) {
      // check if there was an error
      if (err == null) {
          // initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis
          const rateLimitLib = require('../lib/ratelimit');
          const rateLimit = new rateLimitLib.RateLimit(1, 2000, false);

          // automatically pause for 2 seconds in between requests
          rateLimit.schedule(function() {
            return cb(err, coin_list);
          });
      } else {
        return cb(err, coin_list);
      }
    });
  } else {
    // return the custom array of known symbols and ids
    return cb(null, coin_array);
  }
}

function find_coingecko_id(symbol, cb) {
  coingecko_coin_list_api(symbol, function (err, coin_list) {
    // check for errors
    if (err == null) {
      // find the index of the first coin symbol match
      const index = coin_list.findIndex(p => p.symbol.toLowerCase() == symbol.toLowerCase());

      // check if the coin symbol is found in the api coin list
      if (index > -1)
        return cb(coin_list[index].id);
      else {
        // coin symbol not found in the api
        console.log('Error: Cannot find symbol "' + symbol + '" in the coingecko api');
        return cb('');
      }
    } else {
      // failed to get the coingecko api list
      console.log(err);
      return cb('');
    }
  });
}

/** Function that count occurrences of a substring in a string;
 * @param {String} string               The string
 * @param {String} subString            The sub string to search for
 * @param {Boolean} [allowOverlapping]  Optional. (Default:false)
 *
 * @author Vitim.us https://gist.github.com/victornpb/7736865
 * @see Unit Test https://jsfiddle.net/Victornpb/5axuh96u/
 * @see http://stackoverflow.com/questions/4009756/how-to-count-string-occurrence-in-string/7924240#7924240
 */
function occurrences(string, subString, allowOverlapping) {
  string += "";
  subString += "";
  if (subString.length <= 0) return (string.length + 1);

  var n = 0,
      pos = 0,
      step = allowOverlapping ? 1 : subString.length;

  while (true) {
      pos = string.indexOf(subString, pos);
      if (pos >= 0) {
          ++n;
          pos += step;
      } else break;
  }
  return n;
}

function block_sync(reindex, stats) {
  // Get the last synced block index value
  var last = (stats.last ? stats.last : 0);
  // Get the total number of blocks
  var count = (stats.count ? stats.count : 0);

  // Check if the sync msg should be shown
  check_show_sync_message(count - last);

  update_tx_db(settings.coin.name, last, count, stats.txes, settings.sync.update_timeout, 0, function() {
    // check if the script stopped prematurely
    if (stopSync) {
      console.log(`${(reindex ? 'Reindex' : 'Block sync')} was stopped prematurely`);
      exit(1);
    } else {
      // update blockchain_last_updated value
      db.update_last_updated_stats(settings.coin.name, { blockchain_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
        // fix data from orphaned blocks
        update_orphans(stats.orphan_index, stats.orphan_current, count, settings.sync.update_timeout, function() {
          // check if the script stopped prematurely
          if (stopSync) {
            console.log(`${(reindex ? 'Reindex' : 'Block sync')} was stopped prematurely`);
            exit(1);
          } else {
            db.update_richlist('received', function() {
              db.update_richlist('balance', function() {
                // update richlist_last_updated value
                db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) {                              
                  db.get_stats(settings.coin.name, function(nstats) {
                    // check for and update heavycoin data if applicable
                    update_heavy(settings.coin.name, count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) {
                      // check for and update network history data if applicable
                      update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) {
                        // always check for and remove the sync msg if exists
                        db.remove_sync_message();

                        console.log(`${(reindex ? 'Reindex' : 'Block sync')} complete (block: %s)`, nstats.last);
                        exit(0);
                      });
                    });
                  });
                });
              });
            });
          }
        });
      });
    }
  });
}

// check options
if (process.argv[2] == null || process.argv[2] == 'index' || process.argv[2] == 'update') {
  mode = null;

  switch (process.argv[3]) {
    case undefined:
    case null:
    case 'update':
      mode = 'update';
      break;
    case 'check':
      mode = 'check';

      // check if the block start value was passed in and is an integer
      if (!isNaN(process.argv[4]) && Number.isInteger(parseFloat(process.argv[4]))) {
        // Check if the block start value is less than 1
        if (parseInt(process.argv[4]) < 1)
          block_start = 1;
        else
          block_start = parseInt(process.argv[4]);
      }

      break;
    case 'reindex':
      mode = 'reindex';
      break;
    case 'reindex-rich':
      mode = 'reindex-rich';
      break;
    case 'reindex-txcount':
      mode = 'reindex-txcount';
      break;
    case 'reindex-last':
      mode = 'reindex-last';
      break;
    default:
      usage();
  }
} else if (process.argv[2] == 'peers' || process.argv[2] == 'masternodes')
  database = process.argv[2];
else if (process.argv[2] == 'market')
  database = `${process.argv[2]}s`;
else
  usage();

// check if this sync option is already running/locked
if (lib.is_locked([database]) == false) {
  // create a new sync lock before checking the rest of the locks to minimize problems with running scripts at the same time
  lib.create_lock(database);
  // ensure the lock will be deleted on exit
  lockCreated = true;
  // check the backup, restore and delete locks since those functions would be problematic when updating data
  if (lib.is_locked(['backup', 'restore', 'delete']) == false) {
    // all tests passed. OK to run sync
    console.log("Script launched with pid: " + process.pid);

    if (mode == 'update')
      console.log(`Syncing ${(database == 'index' ? 'blocks' : database)}.. Please wait..`);

    var dbString = 'mongodb://' + encodeURIComponent(settings.dbsettings.user);
    dbString = dbString + ':' + encodeURIComponent(settings.dbsettings.password);
    dbString = dbString + '@' + settings.dbsettings.address;
    dbString = dbString + ':' + settings.dbsettings.port;
    dbString = dbString + '/' + settings.dbsettings.database;

    mongoose.set('strictQuery', true);

    mongoose.connect(dbString).then(() => {
      if (database == 'index') {
        db.check_stats(settings.coin.name, function(exists) {
          if (exists == false) {
            console.log('Run \'npm start\' to create database structures before running this script.');
            exit(1);
          } else {
            // determine which index mode to run
            if (mode == 'reindex') {
              const { execSync } = require('child_process');

              try {
                // delete the database
                execSync(`node ./scripts/delete_database.js ${mode}`, {stdio : 'inherit'});
              } catch (err) {
                // delete_database.js was not successful, so exit
                exit(1);
              }

              db.update_db(settings.coin.name, function(stats) {
                // check if stats returned properly
                if (stats !== false) {
                  // start the block sync
                  block_sync(true, stats);
                } else {
                  // update_db threw an error so exit
                  exit(1);
                }
              });
            } else if (mode == 'check') {
              db.update_db(settings.coin.name, function(stats) {
                // check if stats returned properly
                if (stats !== false) {
                  console.log('Checking blocks.. Please wait..');

                  update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, 1, function() {
                    // check if the script stopped prematurely
                    if (stopSync) {
                      console.log('Block check was stopped prematurely');
                      exit(1);
                    } else {
                      db.get_stats(settings.coin.name, function(nstats) {
                        console.log('Block check complete (block: %s)', nstats.last);
                        exit(0);
                      });
                    }
                  });
                } else {
                  // update_db threw an error so exit
                  exit(1);
                }
              });
            } else if (mode == 'update') {
              db.update_db(settings.coin.name, function(stats) {
                // check if stats returned properly
                if (stats !== false) {
                  // start the block sync
                  block_sync(false, stats);
                } else {
                  // update_db threw an error so exit
                  exit(1);
                }
              });
            } else if (mode == 'reindex-rich') {
              db.update_db(settings.coin.name, function(stats) {
                // check if stats returned properly
                if (stats !== false) {
                  console.log('Check richlist');

                  db.check_richlist(settings.coin.name, function(exists) {
                    if (exists)
                      console.log('Richlist entry found, deleting now..');

                    db.delete_richlist(settings.coin.name, function(deleted) {
                      if (deleted)
                        console.log('Richlist entry deleted');

                      db.create_richlist(settings.coin.name, false, function() {
                        console.log('Richlist created');

                        db.update_richlist('received', function() {
                          console.log('Richlist updated received');

                          db.update_richlist('balance', function() {
                            // update richlist_last_updated value
                            db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
                              console.log('Richlist update complete');
                              exit(0);
                            });
                          });
                        });
                      });
                    });
                  });
                } else {
                  // update_db threw an error so exit
                  exit(1);
                }
              });
            } else if (mode == 'reindex-txcount') {
              db.update_db(settings.coin.name, function(stats) {
                // check if stats returned properly
                if (stats !== false) {
                  console.log('Calculating tx count.. Please wait..');

                  // Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout
                  Tx.find({'total': {$gte: 0}, 'vout': { $gte: { $size: 1 }}}).countDocuments().then((count) => {
                    console.log('Found tx count: ' + count.toString());
                    Stats.updateOne({coin: settings.coin.name}, {
                      txes: count
                    }).then(() => {
                      console.log('Tx count update complete');
                      exit(0);
                    }).catch((err) => {
                      console.log(err);
                      exit(1);
                    });
                  }).catch((err) => {
                    console.log(err);
                    exit(1);
                  });
                } else {
                  // update_db threw an error so exit
                  exit(1);
                }
              });
            } else if (mode == 'reindex-last') {
              db.update_db(settings.coin.name, function(stats) {
                // check if stats returned properly
                if (stats !== false) {
                  console.log('Finding last blockindex.. Please wait..');

                  // Resetting the last blockindex counter requires a single lookup on the txes collection to find the last indexed blockindex
                  Tx.find({}, {blockindex:1, _id:0}).sort({blockindex: -1}).limit(1).exec().then((tx) => {
                    // check if any blocks exists
                    if (tx == null || tx.length == 0) {
                      console.log('No blocks found. setting last blockindex to 0.');

                      Stats.updateOne({coin: settings.coin.name}, {
                        last: 0
                      }).then(() => {
                        console.log('Last blockindex update complete');
                        exit(0);
                      }).catch((err) => {
                        console.log(err);
                        exit(1);
                      });
                    } else {
                      console.log('Found last blockindex: ' + tx[0].blockindex.toString());

                      Stats.updateOne({coin: settings.coin.name}, {
                        last: tx[0].blockindex
                      }).then(() => {
                        console.log('Last blockindex update complete');
                        exit(0);
                      }).catch((err) => {
                        console.log(err);
                        exit(1);
                      });
                    }
                  }).catch((err) => {
                    console.log(err);
                    exit(1);
                  });
                } else {
                  // update_db threw an error so exit
                  exit(1);
                }
              });
            }
          }
        });
      } else if (database == 'peers') {
        lib.get_peerinfo(function(body) {
          if (body != null) {
            lib.syncLoop(body.length, function(loop) {
              var i = loop.iteration();
              var address = body[i].addr;
              var port = null;

              if (occurrences(address, ':') == 1 || occurrences(address, ']:') == 1) {
                // Separate the port # from the IP address
                address = address.substring(0, address.lastIndexOf(":")).replace("[", "").replace("]", "");
                port = body[i].addr.substring(body[i].addr.lastIndexOf(":") + 1);
              }

              if (address.indexOf("]") > -1) {
                // Remove [] characters from IPv6 addresses
                address = address.replace("[", "").replace("]", "");
              }

              db.find_peer(address, port, function(peer) {
                if (peer) {
                  if (peer['port'] != null && (isNaN(peer['port']) || peer['port'].length < 2)) {
                    db.drop_peers(function() {
                      console.log('Removing peers due to missing port information. Re-run this script to add peers again.');
                      exit(1);
                    });
                  }

                  // peer already exists and should be refreshed
                  // drop peer
                  db.drop_peer(address, port, function() {
                    // re-add the peer to refresh the data and extend the expiry date
                    db.create_peer({
                      address: address,
                      port: port,
                      protocol: peer.protocol,
                      version: peer.version,
                      country: peer.country,
                      country_code: peer.country_code
                    }, function() {
                      console.log('Updated peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString());

                      // check if the script is stopping
                      if (stopSync) {
                        // stop the loop
                        loop.break(true);
                      }

                      loop.next();
                    });
                  });
                } else {
                  const rateLimitLib = require('../lib/ratelimit');
                  const rateLimit = new rateLimitLib.RateLimit(1, 2000, false);

                  rateLimit.schedule(function() {
                    lib.get_geo_location(address, function(error, geo) {
                      // check if an error was returned
                      if (error) {
                        console.log(error);
                        exit(1);
                      } else if (geo == null || typeof geo != 'object') {
                        console.log('Error: geolocation api did not return a valid object');
                        exit(1);
                      } else {
                        // add peer to collection
                        db.create_peer({
                          address: address,
                          port: port,
                          protocol: body[i].version,
                          version: body[i].subver.replace('/', '').replace('/', ''),
                          country: geo.country_name,
                          country_code: geo.country_code
                        }, function() {
                          console.log('Added new peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString());

                          // check if the script is stopping
                          if (stopSync) {
                            // stop the loop
                            loop.break(true);
                          }

                          loop.next();
                        });
                      }
                    });
                  });
                }
              });
            }, function() {
              // update network_last_updated value
              db.update_last_updated_stats(settings.coin.name, { network_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
                // check if the script stopped prematurely
                if (stopSync) {
                  console.log('Peer sync was stopped prematurely');
                  exit(1);
                } else {
                  console.log('Peer sync complete');
                  exit(0);
                }
              });
            });
          } else {
            console.log('No peers found');
            exit(2);
          }
        });
      } else if (database == 'masternodes') {
        lib.get_masternodelist(function(body) {
          if (body != null) {
            var isObject = false;
            var objectKeys = null;

            // Check if the masternode data is an array or an object
            if (body.length == null) {
              // Process data as an object
              objectKeys = Object.keys(body);
              isObject = true;
            }

            lib.syncLoop((isObject ? objectKeys : body).length, function(loop) {
              var i = loop.iteration();

              db.save_masternode((isObject ? body[objectKeys[i]] : body[i]), function(success) {
                if (success) {
                  // check if the script is stopping
                  if (stopSync) {
                    // stop the loop
                    loop.break(true);
                  }

                  loop.next();
                } else {
                  console.log('Error: Cannot save masternode %s.', (isObject ? (body[objectKeys[i]].payee ? body[objectKeys[i]].payee : 'UNKNOWN') : (body[i].addr ? body[i].addr : 'UNKNOWN')));
                  exit(1);
                }
              });
            }, function() {
              db.remove_old_masternodes(function(cb) {
                db.update_last_updated_stats(settings.coin.name, { masternodes_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
                  // check if the script stopped prematurely
                  if (stopSync) {
                    console.log('Masternode sync was stopped prematurely');
                    exit(1);
                  } else {
                    console.log('Masternode sync complete');
                    exit(0);
                  }
                });
              });
            });
          } else {
            console.log('No masternodes found');
            exit(2);
          }
        });
      } else {
        // start market sync
        // check if market feature is enabled or the market_price option is set to COINGECKO
        if (settings.markets_page.enabled == true || settings.markets_page.market_price == 'COINGECKO') {
          var total_pairs = 0;
          var exchanges = Object.keys(settings.markets_page.exchanges);

          // loop through all exchanges to determine how many trading pairs must be updated
          exchanges.forEach(function(key, index, map) {
            // check if market is enabled via settings
            if (settings.markets_page.enabled == true && settings.markets_page.exchanges[key].enabled == true) {
              // check if market is installed/supported
              if (db.fs.existsSync('./lib/markets/' + key + '.js')) {
                // add trading pairs to total
                total_pairs += settings.markets_page.exchanges[key].trading_pairs.length;

                // loop through all trading pairs for this market
                for (var i = 0; i < settings.markets_page.exchanges[key].trading_pairs.length; i++) {
                  // ensure trading pair setting is always uppercase
                  settings.markets_page.exchanges[key].trading_pairs[i] = settings.markets_page.exchanges[key].trading_pairs[i].toUpperCase();
                }
              }
            }
          });

          // check if there are any trading pairs to update
          if (total_pairs > 0) {
            let market_array = [];

            // initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis
            var rateLimitLib = require('../lib/ratelimit');
            var rateLimit = new rateLimitLib.RateLimit(1, 2000, false);
            var complete = 0;

            // loop through and test all exchanges defined in the settings.json file
            exchanges.forEach(function(key, index, map) {
              // check if market is enabled via settings
              if (settings.markets_page.exchanges[key].enabled == true) {
                // check if market is installed/supported
                if (db.fs.existsSync('./lib/markets/' + key + '.js')) {
                  // loop through all trading pairs
                  settings.markets_page.exchanges[key].trading_pairs.forEach(function(pair_key, pair_index, pair_map) {
                    // split the pair data
                    var split_pair = pair_key.split('/');
                    // check if this is a valid trading pair
                    if (split_pair.length == 2) {
                      // lookup the exchange in the market collection
                      db.check_market(key, split_pair[0], split_pair[1], function(mkt, exists) {
                        // check if exchange trading pair exists in the market collection
                        if (exists) {
                          // automatically pause for 2 seconds in between requests
                          rateLimit.schedule(function() {
                            // update market data
                            db.update_markets_db(key, split_pair[0], split_pair[1], function(err, last_price) {
                              if (!err) {
                                console.log('%s[%s]: Market data updated successfully', key, pair_key);

                                // only add to the market_array if market data is being averaged
                                if (settings.markets_page.market_price == 'AVERAGE') {
                                  // check if the currency already exists in the market array
                                  const index = market_array.findIndex(item => item.currency.toUpperCase() == split_pair[1].toUpperCase());

                                  if (index != -1) {
                                    // update the last_price
                                    market_array[index].last_price = (market_array[index].last_price + last_price) / 2;
                                  } else {
                                    // add new object to the array
                                    market_array.push({currency: split_pair[1], last_price: last_price});
                                  }
                                }
                              } else
                                console.log('%s[%s] Error: %s', key, pair_key, err);

                              complete++;

                              if (complete == total_pairs || stopSync)
                                get_market_price(market_array);
                            });
                          });
                        } else {
                          console.log('%s[%s] Error: Market not found in local database. Please restart the explorer', key, pair_key);
                          complete++;

                          if (complete == total_pairs || stopSync)
                            get_market_price(market_array);
                        }
                      });
                    } else {
                      // market pair not formatted correctly
                      console.log('%s market pair is invalid', pair_key);
                      complete++;

                      if (complete == total_pairs || stopSync)
                        get_market_price(market_array);
                    }
                  });
                } else {
                  // market not installed
                  console.log('%s market not installed', key);
                  complete++;

                  if (complete == total_pairs || stopSync)
                    get_market_price(market_array);
                }
              }
            });
          } else if (settings.markets_page.market_price == 'COINGECKO')
            get_market_price([]);
          else {
            // no market trading pairs are enabled
            console.log('Error: No market trading pairs are enabled in settings');
            exit(1);
          }
        } else {
          // market page is not enabled
          console.log('Error: Market feature is disabled in settings');
          exit(1);
        }
      }
    }).catch((err) => {
      console.log('Error: Unable to connect to database: %s', err);
      exit(1);
    });
  } else {
    // another script process is currently running
    console.log("Sync aborted");
    exit(2);
  }
} else {
  // sync process is already running
  console.log("Sync aborted");
  exit(2);
}