From 7349560bfaf545746711e028abf166cb82b2098c Mon Sep 17 00:00:00 2001 From: Joe Uhren Date: Sun, 1 Oct 2023 19:10:52 -0600 Subject: [PATCH] Reindex and delete-database script improvements -The delete-database script now checks the claimaddresses collection for data and if at least 1 record exists, it will ask an additional question to see if you want to preserve that data or delete everything -The delete-database script now issues the prompt for the reindex with a slightly different msg than the delete without reindex -The prompt for deleting now happens after the locks have been properly checked --- scripts/delete_database.js | 178 +++++--- scripts/sync.js | 849 ++++++++++++++++++------------------- 2 files changed, 528 insertions(+), 499 deletions(-) diff --git a/scripts/delete_database.js b/scripts/delete_database.js index acc0d47..4afa64b 100644 --- a/scripts/delete_database.js +++ b/scripts/delete_database.js @@ -2,6 +2,7 @@ const lib = require('../lib/explorer'); const readline = require('readline'); const deleteLockName = 'delete'; var lockCreated = false; +var preserveClaimAddressNames = false; // exit function used to cleanup lock before finishing script function exit(mongoose, exitCode) { @@ -61,14 +62,26 @@ function drop_collection(mongoose, colName, cb) { } function delete_prompt(cb) { - // Check if the delete prompt should be skipped - if (process.argv[2] == null || process.argv[2] != 'reindex') { + preserve_claimaddress_prompt(function() { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - console.log('You are about to delete the entire eIquidus database.'); + // Change the delete prompt based on whether this is a reindex or regular delete + if (process.argv[2] != null && process.argv[2] == 'reindex') { + console.log('You are about to delete all data from the entire eIquidus database'); + + if (preserveClaimAddressNames) + console.log('(claim address name data will not be deleted)'); + + console.log('and resync from the genesis block.'); + } else { + console.log('You are about to delete all data from the entire eIquidus database.'); + + if (preserveClaimAddressNames) + console.log('(claim address name data will not be deleted)'); + } // prompt for deleting explorer database rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer) { @@ -88,43 +101,87 @@ function delete_prompt(cb) { return cb(false); } }); - } else { - // skip the delete prompt - return cb(true); - } + }); } -delete_prompt(function(continue_process) { - if (continue_process) { - // check if the "delete database" process is already running - if (lib.is_locked([deleteLockName]) == false) { - // create a new delete lock before checking the rest of the locks to minimize problems with running scripts at the same time - lib.create_lock(deleteLockName); - // ensure the lock will be deleted on exit - lockCreated = true; +function preserve_claimaddress_prompt(cb) { + const ClaimAddress = require('../models/claimaddress'); - var lock_list = ['backup', 'restore', 'markets', 'peers', 'masternodes']; + // check how many claim address records there are + ClaimAddress.find({}).countDocuments().then((count) => { + // display an additional prompt in the event the claimaddress collection has data + if (count > 0) { + console.log(`The current database has ${count} custom claim address names`); + console.log('Would you like to preserve this data?'); - // do not check the index lock if this is called from the reindex process - if (process.argv[2] == null || process.argv[2] != 'reindex') { - lock_list.push('index'); - } + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); - // check all other possible locks since database deletion should not run at the same time that data is being changed - if (lib.is_locked(lock_list) == false) { - // all tests passed. OK to run delete - console.log("Script launched with pid: " + process.pid); + // prompt for deleting claim address data + rl.question('y = keep claim address data, n = delete claim address data [y/n]: ', function (preserveClaimAddresses) { + // stop prompting + rl.close(); - const settings = require('../lib/settings'); - const mongoose = require('mongoose'); - const dbString = `mongodb://${encodeURIComponent(settings.dbsettings.user)}:${encodeURIComponent(settings.dbsettings.password)}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`; + // determine if the claim address data should be preserved + switch (preserveClaimAddresses) { + case 'y': + case 'Y': + case 'yes': + case 'YES': + case 'Yes': + preserveClaimAddressNames = true; + console.log('Claim address name data will be saved' + '\n'); + break; + default: + console.log('Claim address name data will be deleted' + '\n'); + } - console.log('Connecting to database..'); + return cb(); + }); + } else + return cb(); + }); +} - mongoose.set('strictQuery', true); +// check if the "delete database" process is already running +if (lib.is_locked([deleteLockName]) == false) { + // create a new delete lock before checking the rest of the locks to minimize problems with running scripts at the same time + lib.create_lock(deleteLockName); + // ensure the lock will be deleted on exit + lockCreated = true; - // connect to mongo database - mongoose.connect(dbString).then(() => { + var lock_list = ['backup', 'restore', 'markets', 'peers', 'masternodes']; + + // do not check the index lock if this is called from the reindex process + if (process.argv[2] == null || process.argv[2] != 'reindex') { + lock_list.push('index'); + } + + // check all other possible locks since database deletion should not run at the same time that data is being changed + if (lib.is_locked(lock_list) == false) { + // all tests passed. OK to run delete + + // suppress the pid message when doing a reindex + if (process.argv[2] == null || process.argv[2] != 'reindex') + console.log("Script launched with pid: " + process.pid); + + const settings = require('../lib/settings'); + const mongoose = require('mongoose'); + const dbString = `mongodb://${encodeURIComponent(settings.dbsettings.user)}:${encodeURIComponent(settings.dbsettings.password)}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`; + + console.log('Connecting to database..'); + + mongoose.set('strictQuery', true); + + // connect to mongo database + mongoose.connect(dbString).then(() => { + console.log('Database connection successful' + '\n'); + + // prompt for database delete + delete_prompt(function(continue_process) { + if (continue_process) { // get the list of collections mongoose.connection.db.listCollections().toArray().then((collections) => { // check if there are any collections @@ -133,13 +190,26 @@ delete_prompt(function(continue_process) { // loop through all collections collections.forEach((collection) => { - console.log(`Deleting ${collection.name}..`); + // check if this is the claim addres collection and that data is being preserved + if (!preserveClaimAddressNames || collection.name != 'claimaddresses') { + console.log(`Deleting ${collection.name}..`); - // delete this collection - drop_collection(mongoose, collection.name, function(retVal) { - // check if the collection was successfully deleted - if (retVal) - counter++; + // delete this collection + drop_collection(mongoose, collection.name, function(retVal) { + // check if the collection was successfully deleted + if (retVal) + counter++; + + // check if the last collection was deleted + if (counter == collections.length) { + // finish the delete process + console.log('Finished deleting database'); + exit(mongoose, 0); + } + }); + } else { + // skipped deleting of the claimaddresses collection + counter++; // check if the last collection was deleted if (counter == collections.length) { @@ -147,7 +217,7 @@ delete_prompt(function(continue_process) { console.log('Finished deleting database'); exit(mongoose, 0); } - }); + } }); } else { // nothing to delete @@ -160,22 +230,22 @@ delete_prompt(function(continue_process) { console.log('Error: Unable to list collections in database: %s', err); exit(mongoose, 1); }); - }).catch((err) => { - console.log('Error: Unable to connect to database: %s', err); - exit(mongoose, 999); - }); - } else { - // another script process is currently running - console.log("Delete aborted"); - exit(null, 2); - } - } else { - // delete process is already running - console.log("Delete aborted"); - exit(null, 2); - } + } else { + console.log('Process aborted. Nothing was deleted.'); + exit(null, 2); + } + }); + }).catch((err) => { + console.log('Error: Unable to connect to database: %s', err); + exit(mongoose, 999); + }); } else { - console.log('Process aborted. Nothing was deleted.'); + // another script process is currently running + console.log("Delete aborted"); exit(null, 2); } -}); \ No newline at end of file +} else { + // delete process is already running + console.log("Delete aborted"); + exit(null, 2); +} \ No newline at end of file diff --git a/scripts/sync.js b/scripts/sync.js index 4cecf8d..5544fbc 100644 --- a/scripts/sync.js +++ b/scripts/sync.js @@ -1116,41 +1116,6 @@ function block_sync(reindex, stats) { }); } -function check_reindex_prompt(cb) { - // check if this is a reindex - if (mode == 'reindex') { - const readline = require('readline'); - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }); - - console.log('You are about to delete all blockchain data (transactions and addresses)'); - console.log('and resync from the genesis block.'); - - // prompt for reindexing the database - rl.question('Are you sure you want to do this? [y/n]: ', function (reindexAnswer) { - // stop prompting - rl.close(); - - // determine if the reindex should proceed - switch (reindexAnswer) { - case 'y': - case 'Y': - case 'yes': - case 'YES': - case 'Yes': - return cb(true); - break; - default: - return cb(false); - } - }); - } else { - return cb(true); - } -} - // check options if (process.argv[2] == null || process.argv[2] == 'index' || process.argv[2] == 'update') { mode = null; @@ -1196,242 +1161,275 @@ else if (process.argv[2] == 'market') else usage(); -// check if this is a reindex which requires a prompt before continuing -check_reindex_prompt(function(reindexAnswer) { - // check if the process should continue - if (reindexAnswer) { - // check if this sync option is already running/locked - if (lib.is_locked([database]) == false) { - // create a new sync lock before checking the rest of the locks to minimize problems with running scripts at the same time - lib.create_lock(database); - // ensure the lock will be deleted on exit - lockCreated = true; - // check the backup, restore and delete locks since those functions would be problematic when updating data - if (lib.is_locked(['backup', 'restore', 'delete']) == false) { - // all tests passed. OK to run sync - console.log("Script launched with pid: " + process.pid); +// check if this sync option is already running/locked +if (lib.is_locked([database]) == false) { + // create a new sync lock before checking the rest of the locks to minimize problems with running scripts at the same time + lib.create_lock(database); + // ensure the lock will be deleted on exit + lockCreated = true; + // check the backup, restore and delete locks since those functions would be problematic when updating data + if (lib.is_locked(['backup', 'restore', 'delete']) == false) { + // all tests passed. OK to run sync + console.log("Script launched with pid: " + process.pid); - if (mode == 'update') - console.log(`Syncing ${(database == 'index' ? 'blocks' : database)}.. Please wait..`); + if (mode == 'update') + console.log(`Syncing ${(database == 'index' ? 'blocks' : database)}.. Please wait..`); - var dbString = 'mongodb://' + encodeURIComponent(settings.dbsettings.user); - dbString = dbString + ':' + encodeURIComponent(settings.dbsettings.password); - dbString = dbString + '@' + settings.dbsettings.address; - dbString = dbString + ':' + settings.dbsettings.port; - dbString = dbString + '/' + settings.dbsettings.database; + var dbString = 'mongodb://' + encodeURIComponent(settings.dbsettings.user); + dbString = dbString + ':' + encodeURIComponent(settings.dbsettings.password); + dbString = dbString + '@' + settings.dbsettings.address; + dbString = dbString + ':' + settings.dbsettings.port; + dbString = dbString + '/' + settings.dbsettings.database; - mongoose.set('strictQuery', true); + mongoose.set('strictQuery', true); - mongoose.connect(dbString).then(() => { - if (database == 'index') { - db.check_stats(settings.coin.name, function(exists) { - if (exists == false) { - console.log('Run \'npm start\' to create database structures before running this script.'); + mongoose.connect(dbString).then(() => { + if (database == 'index') { + db.check_stats(settings.coin.name, function(exists) { + if (exists == false) { + console.log('Run \'npm start\' to create database structures before running this script.'); + exit(1); + } else { + // determine which index mode to run + if (mode == 'reindex') { + const { execSync } = require('child_process'); + + try { + // delete the database + execSync(`node ./scripts/delete_database.js ${mode}`, {stdio : 'inherit'}); + } catch (err) { + // delete_database.js was not successful, so exit exit(1); - } else { - // determine which index mode to run - if (mode == 'reindex') { - const { execSync } = require('child_process'); + } - console.log('Deleting database.. Please wait..'); + db.update_db(settings.coin.name, function(stats) { + // check if stats returned properly + if (stats !== false) { + // start the block sync + block_sync(true, stats); + } else { + // update_db threw an error so exit + exit(1); + } + }); + } else if (mode == 'check') { + db.update_db(settings.coin.name, function(stats) { + // check if stats returned properly + if (stats !== false) { + console.log('Checking blocks.. Please wait..'); - // delete the database - execSync(`node ./scripts/delete_database.js ${mode}`, {stdio : 'inherit'}); - - db.update_db(settings.coin.name, function(stats) { - // check if stats returned properly - if (stats !== false) { - // start the block sync - block_sync(true, stats); - } else { - // update_db threw an error so exit + update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, 1, function() { + // check if the script stopped prematurely + if (stopSync) { + console.log('Block check was stopped prematurely'); exit(1); - } - }); - } else if (mode == 'check') { - db.update_db(settings.coin.name, function(stats) { - // check if stats returned properly - if (stats !== false) { - console.log('Checking blocks.. Please wait..'); - - update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, 1, function() { - // check if the script stopped prematurely - if (stopSync) { - console.log('Block check was stopped prematurely'); - exit(1); - } else { - db.get_stats(settings.coin.name, function(nstats) { - console.log('Block check complete (block: %s)', nstats.last); - exit(0); - }); - } + } else { + db.get_stats(settings.coin.name, function(nstats) { + console.log('Block check complete (block: %s)', nstats.last); + exit(0); }); - } else { - // update_db threw an error so exit - exit(1); } }); - } else if (mode == 'update') { - db.update_db(settings.coin.name, function(stats) { - // check if stats returned properly - if (stats !== false) { - // start the block sync - block_sync(false, stats); - } else { - // update_db threw an error so exit - exit(1); - } - }); - } else if (mode == 'reindex-rich') { - db.update_db(settings.coin.name, function(stats) { - // check if stats returned properly - if (stats !== false) { - console.log('Check richlist'); + } else { + // update_db threw an error so exit + exit(1); + } + }); + } else if (mode == 'update') { + db.update_db(settings.coin.name, function(stats) { + // check if stats returned properly + if (stats !== false) { + // start the block sync + block_sync(false, stats); + } else { + // update_db threw an error so exit + exit(1); + } + }); + } else if (mode == 'reindex-rich') { + db.update_db(settings.coin.name, function(stats) { + // check if stats returned properly + if (stats !== false) { + console.log('Check richlist'); - db.check_richlist(settings.coin.name, function(exists) { - if (exists) - console.log('Richlist entry found, deleting now..'); + db.check_richlist(settings.coin.name, function(exists) { + if (exists) + console.log('Richlist entry found, deleting now..'); - db.delete_richlist(settings.coin.name, function(deleted) { - if (deleted) - console.log('Richlist entry deleted'); + db.delete_richlist(settings.coin.name, function(deleted) { + if (deleted) + console.log('Richlist entry deleted'); - db.create_richlist(settings.coin.name, false, function() { - console.log('Richlist created'); + db.create_richlist(settings.coin.name, false, function() { + console.log('Richlist created'); - db.update_richlist('received', function() { - console.log('Richlist updated received'); + db.update_richlist('received', function() { + console.log('Richlist updated received'); - db.update_richlist('balance', function() { - // update richlist_last_updated value - db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - console.log('Richlist update complete'); - exit(0); - }); - }); + db.update_richlist('balance', function() { + // update richlist_last_updated value + db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) { + console.log('Richlist update complete'); + exit(0); }); }); }); }); - } else { - // update_db threw an error so exit - exit(1); - } - }); - } else if (mode == 'reindex-txcount') { - db.update_db(settings.coin.name, function(stats) { - // check if stats returned properly - if (stats !== false) { - console.log('Calculating tx count.. Please wait..'); - - // Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout - Tx.find({'total': {$gte: 0}, 'vout': { $gte: { $size: 1 }}}).countDocuments().then((count) => { - console.log('Found tx count: ' + count.toString()); - Stats.updateOne({coin: settings.coin.name}, { - txes: count - }).then(() => { - console.log('Tx count update complete'); - exit(0); - }).catch((err) => { - console.log(err); - exit(1); - }); - }).catch((err) => { - console.log(err); - exit(1); - }); - } else { - // update_db threw an error so exit - exit(1); - } - }); - } else if (mode == 'reindex-last') { - db.update_db(settings.coin.name, function(stats) { - // check if stats returned properly - if (stats !== false) { - console.log('Finding last blockindex.. Please wait..'); - - // Resetting the last blockindex counter requires a single lookup on the txes collection to find the last indexed blockindex - Tx.find({}, {blockindex:1, _id:0}).sort({blockindex: -1}).limit(1).exec().then((tx) => { - // check if any blocks exists - if (tx == null || tx.length == 0) { - console.log('No blocks found. setting last blockindex to 0.'); - - Stats.updateOne({coin: settings.coin.name}, { - last: 0 - }).then(() => { - console.log('Last blockindex update complete'); - exit(0); - }).catch((err) => { - console.log(err); - exit(1); - }); - } else { - console.log('Found last blockindex: ' + tx[0].blockindex.toString()); - - Stats.updateOne({coin: settings.coin.name}, { - last: tx[0].blockindex - }).then(() => { - console.log('Last blockindex update complete'); - exit(0); - }).catch((err) => { - console.log(err); - exit(1); - }); - } - }).catch((err) => { - console.log(err); - exit(1); - }); - } else { - // update_db threw an error so exit - exit(1); - } + }); }); + } else { + // update_db threw an error so exit + exit(1); } + }); + } else if (mode == 'reindex-txcount') { + db.update_db(settings.coin.name, function(stats) { + // check if stats returned properly + if (stats !== false) { + console.log('Calculating tx count.. Please wait..'); + + // Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout + Tx.find({'total': {$gte: 0}, 'vout': { $gte: { $size: 1 }}}).countDocuments().then((count) => { + console.log('Found tx count: ' + count.toString()); + Stats.updateOne({coin: settings.coin.name}, { + txes: count + }).then(() => { + console.log('Tx count update complete'); + exit(0); + }).catch((err) => { + console.log(err); + exit(1); + }); + }).catch((err) => { + console.log(err); + exit(1); + }); + } else { + // update_db threw an error so exit + exit(1); + } + }); + } else if (mode == 'reindex-last') { + db.update_db(settings.coin.name, function(stats) { + // check if stats returned properly + if (stats !== false) { + console.log('Finding last blockindex.. Please wait..'); + + // Resetting the last blockindex counter requires a single lookup on the txes collection to find the last indexed blockindex + Tx.find({}, {blockindex:1, _id:0}).sort({blockindex: -1}).limit(1).exec().then((tx) => { + // check if any blocks exists + if (tx == null || tx.length == 0) { + console.log('No blocks found. setting last blockindex to 0.'); + + Stats.updateOne({coin: settings.coin.name}, { + last: 0 + }).then(() => { + console.log('Last blockindex update complete'); + exit(0); + }).catch((err) => { + console.log(err); + exit(1); + }); + } else { + console.log('Found last blockindex: ' + tx[0].blockindex.toString()); + + Stats.updateOne({coin: settings.coin.name}, { + last: tx[0].blockindex + }).then(() => { + console.log('Last blockindex update complete'); + exit(0); + }).catch((err) => { + console.log(err); + exit(1); + }); + } + }).catch((err) => { + console.log(err); + exit(1); + }); + } else { + // update_db threw an error so exit + exit(1); + } + }); + } + } + }); + } else if (database == 'peers') { + lib.get_peerinfo(function(body) { + if (body != null) { + lib.syncLoop(body.length, function(loop) { + var i = loop.iteration(); + var address = body[i].addr; + var port = null; + + if (occurrences(address, ':') == 1 || occurrences(address, ']:') == 1) { + // Separate the port # from the IP address + address = address.substring(0, address.lastIndexOf(":")).replace("[", "").replace("]", ""); + port = body[i].addr.substring(body[i].addr.lastIndexOf(":") + 1); } - }); - } else if (database == 'peers') { - lib.get_peerinfo(function(body) { - if (body != null) { - lib.syncLoop(body.length, function(loop) { - var i = loop.iteration(); - var address = body[i].addr; - var port = null; - if (occurrences(address, ':') == 1 || occurrences(address, ']:') == 1) { - // Separate the port # from the IP address - address = address.substring(0, address.lastIndexOf(":")).replace("[", "").replace("]", ""); - port = body[i].addr.substring(body[i].addr.lastIndexOf(":") + 1); + if (address.indexOf("]") > -1) { + // Remove [] characters from IPv6 addresses + address = address.replace("[", "").replace("]", ""); + } + + db.find_peer(address, port, function(peer) { + if (peer) { + if (peer['port'] != null && (isNaN(peer['port']) || peer['port'].length < 2)) { + db.drop_peers(function() { + console.log('Removing peers due to missing port information. Re-run this script to add peers again.'); + exit(1); + }); } - if (address.indexOf("]") > -1) { - // Remove [] characters from IPv6 addresses - address = address.replace("[", "").replace("]", ""); - } + // peer already exists and should be refreshed + // drop peer + db.drop_peer(address, port, function() { + // re-add the peer to refresh the data and extend the expiry date + db.create_peer({ + address: address, + port: port, + protocol: peer.protocol, + version: peer.version, + country: peer.country, + country_code: peer.country_code + }, function() { + console.log('Updated peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString()); - db.find_peer(address, port, function(peer) { - if (peer) { - if (peer['port'] != null && (isNaN(peer['port']) || peer['port'].length < 2)) { - db.drop_peers(function() { - console.log('Removing peers due to missing port information. Re-run this script to add peers again.'); - exit(1); - }); + // check if the script is stopping + if (stopSync) { + // stop the loop + loop.break(true); } - // peer already exists and should be refreshed - // drop peer - db.drop_peer(address, port, function() { - // re-add the peer to refresh the data and extend the expiry date + loop.next(); + }); + }); + } else { + const rateLimitLib = require('../lib/ratelimit'); + const rateLimit = new rateLimitLib.RateLimit(1, 2000, false); + + rateLimit.schedule(function() { + lib.get_geo_location(address, function(error, geo) { + // check if an error was returned + if (error) { + console.log(error); + exit(1); + } else if (geo == null || typeof geo != 'object') { + console.log('Error: geolocation api did not return a valid object'); + exit(1); + } else { + // add peer to collection db.create_peer({ address: address, port: port, - protocol: peer.protocol, - version: peer.version, - country: peer.country, - country_code: peer.country_code + protocol: body[i].version, + version: body[i].subver.replace('/', '').replace('/', ''), + country: geo.country_name, + country_code: geo.country_code }, function() { - console.log('Updated peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString()); + console.log('Added new peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString()); // check if the script is stopping if (stopSync) { @@ -1441,223 +1439,184 @@ check_reindex_prompt(function(reindexAnswer) { loop.next(); }); - }); - } else { - const rateLimitLib = require('../lib/ratelimit'); - const rateLimit = new rateLimitLib.RateLimit(1, 2000, false); - - rateLimit.schedule(function() { - lib.get_geo_location(address, function(error, geo) { - // check if an error was returned - if (error) { - console.log(error); - exit(1); - } else if (geo == null || typeof geo != 'object') { - console.log('Error: geolocation api did not return a valid object'); - exit(1); - } else { - // add peer to collection - db.create_peer({ - address: address, - port: port, - protocol: body[i].version, - version: body[i].subver.replace('/', '').replace('/', ''), - country: geo.country_name, - country_code: geo.country_code - }, function() { - console.log('Added new peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString()); - - // check if the script is stopping - if (stopSync) { - // stop the loop - loop.break(true); - } - - loop.next(); - }); - } - }); - }); - } - }); - }, function() { - // update network_last_updated value - db.update_last_updated_stats(settings.coin.name, { network_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - // check if the script stopped prematurely - if (stopSync) { - console.log('Peer sync was stopped prematurely'); - exit(1); - } else { - console.log('Peer sync complete'); - exit(0); - } - }); - }); - } else { - console.log('No peers found'); - exit(2); - } - }); - } else if (database == 'masternodes') { - lib.get_masternodelist(function(body) { - if (body != null) { - var isObject = false; - var objectKeys = null; - - // Check if the masternode data is an array or an object - if (body.length == null) { - // Process data as an object - objectKeys = Object.keys(body); - isObject = true; - } - - lib.syncLoop((isObject ? objectKeys : body).length, function(loop) { - var i = loop.iteration(); - - db.save_masternode((isObject ? body[objectKeys[i]] : body[i]), function(success) { - if (success) { - // check if the script is stopping - if (stopSync) { - // stop the loop - loop.break(true); - } - - loop.next(); - } else { - console.log('Error: Cannot save masternode %s.', (isObject ? (body[objectKeys[i]].payee ? body[objectKeys[i]].payee : 'UNKNOWN') : (body[i].addr ? body[i].addr : 'UNKNOWN'))); - exit(1); - } - }); - }, function() { - db.remove_old_masternodes(function(cb) { - db.update_last_updated_stats(settings.coin.name, { masternodes_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - // check if the script stopped prematurely - if (stopSync) { - console.log('Masternode sync was stopped prematurely'); - exit(1); - } else { - console.log('Masternode sync complete'); - exit(0); } }); }); + } + }); + }, function() { + // update network_last_updated value + db.update_last_updated_stats(settings.coin.name, { network_last_updated: Math.floor(new Date() / 1000) }, function(cb) { + // check if the script stopped prematurely + if (stopSync) { + console.log('Peer sync was stopped prematurely'); + exit(1); + } else { + console.log('Peer sync complete'); + exit(0); + } + }); + }); + } else { + console.log('No peers found'); + exit(2); + } + }); + } else if (database == 'masternodes') { + lib.get_masternodelist(function(body) { + if (body != null) { + var isObject = false; + var objectKeys = null; + + // Check if the masternode data is an array or an object + if (body.length == null) { + // Process data as an object + objectKeys = Object.keys(body); + isObject = true; + } + + lib.syncLoop((isObject ? objectKeys : body).length, function(loop) { + var i = loop.iteration(); + + db.save_masternode((isObject ? body[objectKeys[i]] : body[i]), function(success) { + if (success) { + // check if the script is stopping + if (stopSync) { + // stop the loop + loop.break(true); + } + + loop.next(); + } else { + console.log('Error: Cannot save masternode %s.', (isObject ? (body[objectKeys[i]].payee ? body[objectKeys[i]].payee : 'UNKNOWN') : (body[i].addr ? body[i].addr : 'UNKNOWN'))); + exit(1); + } + }); + }, function() { + db.remove_old_masternodes(function(cb) { + db.update_last_updated_stats(settings.coin.name, { masternodes_last_updated: Math.floor(new Date() / 1000) }, function(cb) { + // check if the script stopped prematurely + if (stopSync) { + console.log('Masternode sync was stopped prematurely'); + exit(1); + } else { + console.log('Masternode sync complete'); + exit(0); + } }); - } else { - console.log('No masternodes found'); - exit(2); + }); + }); + } else { + console.log('No masternodes found'); + exit(2); + } + }); + } else { + // check if market feature is enabled + if (settings.markets_page.enabled == true) { + var complete = 0; + var total_pairs = 0; + var exchanges = Object.keys(settings.markets_page.exchanges); + + // loop through all exchanges to determine how many trading pairs must be updated + exchanges.forEach(function(key, index, map) { + // check if market is enabled via settings + if (settings.markets_page.exchanges[key].enabled == true) { + // check if market is installed/supported + if (db.fs.existsSync('./lib/markets/' + key + '.js')) { + // add trading pairs to total + total_pairs += settings.markets_page.exchanges[key].trading_pairs.length; + + // loop through all trading pairs for this market + for (var i = 0; i < settings.markets_page.exchanges[key].trading_pairs.length; i++) { + // ensure trading pair setting is always uppercase + settings.markets_page.exchanges[key].trading_pairs[i] = settings.markets_page.exchanges[key].trading_pairs[i].toUpperCase(); + } + } + } + }); + + // check if there are any trading pairs to update + if (total_pairs > 0) { + // initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis + var rateLimitLib = require('../lib/ratelimit'); + var rateLimit = new rateLimitLib.RateLimit(1, 2000, false); + // loop through and test all exchanges defined in the settings.json file + exchanges.forEach(function(key, index, map) { + // check if market is enabled via settings + if (settings.markets_page.exchanges[key].enabled == true) { + // check if market is installed/supported + if (db.fs.existsSync('./lib/markets/' + key + '.js')) { + // loop through all trading pairs + settings.markets_page.exchanges[key].trading_pairs.forEach(function(pair_key, pair_index, pair_map) { + // split the pair data + var split_pair = pair_key.split('/'); + // check if this is a valid trading pair + if (split_pair.length == 2) { + // lookup the exchange in the market collection + db.check_market(key, split_pair[0], split_pair[1], function(mkt, exists) { + // check if exchange trading pair exists in the market collection + if (exists) { + // automatically pause for 2 seconds in between requests + rateLimit.schedule(function() { + // update market data + db.update_markets_db(key, split_pair[0], split_pair[1], function(err) { + if (!err) { + console.log('%s[%s]: Market data updated successfully.', key, pair_key); + complete++; + + if (complete == total_pairs || stopSync) + get_last_usd_price(); + } else { + console.log('%s[%s] Error: %s', key, pair_key, err); + complete++; + + if (complete == total_pairs || stopSync) + get_last_usd_price(); + } + }); + }); + } else { + console.log('Error: Entry for %s[%s] does not exist in markets database.', key, pair_key); + complete++; + if (complete == total_pairs || stopSync) + get_last_usd_price(); + } + }); + } + }); + } else { + // market not installed + console.log('%s market not installed', key); + complete++; + + if (complete == total_pairs || stopSync) + get_last_usd_price(); + } } }); } else { - // check if market feature is enabled - if (settings.markets_page.enabled == true) { - var complete = 0; - var total_pairs = 0; - var exchanges = Object.keys(settings.markets_page.exchanges); - - // loop through all exchanges to determine how many trading pairs must be updated - exchanges.forEach(function(key, index, map) { - // check if market is enabled via settings - if (settings.markets_page.exchanges[key].enabled == true) { - // check if market is installed/supported - if (db.fs.existsSync('./lib/markets/' + key + '.js')) { - // add trading pairs to total - total_pairs += settings.markets_page.exchanges[key].trading_pairs.length; - - // loop through all trading pairs for this market - for (var i = 0; i < settings.markets_page.exchanges[key].trading_pairs.length; i++) { - // ensure trading pair setting is always uppercase - settings.markets_page.exchanges[key].trading_pairs[i] = settings.markets_page.exchanges[key].trading_pairs[i].toUpperCase(); - } - } - } - }); - - // check if there are any trading pairs to update - if (total_pairs > 0) { - // initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis - var rateLimitLib = require('../lib/ratelimit'); - var rateLimit = new rateLimitLib.RateLimit(1, 2000, false); - // loop through and test all exchanges defined in the settings.json file - exchanges.forEach(function(key, index, map) { - // check if market is enabled via settings - if (settings.markets_page.exchanges[key].enabled == true) { - // check if market is installed/supported - if (db.fs.existsSync('./lib/markets/' + key + '.js')) { - // loop through all trading pairs - settings.markets_page.exchanges[key].trading_pairs.forEach(function(pair_key, pair_index, pair_map) { - // split the pair data - var split_pair = pair_key.split('/'); - // check if this is a valid trading pair - if (split_pair.length == 2) { - // lookup the exchange in the market collection - db.check_market(key, split_pair[0], split_pair[1], function(mkt, exists) { - // check if exchange trading pair exists in the market collection - if (exists) { - // automatically pause for 2 seconds in between requests - rateLimit.schedule(function() { - // update market data - db.update_markets_db(key, split_pair[0], split_pair[1], function(err) { - if (!err) { - console.log('%s[%s]: Market data updated successfully.', key, pair_key); - complete++; - - if (complete == total_pairs || stopSync) - get_last_usd_price(); - } else { - console.log('%s[%s] Error: %s', key, pair_key, err); - complete++; - - if (complete == total_pairs || stopSync) - get_last_usd_price(); - } - }); - }); - } else { - console.log('Error: Entry for %s[%s] does not exist in markets database.', key, pair_key); - complete++; - if (complete == total_pairs || stopSync) - get_last_usd_price(); - } - }); - } - }); - } else { - // market not installed - console.log('%s market not installed', key); - complete++; - - if (complete == total_pairs || stopSync) - get_last_usd_price(); - } - } - }); - } else { - // no market trading pairs are enabled - console.log('Error: No market trading pairs are enabled in settings'); - exit(1); - } - } else { - // market page is not enabled - console.log('Error: Market feature is disabled in settings'); - exit(1); - } + // no market trading pairs are enabled + console.log('Error: No market trading pairs are enabled in settings'); + exit(1); } - }).catch((err) => { - console.log('Error: Unable to connect to database: %s', err); + } else { + // market page is not enabled + console.log('Error: Market feature is disabled in settings'); exit(1); - }); - } else { - // another script process is currently running - console.log("Sync aborted"); - exit(2); + } } - } else { - // sync process is already running - console.log("Sync aborted"); - exit(2); - } + }).catch((err) => { + console.log('Error: Unable to connect to database: %s', err); + exit(1); + }); } else { - console.log('Process aborted. Nothing was deleted'); + // another script process is currently running + console.log("Sync aborted"); exit(2); } -}); \ No newline at end of file +} else { + // sync process is already running + console.log("Sync aborted"); + exit(2); +} \ No newline at end of file