Reindex and delete-database script improvements
-The delete-database script now checks the claimaddresses collection for data and if at least 1 record exists, it will ask an additional question to see if you want to preserve that data or delete everything -The delete-database script now issues the prompt for the reindex with a slightly different msg than the delete without reindex -The prompt for deleting now happens after the locks have been properly checked
This commit is contained in:
Joe Uhren
+124
-54
@@ -2,6 +2,7 @@ const lib = require('../lib/explorer');
|
||||
const readline = require('readline');
|
||||
const deleteLockName = 'delete';
|
||||
var lockCreated = false;
|
||||
var preserveClaimAddressNames = false;
|
||||
|
||||
// exit function used to cleanup lock before finishing script
|
||||
function exit(mongoose, exitCode) {
|
||||
@@ -61,14 +62,26 @@ function drop_collection(mongoose, colName, cb) {
|
||||
}
|
||||
|
||||
function delete_prompt(cb) {
|
||||
// Check if the delete prompt should be skipped
|
||||
if (process.argv[2] == null || process.argv[2] != 'reindex') {
|
||||
preserve_claimaddress_prompt(function() {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
});
|
||||
|
||||
console.log('You are about to delete the entire eIquidus database.');
|
||||
// Change the delete prompt based on whether this is a reindex or regular delete
|
||||
if (process.argv[2] != null && process.argv[2] == 'reindex') {
|
||||
console.log('You are about to delete all data from the entire eIquidus database');
|
||||
|
||||
if (preserveClaimAddressNames)
|
||||
console.log('(claim address name data will not be deleted)');
|
||||
|
||||
console.log('and resync from the genesis block.');
|
||||
} else {
|
||||
console.log('You are about to delete all data from the entire eIquidus database.');
|
||||
|
||||
if (preserveClaimAddressNames)
|
||||
console.log('(claim address name data will not be deleted)');
|
||||
}
|
||||
|
||||
// prompt for deleting explorer database
|
||||
rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer) {
|
||||
@@ -88,43 +101,87 @@ function delete_prompt(cb) {
|
||||
return cb(false);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// skip the delete prompt
|
||||
return cb(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
delete_prompt(function(continue_process) {
|
||||
if (continue_process) {
|
||||
// check if the "delete database" process is already running
|
||||
if (lib.is_locked([deleteLockName]) == false) {
|
||||
// create a new delete lock before checking the rest of the locks to minimize problems with running scripts at the same time
|
||||
lib.create_lock(deleteLockName);
|
||||
// ensure the lock will be deleted on exit
|
||||
lockCreated = true;
|
||||
function preserve_claimaddress_prompt(cb) {
|
||||
const ClaimAddress = require('../models/claimaddress');
|
||||
|
||||
var lock_list = ['backup', 'restore', 'markets', 'peers', 'masternodes'];
|
||||
// check how many claim address records there are
|
||||
ClaimAddress.find({}).countDocuments().then((count) => {
|
||||
// display an additional prompt in the event the claimaddress collection has data
|
||||
if (count > 0) {
|
||||
console.log(`The current database has ${count} custom claim address names`);
|
||||
console.log('Would you like to preserve this data?');
|
||||
|
||||
// do not check the index lock if this is called from the reindex process
|
||||
if (process.argv[2] == null || process.argv[2] != 'reindex') {
|
||||
lock_list.push('index');
|
||||
}
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
});
|
||||
|
||||
// check all other possible locks since database deletion should not run at the same time that data is being changed
|
||||
if (lib.is_locked(lock_list) == false) {
|
||||
// all tests passed. OK to run delete
|
||||
console.log("Script launched with pid: " + process.pid);
|
||||
// prompt for deleting claim address data
|
||||
rl.question('y = keep claim address data, n = delete claim address data [y/n]: ', function (preserveClaimAddresses) {
|
||||
// stop prompting
|
||||
rl.close();
|
||||
|
||||
const settings = require('../lib/settings');
|
||||
const mongoose = require('mongoose');
|
||||
const dbString = `mongodb://${encodeURIComponent(settings.dbsettings.user)}:${encodeURIComponent(settings.dbsettings.password)}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
|
||||
// determine if the claim address data should be preserved
|
||||
switch (preserveClaimAddresses) {
|
||||
case 'y':
|
||||
case 'Y':
|
||||
case 'yes':
|
||||
case 'YES':
|
||||
case 'Yes':
|
||||
preserveClaimAddressNames = true;
|
||||
console.log('Claim address name data will be saved' + '\n');
|
||||
break;
|
||||
default:
|
||||
console.log('Claim address name data will be deleted' + '\n');
|
||||
}
|
||||
|
||||
console.log('Connecting to database..');
|
||||
return cb();
|
||||
});
|
||||
} else
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
|
||||
mongoose.set('strictQuery', true);
|
||||
// check if the "delete database" process is already running
|
||||
if (lib.is_locked([deleteLockName]) == false) {
|
||||
// create a new delete lock before checking the rest of the locks to minimize problems with running scripts at the same time
|
||||
lib.create_lock(deleteLockName);
|
||||
// ensure the lock will be deleted on exit
|
||||
lockCreated = true;
|
||||
|
||||
// connect to mongo database
|
||||
mongoose.connect(dbString).then(() => {
|
||||
var lock_list = ['backup', 'restore', 'markets', 'peers', 'masternodes'];
|
||||
|
||||
// do not check the index lock if this is called from the reindex process
|
||||
if (process.argv[2] == null || process.argv[2] != 'reindex') {
|
||||
lock_list.push('index');
|
||||
}
|
||||
|
||||
// check all other possible locks since database deletion should not run at the same time that data is being changed
|
||||
if (lib.is_locked(lock_list) == false) {
|
||||
// all tests passed. OK to run delete
|
||||
|
||||
// suppress the pid message when doing a reindex
|
||||
if (process.argv[2] == null || process.argv[2] != 'reindex')
|
||||
console.log("Script launched with pid: " + process.pid);
|
||||
|
||||
const settings = require('../lib/settings');
|
||||
const mongoose = require('mongoose');
|
||||
const dbString = `mongodb://${encodeURIComponent(settings.dbsettings.user)}:${encodeURIComponent(settings.dbsettings.password)}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
|
||||
|
||||
console.log('Connecting to database..');
|
||||
|
||||
mongoose.set('strictQuery', true);
|
||||
|
||||
// connect to mongo database
|
||||
mongoose.connect(dbString).then(() => {
|
||||
console.log('Database connection successful' + '\n');
|
||||
|
||||
// prompt for database delete
|
||||
delete_prompt(function(continue_process) {
|
||||
if (continue_process) {
|
||||
// get the list of collections
|
||||
mongoose.connection.db.listCollections().toArray().then((collections) => {
|
||||
// check if there are any collections
|
||||
@@ -133,13 +190,26 @@ delete_prompt(function(continue_process) {
|
||||
|
||||
// loop through all collections
|
||||
collections.forEach((collection) => {
|
||||
console.log(`Deleting ${collection.name}..`);
|
||||
// check if this is the claim addres collection and that data is being preserved
|
||||
if (!preserveClaimAddressNames || collection.name != 'claimaddresses') {
|
||||
console.log(`Deleting ${collection.name}..`);
|
||||
|
||||
// delete this collection
|
||||
drop_collection(mongoose, collection.name, function(retVal) {
|
||||
// check if the collection was successfully deleted
|
||||
if (retVal)
|
||||
counter++;
|
||||
// delete this collection
|
||||
drop_collection(mongoose, collection.name, function(retVal) {
|
||||
// check if the collection was successfully deleted
|
||||
if (retVal)
|
||||
counter++;
|
||||
|
||||
// check if the last collection was deleted
|
||||
if (counter == collections.length) {
|
||||
// finish the delete process
|
||||
console.log('Finished deleting database');
|
||||
exit(mongoose, 0);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// skipped deleting of the claimaddresses collection
|
||||
counter++;
|
||||
|
||||
// check if the last collection was deleted
|
||||
if (counter == collections.length) {
|
||||
@@ -147,7 +217,7 @@ delete_prompt(function(continue_process) {
|
||||
console.log('Finished deleting database');
|
||||
exit(mongoose, 0);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// nothing to delete
|
||||
@@ -160,22 +230,22 @@ delete_prompt(function(continue_process) {
|
||||
console.log('Error: Unable to list collections in database: %s', err);
|
||||
exit(mongoose, 1);
|
||||
});
|
||||
}).catch((err) => {
|
||||
console.log('Error: Unable to connect to database: %s', err);
|
||||
exit(mongoose, 999);
|
||||
});
|
||||
} else {
|
||||
// another script process is currently running
|
||||
console.log("Delete aborted");
|
||||
exit(null, 2);
|
||||
}
|
||||
} else {
|
||||
// delete process is already running
|
||||
console.log("Delete aborted");
|
||||
exit(null, 2);
|
||||
}
|
||||
} else {
|
||||
console.log('Process aborted. Nothing was deleted.');
|
||||
exit(null, 2);
|
||||
}
|
||||
});
|
||||
}).catch((err) => {
|
||||
console.log('Error: Unable to connect to database: %s', err);
|
||||
exit(mongoose, 999);
|
||||
});
|
||||
} else {
|
||||
console.log('Process aborted. Nothing was deleted.');
|
||||
// another script process is currently running
|
||||
console.log("Delete aborted");
|
||||
exit(null, 2);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// delete process is already running
|
||||
console.log("Delete aborted");
|
||||
exit(null, 2);
|
||||
}
|
||||
+404
-445
@@ -1116,41 +1116,6 @@ function block_sync(reindex, stats) {
|
||||
});
|
||||
}
|
||||
|
||||
function check_reindex_prompt(cb) {
|
||||
// check if this is a reindex
|
||||
if (mode == 'reindex') {
|
||||
const readline = require('readline');
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
});
|
||||
|
||||
console.log('You are about to delete all blockchain data (transactions and addresses)');
|
||||
console.log('and resync from the genesis block.');
|
||||
|
||||
// prompt for reindexing the database
|
||||
rl.question('Are you sure you want to do this? [y/n]: ', function (reindexAnswer) {
|
||||
// stop prompting
|
||||
rl.close();
|
||||
|
||||
// determine if the reindex should proceed
|
||||
switch (reindexAnswer) {
|
||||
case 'y':
|
||||
case 'Y':
|
||||
case 'yes':
|
||||
case 'YES':
|
||||
case 'Yes':
|
||||
return cb(true);
|
||||
break;
|
||||
default:
|
||||
return cb(false);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
return cb(true);
|
||||
}
|
||||
}
|
||||
|
||||
// check options
|
||||
if (process.argv[2] == null || process.argv[2] == 'index' || process.argv[2] == 'update') {
|
||||
mode = null;
|
||||
@@ -1196,242 +1161,275 @@ else if (process.argv[2] == 'market')
|
||||
else
|
||||
usage();
|
||||
|
||||
// check if this is a reindex which requires a prompt before continuing
|
||||
check_reindex_prompt(function(reindexAnswer) {
|
||||
// check if the process should continue
|
||||
if (reindexAnswer) {
|
||||
// check if this sync option is already running/locked
|
||||
if (lib.is_locked([database]) == false) {
|
||||
// create a new sync lock before checking the rest of the locks to minimize problems with running scripts at the same time
|
||||
lib.create_lock(database);
|
||||
// ensure the lock will be deleted on exit
|
||||
lockCreated = true;
|
||||
// check the backup, restore and delete locks since those functions would be problematic when updating data
|
||||
if (lib.is_locked(['backup', 'restore', 'delete']) == false) {
|
||||
// all tests passed. OK to run sync
|
||||
console.log("Script launched with pid: " + process.pid);
|
||||
// check if this sync option is already running/locked
|
||||
if (lib.is_locked([database]) == false) {
|
||||
// create a new sync lock before checking the rest of the locks to minimize problems with running scripts at the same time
|
||||
lib.create_lock(database);
|
||||
// ensure the lock will be deleted on exit
|
||||
lockCreated = true;
|
||||
// check the backup, restore and delete locks since those functions would be problematic when updating data
|
||||
if (lib.is_locked(['backup', 'restore', 'delete']) == false) {
|
||||
// all tests passed. OK to run sync
|
||||
console.log("Script launched with pid: " + process.pid);
|
||||
|
||||
if (mode == 'update')
|
||||
console.log(`Syncing ${(database == 'index' ? 'blocks' : database)}.. Please wait..`);
|
||||
if (mode == 'update')
|
||||
console.log(`Syncing ${(database == 'index' ? 'blocks' : database)}.. Please wait..`);
|
||||
|
||||
var dbString = 'mongodb://' + encodeURIComponent(settings.dbsettings.user);
|
||||
dbString = dbString + ':' + encodeURIComponent(settings.dbsettings.password);
|
||||
dbString = dbString + '@' + settings.dbsettings.address;
|
||||
dbString = dbString + ':' + settings.dbsettings.port;
|
||||
dbString = dbString + '/' + settings.dbsettings.database;
|
||||
var dbString = 'mongodb://' + encodeURIComponent(settings.dbsettings.user);
|
||||
dbString = dbString + ':' + encodeURIComponent(settings.dbsettings.password);
|
||||
dbString = dbString + '@' + settings.dbsettings.address;
|
||||
dbString = dbString + ':' + settings.dbsettings.port;
|
||||
dbString = dbString + '/' + settings.dbsettings.database;
|
||||
|
||||
mongoose.set('strictQuery', true);
|
||||
mongoose.set('strictQuery', true);
|
||||
|
||||
mongoose.connect(dbString).then(() => {
|
||||
if (database == 'index') {
|
||||
db.check_stats(settings.coin.name, function(exists) {
|
||||
if (exists == false) {
|
||||
console.log('Run \'npm start\' to create database structures before running this script.');
|
||||
mongoose.connect(dbString).then(() => {
|
||||
if (database == 'index') {
|
||||
db.check_stats(settings.coin.name, function(exists) {
|
||||
if (exists == false) {
|
||||
console.log('Run \'npm start\' to create database structures before running this script.');
|
||||
exit(1);
|
||||
} else {
|
||||
// determine which index mode to run
|
||||
if (mode == 'reindex') {
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
try {
|
||||
// delete the database
|
||||
execSync(`node ./scripts/delete_database.js ${mode}`, {stdio : 'inherit'});
|
||||
} catch (err) {
|
||||
// delete_database.js was not successful, so exit
|
||||
exit(1);
|
||||
} else {
|
||||
// determine which index mode to run
|
||||
if (mode == 'reindex') {
|
||||
const { execSync } = require('child_process');
|
||||
}
|
||||
|
||||
console.log('Deleting database.. Please wait..');
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
// start the block sync
|
||||
block_sync(true, stats);
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'check') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Checking blocks.. Please wait..');
|
||||
|
||||
// delete the database
|
||||
execSync(`node ./scripts/delete_database.js ${mode}`, {stdio : 'inherit'});
|
||||
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
// start the block sync
|
||||
block_sync(true, stats);
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, 1, function() {
|
||||
// check if the script stopped prematurely
|
||||
if (stopSync) {
|
||||
console.log('Block check was stopped prematurely');
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'check') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Checking blocks.. Please wait..');
|
||||
|
||||
update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, 1, function() {
|
||||
// check if the script stopped prematurely
|
||||
if (stopSync) {
|
||||
console.log('Block check was stopped prematurely');
|
||||
exit(1);
|
||||
} else {
|
||||
db.get_stats(settings.coin.name, function(nstats) {
|
||||
console.log('Block check complete (block: %s)', nstats.last);
|
||||
exit(0);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
db.get_stats(settings.coin.name, function(nstats) {
|
||||
console.log('Block check complete (block: %s)', nstats.last);
|
||||
exit(0);
|
||||
});
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'update') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
// start the block sync
|
||||
block_sync(false, stats);
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'reindex-rich') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Check richlist');
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'update') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
// start the block sync
|
||||
block_sync(false, stats);
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'reindex-rich') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Check richlist');
|
||||
|
||||
db.check_richlist(settings.coin.name, function(exists) {
|
||||
if (exists)
|
||||
console.log('Richlist entry found, deleting now..');
|
||||
db.check_richlist(settings.coin.name, function(exists) {
|
||||
if (exists)
|
||||
console.log('Richlist entry found, deleting now..');
|
||||
|
||||
db.delete_richlist(settings.coin.name, function(deleted) {
|
||||
if (deleted)
|
||||
console.log('Richlist entry deleted');
|
||||
db.delete_richlist(settings.coin.name, function(deleted) {
|
||||
if (deleted)
|
||||
console.log('Richlist entry deleted');
|
||||
|
||||
db.create_richlist(settings.coin.name, false, function() {
|
||||
console.log('Richlist created');
|
||||
db.create_richlist(settings.coin.name, false, function() {
|
||||
console.log('Richlist created');
|
||||
|
||||
db.update_richlist('received', function() {
|
||||
console.log('Richlist updated received');
|
||||
db.update_richlist('received', function() {
|
||||
console.log('Richlist updated received');
|
||||
|
||||
db.update_richlist('balance', function() {
|
||||
// update richlist_last_updated value
|
||||
db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
|
||||
console.log('Richlist update complete');
|
||||
exit(0);
|
||||
});
|
||||
});
|
||||
db.update_richlist('balance', function() {
|
||||
// update richlist_last_updated value
|
||||
db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
|
||||
console.log('Richlist update complete');
|
||||
exit(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'reindex-txcount') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Calculating tx count.. Please wait..');
|
||||
|
||||
// Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout
|
||||
Tx.find({'total': {$gte: 0}, 'vout': { $gte: { $size: 1 }}}).countDocuments().then((count) => {
|
||||
console.log('Found tx count: ' + count.toString());
|
||||
Stats.updateOne({coin: settings.coin.name}, {
|
||||
txes: count
|
||||
}).then(() => {
|
||||
console.log('Tx count update complete');
|
||||
exit(0);
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'reindex-last') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Finding last blockindex.. Please wait..');
|
||||
|
||||
// Resetting the last blockindex counter requires a single lookup on the txes collection to find the last indexed blockindex
|
||||
Tx.find({}, {blockindex:1, _id:0}).sort({blockindex: -1}).limit(1).exec().then((tx) => {
|
||||
// check if any blocks exists
|
||||
if (tx == null || tx.length == 0) {
|
||||
console.log('No blocks found. setting last blockindex to 0.');
|
||||
|
||||
Stats.updateOne({coin: settings.coin.name}, {
|
||||
last: 0
|
||||
}).then(() => {
|
||||
console.log('Last blockindex update complete');
|
||||
exit(0);
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
console.log('Found last blockindex: ' + tx[0].blockindex.toString());
|
||||
|
||||
Stats.updateOne({coin: settings.coin.name}, {
|
||||
last: tx[0].blockindex
|
||||
}).then(() => {
|
||||
console.log('Last blockindex update complete');
|
||||
exit(0);
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
}
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'reindex-txcount') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Calculating tx count.. Please wait..');
|
||||
|
||||
// Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout
|
||||
Tx.find({'total': {$gte: 0}, 'vout': { $gte: { $size: 1 }}}).countDocuments().then((count) => {
|
||||
console.log('Found tx count: ' + count.toString());
|
||||
Stats.updateOne({coin: settings.coin.name}, {
|
||||
txes: count
|
||||
}).then(() => {
|
||||
console.log('Tx count update complete');
|
||||
exit(0);
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
} else if (mode == 'reindex-last') {
|
||||
db.update_db(settings.coin.name, function(stats) {
|
||||
// check if stats returned properly
|
||||
if (stats !== false) {
|
||||
console.log('Finding last blockindex.. Please wait..');
|
||||
|
||||
// Resetting the last blockindex counter requires a single lookup on the txes collection to find the last indexed blockindex
|
||||
Tx.find({}, {blockindex:1, _id:0}).sort({blockindex: -1}).limit(1).exec().then((tx) => {
|
||||
// check if any blocks exists
|
||||
if (tx == null || tx.length == 0) {
|
||||
console.log('No blocks found. setting last blockindex to 0.');
|
||||
|
||||
Stats.updateOne({coin: settings.coin.name}, {
|
||||
last: 0
|
||||
}).then(() => {
|
||||
console.log('Last blockindex update complete');
|
||||
exit(0);
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
console.log('Found last blockindex: ' + tx[0].blockindex.toString());
|
||||
|
||||
Stats.updateOne({coin: settings.coin.name}, {
|
||||
last: tx[0].blockindex
|
||||
}).then(() => {
|
||||
console.log('Last blockindex update complete');
|
||||
exit(0);
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
}
|
||||
}).catch((err) => {
|
||||
console.log(err);
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
// update_db threw an error so exit
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
} else if (database == 'peers') {
|
||||
lib.get_peerinfo(function(body) {
|
||||
if (body != null) {
|
||||
lib.syncLoop(body.length, function(loop) {
|
||||
var i = loop.iteration();
|
||||
var address = body[i].addr;
|
||||
var port = null;
|
||||
|
||||
if (occurrences(address, ':') == 1 || occurrences(address, ']:') == 1) {
|
||||
// Separate the port # from the IP address
|
||||
address = address.substring(0, address.lastIndexOf(":")).replace("[", "").replace("]", "");
|
||||
port = body[i].addr.substring(body[i].addr.lastIndexOf(":") + 1);
|
||||
}
|
||||
});
|
||||
} else if (database == 'peers') {
|
||||
lib.get_peerinfo(function(body) {
|
||||
if (body != null) {
|
||||
lib.syncLoop(body.length, function(loop) {
|
||||
var i = loop.iteration();
|
||||
var address = body[i].addr;
|
||||
var port = null;
|
||||
|
||||
if (occurrences(address, ':') == 1 || occurrences(address, ']:') == 1) {
|
||||
// Separate the port # from the IP address
|
||||
address = address.substring(0, address.lastIndexOf(":")).replace("[", "").replace("]", "");
|
||||
port = body[i].addr.substring(body[i].addr.lastIndexOf(":") + 1);
|
||||
if (address.indexOf("]") > -1) {
|
||||
// Remove [] characters from IPv6 addresses
|
||||
address = address.replace("[", "").replace("]", "");
|
||||
}
|
||||
|
||||
db.find_peer(address, port, function(peer) {
|
||||
if (peer) {
|
||||
if (peer['port'] != null && (isNaN(peer['port']) || peer['port'].length < 2)) {
|
||||
db.drop_peers(function() {
|
||||
console.log('Removing peers due to missing port information. Re-run this script to add peers again.');
|
||||
exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
if (address.indexOf("]") > -1) {
|
||||
// Remove [] characters from IPv6 addresses
|
||||
address = address.replace("[", "").replace("]", "");
|
||||
}
|
||||
// peer already exists and should be refreshed
|
||||
// drop peer
|
||||
db.drop_peer(address, port, function() {
|
||||
// re-add the peer to refresh the data and extend the expiry date
|
||||
db.create_peer({
|
||||
address: address,
|
||||
port: port,
|
||||
protocol: peer.protocol,
|
||||
version: peer.version,
|
||||
country: peer.country,
|
||||
country_code: peer.country_code
|
||||
}, function() {
|
||||
console.log('Updated peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString());
|
||||
|
||||
db.find_peer(address, port, function(peer) {
|
||||
if (peer) {
|
||||
if (peer['port'] != null && (isNaN(peer['port']) || peer['port'].length < 2)) {
|
||||
db.drop_peers(function() {
|
||||
console.log('Removing peers due to missing port information. Re-run this script to add peers again.');
|
||||
exit(1);
|
||||
});
|
||||
// check if the script is stopping
|
||||
if (stopSync) {
|
||||
// stop the loop
|
||||
loop.break(true);
|
||||
}
|
||||
|
||||
// peer already exists and should be refreshed
|
||||
// drop peer
|
||||
db.drop_peer(address, port, function() {
|
||||
// re-add the peer to refresh the data and extend the expiry date
|
||||
loop.next();
|
||||
});
|
||||
});
|
||||
} else {
|
||||
const rateLimitLib = require('../lib/ratelimit');
|
||||
const rateLimit = new rateLimitLib.RateLimit(1, 2000, false);
|
||||
|
||||
rateLimit.schedule(function() {
|
||||
lib.get_geo_location(address, function(error, geo) {
|
||||
// check if an error was returned
|
||||
if (error) {
|
||||
console.log(error);
|
||||
exit(1);
|
||||
} else if (geo == null || typeof geo != 'object') {
|
||||
console.log('Error: geolocation api did not return a valid object');
|
||||
exit(1);
|
||||
} else {
|
||||
// add peer to collection
|
||||
db.create_peer({
|
||||
address: address,
|
||||
port: port,
|
||||
protocol: peer.protocol,
|
||||
version: peer.version,
|
||||
country: peer.country,
|
||||
country_code: peer.country_code
|
||||
protocol: body[i].version,
|
||||
version: body[i].subver.replace('/', '').replace('/', ''),
|
||||
country: geo.country_name,
|
||||
country_code: geo.country_code
|
||||
}, function() {
|
||||
console.log('Updated peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString());
|
||||
console.log('Added new peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString());
|
||||
|
||||
// check if the script is stopping
|
||||
if (stopSync) {
|
||||
@@ -1441,223 +1439,184 @@ check_reindex_prompt(function(reindexAnswer) {
|
||||
|
||||
loop.next();
|
||||
});
|
||||
});
|
||||
} else {
|
||||
const rateLimitLib = require('../lib/ratelimit');
|
||||
const rateLimit = new rateLimitLib.RateLimit(1, 2000, false);
|
||||
|
||||
rateLimit.schedule(function() {
|
||||
lib.get_geo_location(address, function(error, geo) {
|
||||
// check if an error was returned
|
||||
if (error) {
|
||||
console.log(error);
|
||||
exit(1);
|
||||
} else if (geo == null || typeof geo != 'object') {
|
||||
console.log('Error: geolocation api did not return a valid object');
|
||||
exit(1);
|
||||
} else {
|
||||
// add peer to collection
|
||||
db.create_peer({
|
||||
address: address,
|
||||
port: port,
|
||||
protocol: body[i].version,
|
||||
version: body[i].subver.replace('/', '').replace('/', ''),
|
||||
country: geo.country_name,
|
||||
country_code: geo.country_code
|
||||
}, function() {
|
||||
console.log('Added new peer %s%s [%s/%s]', address, (port == null || port == '' ? '' : ':' + port.toString()), (i + 1).toString(), body.length.toString());
|
||||
|
||||
// check if the script is stopping
|
||||
if (stopSync) {
|
||||
// stop the loop
|
||||
loop.break(true);
|
||||
}
|
||||
|
||||
loop.next();
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
}, function() {
|
||||
// update network_last_updated value
|
||||
db.update_last_updated_stats(settings.coin.name, { network_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
|
||||
// check if the script stopped prematurely
|
||||
if (stopSync) {
|
||||
console.log('Peer sync was stopped prematurely');
|
||||
exit(1);
|
||||
} else {
|
||||
console.log('Peer sync complete');
|
||||
exit(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
console.log('No peers found');
|
||||
exit(2);
|
||||
}
|
||||
});
|
||||
} else if (database == 'masternodes') {
|
||||
lib.get_masternodelist(function(body) {
|
||||
if (body != null) {
|
||||
var isObject = false;
|
||||
var objectKeys = null;
|
||||
|
||||
// Check if the masternode data is an array or an object
|
||||
if (body.length == null) {
|
||||
// Process data as an object
|
||||
objectKeys = Object.keys(body);
|
||||
isObject = true;
|
||||
}
|
||||
|
||||
lib.syncLoop((isObject ? objectKeys : body).length, function(loop) {
|
||||
var i = loop.iteration();
|
||||
|
||||
db.save_masternode((isObject ? body[objectKeys[i]] : body[i]), function(success) {
|
||||
if (success) {
|
||||
// check if the script is stopping
|
||||
if (stopSync) {
|
||||
// stop the loop
|
||||
loop.break(true);
|
||||
}
|
||||
|
||||
loop.next();
|
||||
} else {
|
||||
console.log('Error: Cannot save masternode %s.', (isObject ? (body[objectKeys[i]].payee ? body[objectKeys[i]].payee : 'UNKNOWN') : (body[i].addr ? body[i].addr : 'UNKNOWN')));
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
}, function() {
|
||||
db.remove_old_masternodes(function(cb) {
|
||||
db.update_last_updated_stats(settings.coin.name, { masternodes_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
|
||||
// check if the script stopped prematurely
|
||||
if (stopSync) {
|
||||
console.log('Masternode sync was stopped prematurely');
|
||||
exit(1);
|
||||
} else {
|
||||
console.log('Masternode sync complete');
|
||||
exit(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
}, function() {
|
||||
// update network_last_updated value
|
||||
db.update_last_updated_stats(settings.coin.name, { network_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
|
||||
// check if the script stopped prematurely
|
||||
if (stopSync) {
|
||||
console.log('Peer sync was stopped prematurely');
|
||||
exit(1);
|
||||
} else {
|
||||
console.log('Peer sync complete');
|
||||
exit(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
console.log('No peers found');
|
||||
exit(2);
|
||||
}
|
||||
});
|
||||
} else if (database == 'masternodes') {
|
||||
lib.get_masternodelist(function(body) {
|
||||
if (body != null) {
|
||||
var isObject = false;
|
||||
var objectKeys = null;
|
||||
|
||||
// Check if the masternode data is an array or an object
|
||||
if (body.length == null) {
|
||||
// Process data as an object
|
||||
objectKeys = Object.keys(body);
|
||||
isObject = true;
|
||||
}
|
||||
|
||||
lib.syncLoop((isObject ? objectKeys : body).length, function(loop) {
|
||||
var i = loop.iteration();
|
||||
|
||||
db.save_masternode((isObject ? body[objectKeys[i]] : body[i]), function(success) {
|
||||
if (success) {
|
||||
// check if the script is stopping
|
||||
if (stopSync) {
|
||||
// stop the loop
|
||||
loop.break(true);
|
||||
}
|
||||
|
||||
loop.next();
|
||||
} else {
|
||||
console.log('Error: Cannot save masternode %s.', (isObject ? (body[objectKeys[i]].payee ? body[objectKeys[i]].payee : 'UNKNOWN') : (body[i].addr ? body[i].addr : 'UNKNOWN')));
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
}, function() {
|
||||
db.remove_old_masternodes(function(cb) {
|
||||
db.update_last_updated_stats(settings.coin.name, { masternodes_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
|
||||
// check if the script stopped prematurely
|
||||
if (stopSync) {
|
||||
console.log('Masternode sync was stopped prematurely');
|
||||
exit(1);
|
||||
} else {
|
||||
console.log('Masternode sync complete');
|
||||
exit(0);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
console.log('No masternodes found');
|
||||
exit(2);
|
||||
});
|
||||
});
|
||||
} else {
|
||||
console.log('No masternodes found');
|
||||
exit(2);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// check if market feature is enabled
|
||||
if (settings.markets_page.enabled == true) {
|
||||
var complete = 0;
|
||||
var total_pairs = 0;
|
||||
var exchanges = Object.keys(settings.markets_page.exchanges);
|
||||
|
||||
// loop through all exchanges to determine how many trading pairs must be updated
|
||||
exchanges.forEach(function(key, index, map) {
|
||||
// check if market is enabled via settings
|
||||
if (settings.markets_page.exchanges[key].enabled == true) {
|
||||
// check if market is installed/supported
|
||||
if (db.fs.existsSync('./lib/markets/' + key + '.js')) {
|
||||
// add trading pairs to total
|
||||
total_pairs += settings.markets_page.exchanges[key].trading_pairs.length;
|
||||
|
||||
// loop through all trading pairs for this market
|
||||
for (var i = 0; i < settings.markets_page.exchanges[key].trading_pairs.length; i++) {
|
||||
// ensure trading pair setting is always uppercase
|
||||
settings.markets_page.exchanges[key].trading_pairs[i] = settings.markets_page.exchanges[key].trading_pairs[i].toUpperCase();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// check if there are any trading pairs to update
|
||||
if (total_pairs > 0) {
|
||||
// initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis
|
||||
var rateLimitLib = require('../lib/ratelimit');
|
||||
var rateLimit = new rateLimitLib.RateLimit(1, 2000, false);
|
||||
// loop through and test all exchanges defined in the settings.json file
|
||||
exchanges.forEach(function(key, index, map) {
|
||||
// check if market is enabled via settings
|
||||
if (settings.markets_page.exchanges[key].enabled == true) {
|
||||
// check if market is installed/supported
|
||||
if (db.fs.existsSync('./lib/markets/' + key + '.js')) {
|
||||
// loop through all trading pairs
|
||||
settings.markets_page.exchanges[key].trading_pairs.forEach(function(pair_key, pair_index, pair_map) {
|
||||
// split the pair data
|
||||
var split_pair = pair_key.split('/');
|
||||
// check if this is a valid trading pair
|
||||
if (split_pair.length == 2) {
|
||||
// lookup the exchange in the market collection
|
||||
db.check_market(key, split_pair[0], split_pair[1], function(mkt, exists) {
|
||||
// check if exchange trading pair exists in the market collection
|
||||
if (exists) {
|
||||
// automatically pause for 2 seconds in between requests
|
||||
rateLimit.schedule(function() {
|
||||
// update market data
|
||||
db.update_markets_db(key, split_pair[0], split_pair[1], function(err) {
|
||||
if (!err) {
|
||||
console.log('%s[%s]: Market data updated successfully.', key, pair_key);
|
||||
complete++;
|
||||
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
} else {
|
||||
console.log('%s[%s] Error: %s', key, pair_key, err);
|
||||
complete++;
|
||||
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
console.log('Error: Entry for %s[%s] does not exist in markets database.', key, pair_key);
|
||||
complete++;
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// market not installed
|
||||
console.log('%s market not installed', key);
|
||||
complete++;
|
||||
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
}
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// check if market feature is enabled
|
||||
if (settings.markets_page.enabled == true) {
|
||||
var complete = 0;
|
||||
var total_pairs = 0;
|
||||
var exchanges = Object.keys(settings.markets_page.exchanges);
|
||||
|
||||
// loop through all exchanges to determine how many trading pairs must be updated
|
||||
exchanges.forEach(function(key, index, map) {
|
||||
// check if market is enabled via settings
|
||||
if (settings.markets_page.exchanges[key].enabled == true) {
|
||||
// check if market is installed/supported
|
||||
if (db.fs.existsSync('./lib/markets/' + key + '.js')) {
|
||||
// add trading pairs to total
|
||||
total_pairs += settings.markets_page.exchanges[key].trading_pairs.length;
|
||||
|
||||
// loop through all trading pairs for this market
|
||||
for (var i = 0; i < settings.markets_page.exchanges[key].trading_pairs.length; i++) {
|
||||
// ensure trading pair setting is always uppercase
|
||||
settings.markets_page.exchanges[key].trading_pairs[i] = settings.markets_page.exchanges[key].trading_pairs[i].toUpperCase();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// check if there are any trading pairs to update
|
||||
if (total_pairs > 0) {
|
||||
// initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis
|
||||
var rateLimitLib = require('../lib/ratelimit');
|
||||
var rateLimit = new rateLimitLib.RateLimit(1, 2000, false);
|
||||
// loop through and test all exchanges defined in the settings.json file
|
||||
exchanges.forEach(function(key, index, map) {
|
||||
// check if market is enabled via settings
|
||||
if (settings.markets_page.exchanges[key].enabled == true) {
|
||||
// check if market is installed/supported
|
||||
if (db.fs.existsSync('./lib/markets/' + key + '.js')) {
|
||||
// loop through all trading pairs
|
||||
settings.markets_page.exchanges[key].trading_pairs.forEach(function(pair_key, pair_index, pair_map) {
|
||||
// split the pair data
|
||||
var split_pair = pair_key.split('/');
|
||||
// check if this is a valid trading pair
|
||||
if (split_pair.length == 2) {
|
||||
// lookup the exchange in the market collection
|
||||
db.check_market(key, split_pair[0], split_pair[1], function(mkt, exists) {
|
||||
// check if exchange trading pair exists in the market collection
|
||||
if (exists) {
|
||||
// automatically pause for 2 seconds in between requests
|
||||
rateLimit.schedule(function() {
|
||||
// update market data
|
||||
db.update_markets_db(key, split_pair[0], split_pair[1], function(err) {
|
||||
if (!err) {
|
||||
console.log('%s[%s]: Market data updated successfully.', key, pair_key);
|
||||
complete++;
|
||||
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
} else {
|
||||
console.log('%s[%s] Error: %s', key, pair_key, err);
|
||||
complete++;
|
||||
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
console.log('Error: Entry for %s[%s] does not exist in markets database.', key, pair_key);
|
||||
complete++;
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// market not installed
|
||||
console.log('%s market not installed', key);
|
||||
complete++;
|
||||
|
||||
if (complete == total_pairs || stopSync)
|
||||
get_last_usd_price();
|
||||
}
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// no market trading pairs are enabled
|
||||
console.log('Error: No market trading pairs are enabled in settings');
|
||||
exit(1);
|
||||
}
|
||||
} else {
|
||||
// market page is not enabled
|
||||
console.log('Error: Market feature is disabled in settings');
|
||||
exit(1);
|
||||
}
|
||||
// no market trading pairs are enabled
|
||||
console.log('Error: No market trading pairs are enabled in settings');
|
||||
exit(1);
|
||||
}
|
||||
}).catch((err) => {
|
||||
console.log('Error: Unable to connect to database: %s', err);
|
||||
} else {
|
||||
// market page is not enabled
|
||||
console.log('Error: Market feature is disabled in settings');
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
// another script process is currently running
|
||||
console.log("Sync aborted");
|
||||
exit(2);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// sync process is already running
|
||||
console.log("Sync aborted");
|
||||
exit(2);
|
||||
}
|
||||
}).catch((err) => {
|
||||
console.log('Error: Unable to connect to database: %s', err);
|
||||
exit(1);
|
||||
});
|
||||
} else {
|
||||
console.log('Process aborted. Nothing was deleted');
|
||||
// another script process is currently running
|
||||
console.log("Sync aborted");
|
||||
exit(2);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// sync process is already running
|
||||
console.log("Sync aborted");
|
||||
exit(2);
|
||||
}
|
||||
Reference in New Issue
Block a user