davide/purple-explorer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reindex improvements

Browse Source 
-Reindex now calls the delete-databse.js script which is much faster and more complete than the previous manual delete code the resync used to use
-The delete-database.js script now accepts a parameter that suppresses the delete prompt when being called from the resync process
-Reindex and normal block sync now reuse the same block sync code
-Reworked the sync script to no longer require the readline-sync dependency
-The readline-sync dependency has been completely removed from package.json since it is no longer used
This commit is contained in:
Joe Uhren
2023-09-23 17:39:50 -06:00
parent f30e34adbb
commit 119cf3e2c6
3 changed files with 578 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
-1
View File
@@ -44,7 +44,6 @@
"postman-request": "^2.88.1-postman.31",
"pug": "~3.0.2",
"qr-image": "^3.2.0",
"readline-sync": "^1.4.10",
"sass": "^1.52.3",
"serve-favicon": "^2.5.0"
},
scripts/delete_database.js
+35 -11
View File
@@ -1,9 +1,5 @@
const lib = require('../lib/explorer');
const readline = require('readline');
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
const deleteLockName = 'delete';
var lockCreated = false;
@@ -64,10 +60,18 @@ function drop_collection(mongoose, colName, cb) {
});
}
console.log('You are about to delete the entire eIquidus database.');
function delete_prompt(cb) {
// Check if the delete prompt should be skipped
if (process.argv[2] == null || process.argv[2] != 'reindex') {
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
// prompt for deleting explorer database
rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer) {
console.log('You are about to delete the entire eIquidus database.');
// prompt for deleting explorer database
rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer) {
// stop prompting
rl.close();
@@ -78,14 +82,36 @@ rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer)
case 'yes':
case 'YES':
case 'Yes':
return cb(true);
break;
default:
return cb(false);
}
});
} else {
// skip the delete prompt
return cb(true);
}
}
delete_prompt(function(continue_process) {
if (continue_process) {
// check if the "delete database" process is already running
if (lib.is_locked([deleteLockName]) == false) {
// create a new delete lock before checking the rest of the locks to minimize problems with running scripts at the same time
lib.create_lock(deleteLockName);
// ensure the lock will be deleted on exit
lockCreated = true;
var lock_list = ['backup', 'restore', 'markets', 'peers', 'masternodes'];
// do not check the index lock if this is called from the reindex process
if (process.argv[2] == null || process.argv[2] != 'reindex') {
lock_list.push('index');
}
// check all other possible locks since database deletion should not run at the same time that data is being changed
if (lib.is_locked(['backup', 'restore', 'index', 'markets', 'peers', 'masternodes']) == false) {
if (lib.is_locked(lock_list) == false) {
// all tests passed. OK to run delete
console.log("Script launched with pid: " + process.pid);
@@ -148,9 +174,7 @@ rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer)
console.log("Delete aborted");
exit(null, 2);
}
break;
default:
} else {
console.log('Process aborted. Nothing was deleted.');
exit(null, 2);
}
scripts/sync.js
+144 -163
View File
@@ -1065,6 +1065,92 @@ function occurrences(string, subString, allowOverlapping) {
return n;
}
function block_sync(reindex, stats) {
// Get the last synced block index value
var last = (stats.last ? stats.last : 0);
// Get the total number of blocks
var count = (stats.count ? stats.count : 0);
// Check if the sync msg should be shown
check_show_sync_message(count - last);
update_tx_db(settings.coin.name, last, count, stats.txes, settings.sync.update_timeout, 0, function() {
// check if the script stopped prematurely
if (stopSync) {
console.log(`${(reindex ? 'Reindex' : 'Block sync')} was stopped prematurely`);
exit(1);
} else {
// update blockchain_last_updated value
db.update_last_updated_stats(settings.coin.name, { blockchain_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
// fix data from orphaned blocks
update_orphans(stats.orphan_index, stats.orphan_current, count, settings.sync.update_timeout, function() {
// check if the script stopped prematurely
if (stopSync) {
console.log(`${(reindex ? 'Reindex' : 'Block sync')} was stopped prematurely`);
exit(1);
} else {
db.update_richlist('received', function() {
db.update_richlist('balance', function() {
// update richlist_last_updated value
db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
db.get_stats(settings.coin.name, function(nstats) {
// check for and update heavycoin data if applicable
update_heavy(settings.coin.name, count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) {
// check for and update network history data if applicable
update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) {
// always check for and remove the sync msg if exists
db.remove_sync_message();
console.log(`${(reindex ? 'Reindex' : 'Block sync')} complete (block: %s)`, nstats.last);
exit(0);
});
});
});
});
});
});
}
});
});
}
});
}
function check_reindex_prompt(cb) {
// check if this is a reindex
if (mode == 'reindex') {
const readline = require('readline');
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
console.log('You are about to delete all blockchain data (transactions and addresses)');
console.log('and resync from the genesis block.');
// prompt for reindexing the database
rl.question('Are you sure you want to do this? [y/n]: ', function (reindexAnswer) {
// stop prompting
rl.close();
// determine if the reindex should proceed
switch (reindexAnswer) {
case 'y':
case 'Y':
case 'yes':
case 'YES':
case 'Yes':
return cb(true);
break;
default:
return cb(false);
}
});
} else {
return cb(true);
}
}
// check options
if (process.argv[2] == null || process.argv[2] == 'index' || process.argv[2] == 'update') {
mode = null;
@@ -1089,30 +1175,7 @@ if (process.argv[2] == null || process.argv[2] == 'index' || process.argv[2] ==
break;
case 'reindex':
// check if readlinesync module is installed
if (!db.fs.existsSync('./node_modules/readline-sync')) {
const { execSync } = require('child_process');
console.log('Installing missing packages.. Please wait..');
// install updated packages
execSync('npm update');
}
const readlineSync = require('readline-sync');
console.log('You are about to delete all blockchain data (transactions and addresses)');
console.log('and resync from the genesis block.');
// prompt for the reindex
if (readlineSync.keyInYN('Are you sure you want to do this? ')) {
// set mode to 'reindex'
mode = 'reindex';
} else {
console.log('Process aborted. Nothing was deleted');
exit(2);
}
break;
case 'reindex-rich':
mode = 'reindex-rich';
@@ -1133,8 +1196,12 @@ else if (process.argv[2] == 'market')
else
usage();
// check if this sync option is already running/locked
if (lib.is_locked([database]) == false) {
// check if this is a reindex which requires a prompt before continuing
check_reindex_prompt(function(reindexAnswer) {
// check if the process should continue
if (reindexAnswer) {
// check if this sync option is already running/locked
if (lib.is_locked([database]) == false) {
// create a new sync lock before checking the rest of the locks to minimize problems with running scripts at the same time
lib.create_lock(database);
// ensure the lock will be deleted on exit
@@ -1162,107 +1229,29 @@ if (lib.is_locked([database]) == false) {
console.log('Run \'npm start\' to create database structures before running this script.');
exit(1);
} else {
// determine which index mode to run
if (mode == 'reindex') {
const { execSync } = require('child_process');
console.log('Deleting database.. Please wait..');
// delete the database
execSync(`node ./scripts/delete_database.js ${mode}`, {stdio : 'inherit'});
db.update_db(settings.coin.name, function(stats) {
// check if stats returned properly
if (stats !== false) {
// determine which index mode to run
if (mode == 'reindex') {
console.log('Deleting transactions.. Please wait..');
Tx.deleteMany({}).then(() => {
console.log('Transactions deleted successfully');
console.log('Deleting addresses.. Please wait..');
Address.deleteMany({}).then(() => {
console.log('Addresses deleted successfully');
console.log('Deleting address transactions.. Please wait..');
AddressTx.deleteMany({}).then(() => {
console.log('Address transactions deleted successfully');
console.log('Deleting top 100 data.. Please wait..');
Richlist.updateOne({coin: settings.coin.name}, {
received: [],
balance: []
}).then(() => {
console.log('Top 100 data deleted successfully');
console.log('Deleting block index.. Please wait..');
Stats.updateOne({coin: settings.coin.name}, {
last: 0,
supply: 0,
txes: 0,
blockchain_last_updated: 0,
richlist_last_updated: 0,
orphan_index: 0,
orphan_current: 0
}).then(() => {
console.log('Block index deleted successfully');
// Check if the sync msg should be shown
check_show_sync_message(stats.count);
console.log('Starting resync of blockchain data.. Please wait..');
update_tx_db(settings.coin.name, block_start, stats.count, 0, settings.sync.update_timeout, 0, function() {
// check if the script stopped prematurely
if (stopSync) {
console.log('Reindex was stopped prematurely');
exit(1);
// start the block sync
block_sync(true, stats);
} else {
// update blockchain_last_updated value
db.update_last_updated_stats(settings.coin.name, { blockchain_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
// fix data from orphaned blocks
update_orphans(0, 0, stats.count, settings.sync.update_timeout, function() {
// check if the script stopped prematurely
if (stopSync) {
console.log('Reindex was stopped prematurely');
// update_db threw an error so exit
exit(1);
} else {
db.update_richlist('received', function() {
db.update_richlist('balance', function() {
// update richlist_last_updated value
db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
db.get_stats(settings.coin.name, function(nstats) {
// check for and update heavycoin data if applicable
update_heavy(settings.coin.name, stats.count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) {
// check for and update network history data if applicable
update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) {
// always check for and remove the sync msg if exists
db.remove_sync_message();
console.log('Reindex complete (block: %s)', nstats.last);
exit(0);
});
});
});
});
});
});
}
});
});
}
});
}).catch((err) => {
console.log(err);
exit(1);
});
}).catch((err) => {
console.log(err);
exit(1);
});
}).catch((err) => {
console.log(err);
exit(1);
});
}).catch((err) => {
console.log(err);
exit(1);
});
}).catch((err) => {
console.log(err);
exit(1);
});
} else if (mode == 'check') {
db.update_db(settings.coin.name, function(stats) {
// check if stats returned properly
if (stats !== false) {
console.log('Checking blocks.. Please wait..');
update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, 1, function() {
@@ -1277,55 +1266,26 @@ if (lib.is_locked([database]) == false) {
});
}
});
} else if (mode == 'update') {
// Get the last synced block index value
var last = (stats.last ? stats.last : 0);
// Get the total number of blocks
var count = (stats.count ? stats.count : 0);
// Check if the sync msg should be shown
check_show_sync_message(count - last);
update_tx_db(settings.coin.name, last, count, stats.txes, settings.sync.update_timeout, 0, function() {
// check if the script stopped prematurely
if (stopSync) {
console.log('Block sync was stopped prematurely');
exit(1);
} else {
// update blockchain_last_updated value
db.update_last_updated_stats(settings.coin.name, { blockchain_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
// fix data from orphaned blocks
update_orphans(stats.orphan_index, stats.orphan_current, count, settings.sync.update_timeout, function() {
// check if the script stopped prematurely
if (stopSync) {
console.log('Block sync was stopped prematurely');
// update_db threw an error so exit
exit(1);
} else {
db.update_richlist('received', function() {
db.update_richlist('balance', function() {
// update richlist_last_updated value
db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) {
db.get_stats(settings.coin.name, function(nstats) {
// check for and update heavycoin data if applicable
update_heavy(settings.coin.name, count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) {
// check for and update network history data if applicable
update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) {
// always check for and remove the sync msg if exists
db.remove_sync_message();
console.log('Block sync complete (block: %s)', nstats.last);
exit(0);
});
});
});
});
});
});
}
});
});
} else if (mode == 'update') {
db.update_db(settings.coin.name, function(stats) {
// check if stats returned properly
if (stats !== false) {
// start the block sync
block_sync(false, stats);
} else {
// update_db threw an error so exit
exit(1);
}
});
} else if (mode == 'reindex-rich') {
db.update_db(settings.coin.name, function(stats) {
// check if stats returned properly
if (stats !== false) {
console.log('Check richlist');
db.check_richlist(settings.coin.name, function(exists) {
@@ -1353,7 +1313,15 @@ if (lib.is_locked([database]) == false) {
});
});
});
} else {
// update_db threw an error so exit
exit(1);
}
});
} else if (mode == 'reindex-txcount') {
db.update_db(settings.coin.name, function(stats) {
// check if stats returned properly
if (stats !== false) {
console.log('Calculating tx count.. Please wait..');
// Resetting the transaction counter requires a single lookup on the txes collection to find all txes that have a positive or zero total and 1 or more vout
@@ -1372,7 +1340,15 @@ if (lib.is_locked([database]) == false) {
console.log(err);
exit(1);
});
} else {
// update_db threw an error so exit
exit(1);
}
});
} else if (mode == 'reindex-last') {
db.update_db(settings.coin.name, function(stats) {
// check if stats returned properly
if (stats !== false) {
console.log('Finding last blockindex.. Please wait..');
// Resetting the last blockindex counter requires a single lookup on the txes collection to find the last indexed blockindex
@@ -1407,13 +1383,13 @@ if (lib.is_locked([database]) == false) {
console.log(err);
exit(1);
});
}
} else {
// update_db threw an error so exit
exit(1);
}
});
}
}
});
} else if (database == 'peers') {
lib.get_peerinfo(function(body) {
@@ -1675,8 +1651,13 @@ if (lib.is_locked([database]) == false) {
console.log("Sync aborted");
exit(2);
}
} else {
} else {
// sync process is already running
console.log("Sync aborted");
exit(2);
}
}
} else {
console.log('Process aborted. Nothing was deleted');
exit(2);
}
});