davide/purple-explorer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved (sync/backup) scripts + misc updates

Browse Source 
-Added locks to all sync processes (blocks, markets, peers, masternodes) as well as "create backup", "restore backup" and "delete database" functions. This helps prevent problems with syncing data while a backup is in progress for example
-The code to initialize certain database collections on startup was moved into database.js and is now called from restore_backup.js and delete_database.js. This effectively allows the database to be deleted or restored to a completely different backup while the explorer is still running
-Lock functions (create_lock, remove_lock, is_locked) were moved into explorer.js for better reusability and rewriten to be synchronous
-is_locked function now accepts an array of lock files to be able to check for multiple locks in a single call
-remove_sync_message() function was moved into database.js so that restore_backup.js and delete_database.js can also check for and remove the sync msg if it exists
-Useful Scripts section updated in the README to make it clear that the explorer no longer needs to be stopped for these scripts to be run
-Most if not all log messages now start with a capitlal letter
This commit is contained in:
Joe Uhren
2022-04-30 20:53:10 -06:00
parent 8e32e294b7
commit d7c18a48f5
9 changed files with 1085 additions and 854 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark.js
+5 -6
View File
@@ -6,9 +6,9 @@ var mongoose = require('mongoose'),
var COUNT = 5000; // number of blocks to index
function exit() {
function exit(exitCode) {
mongoose.disconnect();
process.exit(0);
process.exit(exitCode);
}
var dbString = 'mongodb://' + settings.dbsettings.user;
@@ -19,9 +19,8 @@ dbString = dbString + "/IQUIDUS-BENCHMARK";
mongoose.connect(dbString, function(err) {
if (err) {
console.log('Unable to connect to database: %s', dbString);
console.log('Aborting');
exit();
console.log('Error: Unable to connect to database: %s', dbString);
exit(999);
}
Tx.deleteMany({}, function(err) {
@@ -40,7 +39,7 @@ mongoose.connect(dbString, function(err) {
};
console.log(stats);
exit();
exit(0);
});
});
});
scripts/create_backup.js
+63 -27
View File
@@ -1,8 +1,23 @@
const fs = require('fs');
const path = require('path');
const lib = require('../lib/explorer');
const archiveSuffix = '.bak';
const backupLockName = 'backup';
var backupPath = path.join(path.dirname(__dirname), 'backups');
var backupFilename;
var lockCreated = false;
// exit function used to cleanup lock before finishing script
function exit(exitCode) {
// only remove backup lock if it was created in this session
if (!lockCreated || lib.remove_lock(backupLockName) == true) {
// clean exit with previous exit code
process.exit(exitCode);
} else {
// error removing lock
process.exit(1);
}
}
// check if a backup filename was passed into the script
if (process.argv[2] != null && process.argv[2] != '') {
@@ -38,39 +53,60 @@ if (!fs.existsSync(backupPath)) {
// check if backup file already exists
if (!fs.existsSync(path.join(backupPath, `${backupFilename}${archiveSuffix}`))) {
const { exec } = require('child_process');
const settings = require('../lib/settings');
const randomDirectoryName = Math.random().toString(36).substring(2, 15) + Math.random().toString(23).substring(2, 5);
// check if the "create backup" process is already running
if (lib.is_locked([backupLockName]) == false) {
// create a new backup lock before checking the rest of the locks to minimize problems with running scripts at the same time
lib.create_lock(backupLockName);
// ensure the lock will be deleted on exit
lockCreated = true;
// check all other possible locks since backups should not run at the same time that data is being changed
if (lib.is_locked(['restore', 'delete', 'index', 'markets', 'peers', 'masternodes']) == false) {
// all tests passed. OK to run backup
console.log("Script launched with pid: " + process.pid);
// execute backup
const backupProcess = exec(`mongodump --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --db="${settings.dbsettings.database}" --archive="${path.join(backupPath, backupFilename + archiveSuffix)}" --gzip`);
const { exec } = require('child_process');
const settings = require('../lib/settings');
const randomDirectoryName = Math.random().toString(36).substring(2, 15) + Math.random().toString(23).substring(2, 5);
backupProcess.stdout.on('data', (data) => {
console.log(data);
});
// execute backup
const backupProcess = exec(`mongodump --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --db="${settings.dbsettings.database}" --archive="${path.join(backupPath, backupFilename + archiveSuffix)}" --gzip`);
backupProcess.stderr.on('data', (data) => {
console.log(Buffer.from(data).toString());
});
backupProcess.stdout.on('data', (data) => {
console.log(data);
});
backupProcess.on('error', (error) => {
console.log(error);
});
backupProcess.stderr.on('data', (data) => {
console.log(Buffer.from(data).toString());
});
backupProcess.on('exit', (code, signal) => {
if (code) {
console.log(`Process exit with code: ${code}`);
process.exit(1);
} else if (signal) {
console.log(`Process killed with signal: ${signal}`);
process.exit(1);
} else {
console.log(`Backup saved successfully to ${path.join(backupPath, backupFilename + archiveSuffix)}`);
process.exit(0);
}
});
backupProcess.on('error', (error) => {
console.log(error);
});
backupProcess.on('exit', (code, signal) => {
if (code) {
console.log(`Process exit with code: ${code}`);
exit(code);
} else if (signal) {
console.log(`Process killed with signal: ${signal}`);
exit(1);
} else {
console.log(`Backup saved successfully to ${path.join(backupPath, backupFilename + archiveSuffix)}`);
exit(0);
}
});
} else {
// another script process is currently running
console.log("Backup aborted");
exit(2);
}
} else {
// backup process is already running
console.log("Backup aborted");
exit(2);
}
} else {
// backup already exists
console.log(`A backup named ${backupFilename}${archiveSuffix} already exists`);
process.exit(1);
exit(2);
}
scripts/delete_database.js
+110 -55
View File
@@ -1,31 +1,66 @@
const lib = require('../lib/explorer');
const readline = require('readline');
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
const deleteLockName = 'delete';
var lockCreated = false;
// exit function used to cleanup lock before finishing script
function exit(mongoose, exitCode) {
const db = require('../lib/database');
// check if mongo was connected
if (mongoose != null) {
// check if this is a clean exit
if (exitCode == 0) {
// initialize the database
db.initialize_data_startup(function() {
// disconnect mongo connection
mongoose.disconnect();
// finish exit cleanup
finishExit(db, exitCode);
});
} else {
// disconnect mongo connection
mongoose.disconnect();
// finish exit cleanup
finishExit(db, exitCode);
}
} else {
// finish exit cleanup
finishExit(db, exitCode);
}
}
function finishExit(db, exitCode) {
// always check for and remove the sync msg if exists
db.remove_sync_message();
// only remove delete lock if it was created in this session
if (!lockCreated || lib.remove_lock(deleteLockName) == true) {
// clean exit with previous exit code
process.exit(exitCode);
} else {
// error removing lock
process.exit(1);
}
}
function drop_collection(mongoose, colName, cb) {
// attempt to delete the collection
mongoose.connection.db.dropCollection(colName, function(err, result) {
if (err || !result) {
console.log(`Unable to delete the ${colName} collection`);
console.log('Aborting');
process.exit(1);
console.log(`Error: Unable to delete the ${colName} collection`);
exit(mongoose, 1);
} else
return cb(true);
});
}
function finished_deleting(mongoose) {
console.log('Finished deleting database');
// disconnect from mongo database
mongoose.disconnect();
// delete database complete
process.exit(0);
}
console.log('You are about to delete the entire eIquidus database.');
// prompt for deleting explorer database
@@ -40,62 +75,82 @@ rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer)
case 'yes':
case 'YES':
case 'Yes':
const settings = require('../lib/settings');
const mongoose = require('mongoose');
const dbString = `mongodb://${settings.dbsettings.user}:${settings.dbsettings.password}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
// check if the "delete database" process is already running
if (lib.is_locked([deleteLockName]) == false) {
// create a new delete lock before checking the rest of the locks to minimize problems with running scripts at the same time
lib.create_lock(deleteLockName);
// ensure the lock will be deleted on exit
lockCreated = true;
// check all other possible locks since database deletion should not run at the same time that data is being changed
if (lib.is_locked(['backup', 'restore', 'index', 'markets', 'peers', 'masternodes']) == false) {
// all tests passed. OK to run delete
console.log("Script launched with pid: " + process.pid);
console.log('Connecting to database..');
const settings = require('../lib/settings');
const mongoose = require('mongoose');
const dbString = `mongodb://${settings.dbsettings.user}:${settings.dbsettings.password}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
// connect to mongo database
mongoose.connect(dbString, function(err) {
if (err) {
console.log('Unable to connect to database: %s', dbString);
console.log('Aborting');
process.exit(1);
} else {
// get the list of collections
mongoose.connection.db.listCollections().toArray(function (err, collections) {
console.log('Connecting to database..');
// connect to mongo database
mongoose.connect(dbString, function(err) {
if (err) {
console.log('Unable to list collections in database: %s', err);
console.log('Aborting');
process.exit(1);
console.log('Error: Unable to connect to database: %s', dbString);
exit(mongoose, 999);
} else {
// check if there are any collections
if (collections.length > 0) {
var counter = 0;
// get the list of collections
mongoose.connection.db.listCollections().toArray(function (err, collections) {
if (err) {
console.log('Error: Unable to list collections in database: %s', err);
exit(mongoose, 1);
} else {
// check if there are any collections
if (collections.length > 0) {
var counter = 0;
// loop through all collections
collections.forEach((collection) => {
console.log(`Deleting ${collection.name}..`);
// loop through all collections
collections.forEach((collection) => {
console.log(`Deleting ${collection.name}..`);
// delete this collection
drop_collection(mongoose, collection.name, function(retVal) {
// check if the collection was successfully deleted
if (retVal)
counter++;
// delete this collection
drop_collection(mongoose, collection.name, function(retVal) {
// check if the collection was successfully deleted
if (retVal)
counter++;
// check if the last collection was deleted
if (counter == collections.length) {
// finish the delete process
finished_deleting(mongoose);
}
});
});
} else {
// nothing to delete
console.log('Nothing to delete, the database is already empty..');
// check if the last collection was deleted
if (counter == collections.length) {
// finish the delete process
console.log('Finished deleting database');
exit(mongoose, 0);
}
});
});
} else {
// nothing to delete
console.log('Nothing to delete, the database is already empty..');
// finish the delete process
finished_deleting(mongoose);
}
// finish the delete process
exit(mongoose, 0);
}
}
});
}
});
} else {
// another script process is currently running
console.log("Delete aborted");
exit(null, 2);
}
});
} else {
// delete process is already running
console.log("Delete aborted");
exit(null, 2);
}
break;
default:
console.log('Process aborted. Nothing was deleted.');
process.exit(1);
exit(null, 2);
}
});
scripts/restore_backup.js
+126 -65
View File
@@ -1,8 +1,54 @@
const fs = require('fs');
const path = require('path');
const lib = require('../lib/explorer');
const archiveSuffix = '.bak';
const oldArchiveSuffix = '.tar.gz';
const restoreLockName = 'restore';
const defaultBackupPath = path.join(path.dirname(__dirname), 'backups');
var lockCreated = false;
// exit function used to cleanup lock before finishing script
function exit(mongoose, exitCode) {
const db = require('../lib/database');
// check if mongo was connected
if (mongoose != null) {
// check if this is a clean exit
if (exitCode == 0) {
// initialize the database
db.initialize_data_startup(function() {
// disconnect mongo connection
mongoose.disconnect();
// finish exit cleanup
finishExit(db, exitCode);
});
} else {
// disconnect mongo connection
mongoose.disconnect();
// finish exit cleanup
finishExit(db, exitCode);
}
} else {
// finish exit cleanup
finishExit(db, exitCode);
}
}
function finishExit(db, exitCode) {
// always check for and remove the sync msg if exists
db.remove_sync_message();
// only remove restore lock if it was created in this session
if (!lockCreated || lib.remove_lock(restoreLockName) == true) {
// clean exit with previous exit code
process.exit(exitCode);
} else {
// error removing lock
process.exit(1);
}
}
function check_module_directory_exists(dirName, cb) {
// check if module directory exists
@@ -24,16 +70,14 @@ function drop_collection(mongoose, colName, cb) {
// attempt to delete the collection
mongoose.connection.db.dropCollection(colName, function(err, result) {
if (err || !result) {
console.log(`Unable to delete the ${colName} collection`);
console.log('Aborting');
process.exit(1);
console.log(`Error: Unable to delete the ${colName} collection`);
exit(mongoose, 1);
} else
return cb(true);
});
}
function delete_database(settings, cb) {
const mongoose = require('mongoose');
function delete_database(mongoose, settings, cb) {
const dbString = `mongodb://${settings.dbsettings.user}:${settings.dbsettings.password}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
console.log('Connecting to database..');
@@ -41,16 +85,14 @@ function delete_database(settings, cb) {
// connect to mongo database
mongoose.connect(dbString, function(err) {
if (err) {
console.log('Unable to connect to database: %s', dbString);
console.log('Aborting');
process.exit(1);
console.log('Error: Unable to connect to database: %s', dbString);
exit(mongoose, 999);
} else {
// get the list of collections
mongoose.connection.db.listCollections().toArray(function (err, collections) {
if (err) {
console.log('Unable to list collections in database: %s', err);
console.log('Aborting');
process.exit(1);
console.log('Error: Unable to list collections in database: %s', err);
exit(mongoose, 1);
} else {
// check if there are any collections
if (collections.length > 0) {
@@ -68,9 +110,6 @@ function delete_database(settings, cb) {
// check if the last collection was deleted
if (counter == collections.length) {
// disconnect from mongo database
mongoose.disconnect();
// finished the delete process
return cb(true);
}
@@ -78,9 +117,6 @@ function delete_database(settings, cb) {
});
} else {
// nothing to delete
// disconnect from mongo database
mongoose.disconnect();
return cb(true);
}
}
@@ -89,7 +125,7 @@ function delete_database(settings, cb) {
});
}
function restore_backup(settings, backupPath, extractedPath, gZip) {
function restore_backup(mongoose, settings, backupPath, extractedPath, gZip) {
const { exec } = require('child_process');
console.log('Restoring backup.. Please wait..');
@@ -112,10 +148,10 @@ function restore_backup(settings, backupPath, extractedPath, gZip) {
restoreProcess.on('exit', (code, signal) => {
if (code) {
console.log(`Process exit with code: ${code}`);
process.exit(1);
exit(mongoose, code);
} else if (signal) {
console.log(`Process killed with signal: ${signal}`);
process.exit(1);
exit(mongoose, 1);
} else {
// check if gZip is enabled
if (!gZip) {
@@ -129,7 +165,7 @@ function restore_backup(settings, backupPath, extractedPath, gZip) {
// restore backup complete
console.log(`Backup restored from ${path.basename(backupPath)} successfully`);
process.exit(0);
exit(mongoose, 0);
}
});
}
@@ -178,67 +214,92 @@ if (process.argv[2] != null && process.argv[2] != '') {
case 'yes':
case 'YES':
case 'Yes':
const settings = require('../lib/settings');
// check if the "restore backup" process is already running
if (lib.is_locked([restoreLockName]) == false) {
// create a new restore lock before checking the rest of the locks to minimize problems with running scripts at the same time
lib.create_lock(restoreLockName);
// ensure the lock will be deleted on exit
lockCreated = true;
// check all other possible locks since restoring backups should not run at the same time that data is being changed
if (lib.is_locked(['backup', 'delete', 'index', 'markets', 'peers', 'masternodes']) == false) {
// all tests passed. OK to run restore
console.log("Script launched with pid: " + process.pid);
// check if this is a tar.gz (older explorer backup format)
if (!backupPath.endsWith(oldArchiveSuffix)) {
// newer backup format (.bak)
// delete all collections from existing database
delete_database(settings, function(retVal) {
if (retVal) {
// move on to the restore process
restore_backup(settings, backupPath, backupPath, true);
}
});
} else {
// older backup format (.tar.gz)
// check if the tar module is already installed
check_module_directory_exists('tar', function(retVal) {
const tar = require('tar');
const settings = require('../lib/settings');
console.log('Extracting backup files.. Please wait..');
// check if this is a tar.gz (older explorer backup format)
if (!backupPath.endsWith(oldArchiveSuffix)) {
const mongoose = require('mongoose');
// extract the backup archive
tar.x({ file: backupPath, cwd: defaultBackupPath, gzip: true }, function() {
var extractedPath = path.join(defaultBackupPath, path.basename(backupPath).replace(oldArchiveSuffix, ''));
// newer backup format (.bak)
// delete all collections from existing database
delete_database(mongoose, settings, function(retVal) {
if (retVal) {
// move on to the restore process
restore_backup(mongoose, settings, backupPath, backupPath, true);
}
});
} else {
// older backup format (.tar.gz)
// check if the tar module is already installed
check_module_directory_exists('tar', function(retVal) {
const tar = require('tar');
// check if this is a valid backup archive now that the files have been extracted
if (fs.existsSync(`${path.join(extractedPath, settings.dbsettings.database)}`)) {
// delete all collections from existing database
delete_database(settings, function(retVal) {
if (retVal) {
// move on to the restore process
restore_backup(settings, backupPath, extractedPath, false);
console.log('Extracting backup files.. Please wait..');
// extract the backup archive
tar.x({ file: backupPath, cwd: defaultBackupPath, gzip: true }, function() {
var extractedPath = path.join(defaultBackupPath, path.basename(backupPath).replace(oldArchiveSuffix, ''));
// check if this is a valid backup archive now that the files have been extracted
if (fs.existsSync(`${path.join(extractedPath, settings.dbsettings.database)}`)) {
const mongoose = require('mongoose');
// delete all collections from existing database
delete_database(mongoose, settings, function(retVal) {
if (retVal) {
// move on to the restore process
restore_backup(mongoose, settings, backupPath, extractedPath, false);
}
});
} else {
// backup file is not a valid mongo database backup
// try to remove the backup directory
try {
fs.rmSync(extractedPath, { recursive: true });
} catch {
// do nothing
} finally {
console.log(`${path.basename(backupPath)} is not a valid backup file`);
exit(null, 1);
}
}
});
} else {
// backup file is not a valid mongo database backup
// try to remove the backup directory
try {
fs.rmSync(extractedPath, { recursive: true });
} catch {
// do nothing
} finally {
console.log(`${path.basename(backupPath)} is not a valid backup file`);
process.exit(1);
}
}
});
});
});
}
} else {
// another script process is currently running
console.log("Restore aborted");
exit(null, 2);
}
} else {
// restore process is already running
console.log("Restore aborted");
exit(null, 2);
}
break;
default:
console.log('Process aborted. Nothing was restored.');
process.exit(1);
exit(null, 2);
}
});
} else {
// backup does not exist
console.log(`${backupPath} cannot be found`);
process.exit(1);
exit(null, 2);
}
} else {
console.log('No backup file specified');
process.exit(1);
exit(null, 2);
}
scripts/sync.js
+478 -585
View File
File diff suppressed because it is too large Load Diff