From d9e0e54dec55aeba7791ac8616bb4101b2f7014e Mon Sep 17 00:00:00 2001 From: Joe Uhren Date: Tue, 4 Mar 2025 20:40:00 -0700 Subject: [PATCH] Allow backup and restore of a single collection -Both the backup and restore scripts now support a new optional parameter that allows backing up and restoring a single collection only -Added new verbiage and examples to the backup and restore script sections of the README -Removed extra "the"'s from some of the restore database examples in the README --- README.md | 36 ++++-- scripts/create_backup.js | 102 +++++++++++---- scripts/restore_backup.js | 254 +++++++++++++++++++++++--------------- 3 files changed, 264 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index 3a6a869..7debced 100644 --- a/README.md +++ b/README.md @@ -873,34 +873,52 @@ cd /path/to/explorer && /path/to/node ./scripts/update_explorer.js "dependencies #### Backup Database Script -Make a complete backup of an eIquidus mongo database and save to compressed file. A built-in locking mechanism prevents data from being updated or changed while a backup is in process. Backups can be safely created while the explorer is actively running and/or while the explorer is turned off. The following backup scenarios are supported: +Make a complete backup of an eIquidus mongo database or single collection and save to compressed file. A built-in locking mechanism prevents data from being updated or changed while a backup is in process. Backups can be safely created while the explorer is actively running and/or while the explorer is turned off. + +Parameters: +1. Backup path or filename (optional) +2. Collection name (optional) **NOTE:** This parameter is useful for backing up a single database collection such as the `claimaddresses` or plugin-related collections that can later be restored into an existing database without affecting any other database collections. + +The following backup scenarios are supported: **Backup Database (No filename specified)** -`npm run create-backup`: Backs up to the explorer/backups directory by default with the current date as the filename in the format yyyy-MMM-dd.bak +`npm run create-backup`: Backs up to the explorer/backups directory by default with the current date as the filename in the format yyyy-MMM-dd.bak **Backup Database (Partial filename specified)** -`npm run create-backup test`: Backs up the the explorer/backups directory by default with the filename test.bak +`npm run create-backup test`: Backs up the explorer/backups directory by default with the filename test.bak **Backup Database (Full filename specified)** -`npm run create-backup today.bak`: Backs up the the explorer/backups directory by default with the filename today.bak +`npm run create-backup today.bak`: Backs up the explorer/backups directory by default with the filename today.bak **Backup Database (Full path with partial filename specified)** -`npm run create-backup /usr/local/bin/abc`: Backs up the the /usr/local/bin directory with the filename abc.bak +`npm run create-backup /usr/local/bin/abc`: Backs up the /usr/local/bin directory with the filename abc.bak **Backup Database (Full path and filename specified)** -`npm run create-backup ~/new.bak`: Backs up the the users home directory with the filename new.bak +`npm run create-backup ~/new.bak`: Backs up the users home directory with the filename new.bak + +**Backup Database (Filename and collection both specified)** + +`npm run create-backup test claimaddresses`: Backs up only the `claimaddresses` collection to the explorer/backups directory by default with the filename test.bak + +**Backup Database (No filename specified, and backup a single collection only)** + +`npm run create-backup "" masternodes`: Backs up only the `masternodes` collection to the explorer/backups directory by default with the current date as the filename in the format yyyy-MMM-dd.bak #### Restore Database Script -Restore a previously saved eIquidus mongo database backup. :warning: **WARNING:** This will completely overwrite your existing eIquidus mongo database, so be sure to make a full backup before proceeding. A built-in locking mechanism prevents data from being updated or changed while a backup is being restored. Backups can be safely restored while the explorer is actively running and/or while the explorer is turned off. +Restore a previously saved eIquidus mongo database backup. :warning: **WARNING:** Unless a single collection name is specified, this will completely overwrite your existing eIquidus mongo database, so be sure to make a full backup before proceeding. A built-in locking mechanism prevents data from being updated or changed while a backup is being restored. Backups can be safely restored while the explorer is actively running and/or while the explorer is turned off. **NOTE:** Older v1.x eIquidus database backups were compressed into tar.gz files. These older tar.gz backups can still be restored, but you must specifically add the .tar.gz suffix. Example: `npm run restore-backup /path/to/old_backup.tar.gz` +Parameters: +1. Backup path or filename (optional) +2. Collection name (optional) **NOTE:** This parameter is useful for restoring a single database collection such as the `claimaddresses` or plugin-related collections without affecting any other database collections. This option can be used with a single collection backup or full database backup and will restore only the specified collection. + The following restore scenarios are supported: **Restore Database (Partial filename specified)** @@ -919,6 +937,10 @@ The following restore scenarios are supported: `npm run restore-backup ~/archive.bak`: Restores the ~/archive.bak file +**Restore Database (Filename and collection both specified)** + +`npm run restore-backup test claimaddresses`: Restores only the `claimaddresses` collection from the explorer/scripts/backups/test.bak file + #### Delete Database Script Wipe the eIquidus mongo database clean to start again from scratch. :warning: **WARNING:** This will completely destroy all data in your existing eIquidus mongo database, so be sure to make a full backup before proceeding. A built-in locking mechanism prevents data from being updated or changed while the database is being deleted. The process to delete the database can be executed while the explorer is actively running and/or while the explorer is turned off. diff --git a/scripts/create_backup.js b/scripts/create_backup.js index 95a1514..226ad0e 100644 --- a/scripts/create_backup.js +++ b/scripts/create_backup.js @@ -6,6 +6,7 @@ const backupLockName = 'backup'; const settings = require('../lib/settings'); let backupPath = path.join(path.dirname(__dirname), 'backups'); let backupFilename; +let singleCollection = ''; let lockCreated = false; // exit function used to cleanup lock before finishing script @@ -20,6 +21,42 @@ function exit(exitCode) { } } +// verify that the collection exists +function verify_collection_exists(cb) { + // check if the backup will be for a single collection + if (singleCollection != null && singleCollection != '') { + const mongoose = require('mongoose'); + const dbString = `mongodb://${encodeURIComponent(settings.dbsettings.user)}:${encodeURIComponent(settings.dbsettings.password)}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`; + + console.log('Connecting to database..'); + + mongoose.set('strictQuery', true); + + // connect to mongo database + mongoose.connect(dbString).then(() => { + // lookup the collection in the list of collections + mongoose.connection.db.listCollections({ name: singleCollection }).toArray().then((collections) => { + // check if the collection exists + if (collections.length > 0) { + // collection exists + return cb(false); + } else { + // collection not found + return cb(true); + } + }).catch((err) => { + console.log(err); + return cb(true); + }); + }).catch((err) => { + console.log('Error: Unable to connect to database: %s', err); + exit(999); + return cb(true); + }); + } else + return cb(false); +} + // check if a backup filename was passed into the script if (process.argv[2] != null && process.argv[2] != '') { // use the backup filename passed into this script @@ -32,6 +69,12 @@ if (process.argv[2] != null && process.argv[2] != '') { backupFilename = `${systemDate.getFullYear()}-${monthName[systemDate.getMonth()]}-${systemDate.getDate()}`; } +// check if a collection name was passed into the script +if (process.argv[3] != null && process.argv[3] != '') { + // save the collection name to a variable for later use + singleCollection = process.argv[3]; +} + // check if backup filename has the archive suffix already if (backupFilename.endsWith(archiveSuffix)) { // remove the archive suffix from the backup filename @@ -58,41 +101,52 @@ if (!fs.existsSync(path.join(backupPath, `${backupFilename}${archiveSuffix}`))) if (lib.is_locked([backupLockName]) == false) { // create a new backup lock before checking the rest of the locks to minimize problems with running scripts at the same time lib.create_lock(backupLockName); + // ensure the lock will be deleted on exit lockCreated = true; + // check all other possible locks since backups should not run at the same time that data is being changed if (lib.is_locked(['restore', 'delete', 'index', 'markets', 'peers', 'masternodes', 'plugin']) == false) { - // all tests passed. OK to run backup - console.log(`${settings.localization.script_launched }: ${process.pid}`); + // check if the collection name exists + verify_collection_exists(function(collection_error) { + // check if there was an error finding the collection by name + if (!collection_error) { + // all tests passed. OK to run backup + console.log(`${settings.localization.script_launched }: ${process.pid}`); - const { exec } = require('child_process'); - const randomDirectoryName = Math.random().toString(36).substring(2, 15) + Math.random().toString(23).substring(2, 5); + const { exec } = require('child_process'); - // execute backup - const backupProcess = exec(`mongodump --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --db="${settings.dbsettings.database}" --archive="${path.join(backupPath, backupFilename + archiveSuffix)}" --gzip`); + // execute backup + const backupProcess = exec(`mongodump --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --db="${settings.dbsettings.database}" --archive="${path.join(backupPath, backupFilename + archiveSuffix)}" --gzip${singleCollection == null || singleCollection == '' ? '' : ` --collection ${singleCollection}`}`); - backupProcess.stdout.on('data', (data) => { - console.log(data); - }); + backupProcess.stdout.on('data', (data) => { + console.log(data); + }); - backupProcess.stderr.on('data', (data) => { - console.log(Buffer.from(data).toString()); - }); + backupProcess.stderr.on('data', (data) => { + console.log(Buffer.from(data).toString()); + }); - backupProcess.on('error', (error) => { - console.log(error); - }); + backupProcess.on('error', (error) => { + console.log(error); + }); - backupProcess.on('exit', (code, signal) => { - if (code) { - console.log(`Process exit with code: ${code}`); - exit(code); - } else if (signal) { - console.log(`Process killed with signal: ${signal}`); - exit(1); + backupProcess.on('exit', (code, signal) => { + if (code) { + console.log(`Process exit with code: ${code}`); + exit(code); + } else if (signal) { + console.log(`Process killed with signal: ${signal}`); + exit(1); + } else { + console.log(`Backup saved successfully to ${path.join(backupPath, backupFilename + archiveSuffix)}`); + exit(0); + } + }); } else { - console.log(`Backup saved successfully to ${path.join(backupPath, backupFilename + archiveSuffix)}`); - exit(0); + // the collection does not exist + console.log(`Collection "${singleCollection}" does not exist`); + exit(2); } }); } else { diff --git a/scripts/restore_backup.js b/scripts/restore_backup.js index 0bd14e9..0048093 100644 --- a/scripts/restore_backup.js +++ b/scripts/restore_backup.js @@ -7,7 +7,8 @@ const restoreLockName = 'restore'; const tarModule = 'tar'; const defaultBackupPath = path.join(path.dirname(__dirname), 'backups'); const settings = require('../lib/settings'); -var lockCreated = false; +let singleCollection = ''; +let lockCreated = false; // exit function used to cleanup lock before finishing script function exit(mongoose, exitCode) { @@ -83,19 +84,13 @@ function drop_collection(mongoose, colName, cb) { } function delete_database(mongoose, cb) { - const dbString = `mongodb://${encodeURIComponent(settings.dbsettings.user)}:${encodeURIComponent(settings.dbsettings.password)}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`; - - console.log('Connecting to database..'); - - mongoose.set('strictQuery', true); - - // connect to mongo database - mongoose.connect(dbString).then(() => { + // check if a single collection is being restored + if (singleCollection != null && singleCollection == '') { // get the list of collections mongoose.connection.db.listCollections().toArray().then((collections) => { // check if there are any collections if (collections.length > 0) { - var counter = 0; + let counter = 0; // loop through all collections collections.forEach((collection) => { @@ -122,10 +117,10 @@ function delete_database(mongoose, cb) { console.log(err); return cb(true); }); - }).catch((err) => { - console.log('Error: Unable to connect to database: %s', err); - exit(mongoose, 999); - }); + } else { + // do not delete any collections since mongorestore will drop the collection automatically + return cb(true); + } } function restore_backup(mongoose, backupPath, extractedPath, gZip) { @@ -134,7 +129,7 @@ function restore_backup(mongoose, backupPath, extractedPath, gZip) { console.log(`${settings.localization.restoring_backup}.. ${settings.localization.please_wait}..`); // restore mongo database from backup - const restoreProcess = exec(`mongorestore --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --authenticationDatabase="${settings.dbsettings.database}" ${(gZip ? `--gzip --archive="${backupPath}"` : `"${extractedPath}"`)}`); + const restoreProcess = exec(`mongorestore --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --authenticationDatabase="${settings.dbsettings.database}" ${(gZip ? `--gzip --archive="${backupPath}"` : `"${extractedPath}"`)}${singleCollection == null || singleCollection == '' ? '' : ` --drop --db explorerdb --collection ${singleCollection}`}`); restoreProcess.stdout.on('data', (data) => { console.log(data); @@ -173,9 +168,73 @@ function restore_backup(mongoose, backupPath, extractedPath, gZip) { }); } +function delete_prompt(cb) { + const readline = require('readline'); + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + // check if a single collection is being restored + if (singleCollection != null && singleCollection != '') + console.log(`You are about to delete and restore the "${singleCollection}" collection from backup.`); + else + console.log('You are about to delete the current eIquidus database and restore from backup.'); + + // prompt for restoring explorer database + rl.question(`${settings.localization.are_you_sure}: `, function (restoreAnswer) { + // stop prompting + rl.close(); + + // return the answer + return cb(restoreAnswer); + }); +} + +// verify that the collection exists +function verify_collection_exists(mongoose, cb) { + const dbString = `mongodb://${encodeURIComponent(settings.dbsettings.user)}:${encodeURIComponent(settings.dbsettings.password)}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`; + + console.log('Connecting to database..'); + + mongoose.set('strictQuery', true); + + // connect to mongo database + mongoose.connect(dbString).then(() => { + // check if the restore will be for a single collection + if (singleCollection != null && singleCollection != '') { + // lookup the collection in the list of collections + mongoose.connection.db.listCollections({ name: singleCollection }).toArray().then((collections) => { + // check if the collection exists + if (collections.length > 0) { + // collection exists + return cb(false); + } else { + // collection not found + return cb(true); + } + }).catch((err) => { + console.log(err); + return cb(true); + }); + } else + return cb(false); + }).catch((err) => { + console.log('Error: Unable to connect to database: %s', err); + exit(999); + return cb(true); + }); +} + // check if a backup filename was passed into the script if (process.argv[2] != null && process.argv[2] != '') { - var backupPath = process.argv[2]; + let backupPath = process.argv[2]; + + // check if a collection name was passed into the script + if (process.argv[3] != null && process.argv[3] != '') { + // save the collection name to a variable for later use + singleCollection = process.argv[3]; + } // check if the backup filename already has a path if (!fs.existsSync(`${backupPath}`)) { @@ -197,99 +256,100 @@ if (process.argv[2] != null && process.argv[2] != '') { // check for the backup file (last time) if (fs.existsSync(`${backupPath}`)) { - const readline = require('readline'); - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }); + const mongoose = require('mongoose'); - console.log('You are about to delete the current eIquidus database and restore from backup.'); + // check if the collection name exists + verify_collection_exists(mongoose, function(collection_error) { + // check if there was an error finding the collection by name + if (!collection_error) { + // prompt for deleting and restoring the database + delete_prompt(function(restoreAnswer) { + // determine if the explorer database should be restored + switch ((restoreAnswer == null ? '' : restoreAnswer).toLowerCase()) { + case settings.localization.short_yes: + case settings.localization.long_yes: + // check if the "restore backup" process is already running + if (lib.is_locked([restoreLockName]) == false) { + // create a new restore lock before checking the rest of the locks to minimize problems with running scripts at the same time + lib.create_lock(restoreLockName); - // prompt for restoring explorer database - rl.question(`${settings.localization.are_you_sure}: `, function (restoreAnswer) { - // stop prompting - rl.close(); + // ensure the lock will be deleted on exit + lockCreated = true; - // determine if the explorer database should be restored - switch ((restoreAnswer == null ? '' : restoreAnswer).toLowerCase()) { - case settings.localization.short_yes: - case settings.localization.long_yes: - // check if the "restore backup" process is already running - if (lib.is_locked([restoreLockName]) == false) { - // create a new restore lock before checking the rest of the locks to minimize problems with running scripts at the same time - lib.create_lock(restoreLockName); - // ensure the lock will be deleted on exit - lockCreated = true; - // check all other possible locks since restoring backups should not run at the same time that data is being changed - if (lib.is_locked(['backup', 'delete', 'index', 'markets', 'peers', 'masternodes', 'plugin']) == false) { - // all tests passed. OK to run restore - console.log(`${settings.localization.script_launched }: ${process.pid}`); + // check all other possible locks since restoring backups should not run at the same time that data is being changed + if (lib.is_locked(['backup', 'delete', 'index', 'markets', 'peers', 'masternodes', 'plugin']) == false) { + // all tests passed. OK to run restore + console.log(`${settings.localization.script_launched }: ${process.pid}`); - // check if this is a tar.gz (older explorer backup format) - if (!backupPath.endsWith(oldArchiveSuffix)) { - const mongoose = require('mongoose'); + // check if this is a tar.gz (older explorer backup format) + if (!backupPath.endsWith(oldArchiveSuffix)) { + const mongoose = require('mongoose'); - // newer backup format (.bak) - // delete all collections from existing database - delete_database(mongoose, function(retVal) { - if (retVal) { - // move on to the restore process - restore_backup(mongoose, backupPath, backupPath, true); - } - }); - } else { - // older backup format (.tar.gz) - // check if the tar module is already installed - check_module_directory_exists(tarModule, function(retVal) { - const tar = require(tarModule); + // newer backup format (.bak) + // delete all collections from existing database + delete_database(mongoose, function(retVal) { + if (retVal) { + // move on to the restore process + restore_backup(mongoose, backupPath, backupPath, true); + } + }); + } else { + // older backup format (.tar.gz) + // check if the tar module is already installed + check_module_directory_exists(tarModule, function(retVal) { + const tar = require(tarModule); - console.log(`${settings.localization.extracting_backup_files}.. ${settings.localization.please_wait}..`); + console.log(`${settings.localization.extracting_backup_files}.. ${settings.localization.please_wait}..`); - // extract the backup archive - tar.x({ file: backupPath, cwd: defaultBackupPath, gzip: true }, function() { - var extractedPath = path.join(defaultBackupPath, path.basename(backupPath).replace(oldArchiveSuffix, '')); + // extract the backup archive + tar.x({ file: backupPath, cwd: defaultBackupPath, gzip: true }, function() { + var extractedPath = path.join(defaultBackupPath, path.basename(backupPath).replace(oldArchiveSuffix, '')); - // check if this is a valid backup archive now that the files have been extracted - if (fs.existsSync(`${path.join(extractedPath, settings.dbsettings.database)}`)) { - const mongoose = require('mongoose'); - - // delete all collections from existing database - delete_database(mongoose, function(retVal) { - if (retVal) { - // move on to the restore process - restore_backup(mongoose, backupPath, extractedPath, false); + // check if this is a valid backup archive now that the files have been extracted + if (fs.existsSync(`${path.join(extractedPath, settings.dbsettings.database)}`)) { + // delete all collections from existing database + delete_database(mongoose, function(retVal) { + if (retVal) { + // move on to the restore process + restore_backup(mongoose, backupPath, extractedPath, false); + } + }); + } else { + // backup file is not a valid mongo database backup + // try to remove the backup directory + try { + fs.rmSync(extractedPath, { recursive: true }); + } catch { + // do nothing + } finally { + console.log(`${path.basename(backupPath)} is not a valid backup file`); + exit(mongoose, 1); + } } }); - } else { - // backup file is not a valid mongo database backup - // try to remove the backup directory - try { - fs.rmSync(extractedPath, { recursive: true }); - } catch { - // do nothing - } finally { - console.log(`${path.basename(backupPath)} is not a valid backup file`); - exit(null, 1); - } - } - }); - }); + }); + } + } else { + // another script process is currently running + console.log("Restore aborted"); + exit(mongoose, 2); + } + } else { + // restore process is already running + console.log("Restore aborted"); + exit(mongoose, 2); } - } else { - // another script process is currently running - console.log("Restore aborted"); - exit(null, 2); - } - } else { - // restore process is already running - console.log("Restore aborted"); - exit(null, 2); - } - break; - default: - console.log(`${settings.localization.process_aborted}. ${settings.localization.nothing_was_restored}.`); - exit(null, 2); + break; + default: + console.log(`${settings.localization.process_aborted}. ${settings.localization.nothing_was_restored}.`); + exit(mongoose, 2); + } + }); + } else { + // the collection does not exist + console.log(`Collection "${singleCollection}" does not exist`); + exit(mongoose, 2); } }); } else {