Improved (sync/backup) scripts + misc updates

-Added locks to all sync processes (blocks, markets, peers, masternodes) as well as "create backup", "restore backup" and "delete database" functions. This helps prevent problems with syncing data while a backup is in progress for example
-The code to initialize certain database collections on startup was moved into database.js and is now called from restore_backup.js and delete_database.js. This effectively allows the database to be deleted or restored to a completely different backup while the explorer is still running
-Lock functions (create_lock, remove_lock, is_locked) were moved into explorer.js for better reusability and rewriten to be synchronous
-is_locked function now accepts an array of lock files to be able to check for multiple locks in a single call
-remove_sync_message() function was moved into database.js so that restore_backup.js and delete_database.js can also check for and remove the sync msg if it exists
-Useful Scripts section updated in the README to make it clear that the explorer no longer needs to be stopped for these scripts to be run
-Most if not all log messages now start with a capitlal letter

README.md

@@ -719,7 +719,7 @@ jQuery(document).ready(function($) {
 
 #### Backup Database Script
 
-Make a complete backup of an eIquidus mongo database collection and save to compressed file. Please note that you must ensure that the explorer is NOT running at the time of backup to prevent corrupting the backup data. The following backup scenarios are supported:
+Make a complete backup of an eIquidus mongo database and save to compressed file. A built-in locking mechanism prevents data from being updated or changed while a backup is in process. Backups can be safely created while the explorer is actively running and/or while the explorer is turned off. The following backup scenarios are supported:
 
 **Backup Database (No filename specified)**
 
@@ -743,7 +743,7 @@ Make a complete backup of an eIquidus mongo database collection and save to comp
 
 #### Restore Database Script
 
-Restore a previously saved eIquidus mongo database collection backup. :warning: **WARNING:** This will completely overwrite your existing eIquidus mongo database, so be sure to make a full backup before proceeding. Please note that the explorer should NOT be running at the time of restore to prevent problems restoring the database.
+Restore a previously saved eIquidus mongo database backup. :warning: **WARNING:** This will completely overwrite your existing eIquidus mongo database, so be sure to make a full backup before proceeding. A built-in locking mechanism prevents data from being updated or changed while a backup is being restored. Backups can be safely restored while the explorer is actively running and/or while the explorer is turned off.
 
 **NOTE:** Older v1.x eIquidus database backups were compressed into tar.gz files. These older tar.gz backups can still be restored, but you must specifically add the .tar.gz suffix. Example: `npm run restore-backup /path/to/old_backup.tar.gz`
 
@@ -767,7 +767,7 @@ The following restore scenarios are supported:
 
 #### Delete Database Script
 
-Completely wipe the eIquidus mongo database collection clean to start again from scratch. :warning: **WARNING:** This will completely destroy all data in your existing eIquidus mongo database, so be sure to make a full backup before proceeding. Please note that the explorer should NOT be running at the time of database deletion to prevent database related problems. Delete the mongo database with the following command:
+Wipe the eIquidus mongo database clean to start again from scratch. :warning: **WARNING:** This will completely destroy all data in your existing eIquidus mongo database, so be sure to make a full backup before proceeding. A built-in locking mechanism prevents data from being updated or changed while the database is being deleted. The process to delete the database can be executed while the explorer is actively running and/or while the explorer is turned off. Delete the mongo database with the following command:
 
 **Delete Database**
 

bin/instance

@@ -13,80 +13,20 @@ dbString = dbString + ':' + settings.dbsettings.port;
 dbString = dbString + '/' + settings.dbsettings.database;
 
 db.connect(dbString, function() {
-  db.check_stats(settings.coin.name, function(exists) {
+  // initialize the database
-    if (exists == false) {
+  db.initialize_data_startup(function() {
-      console.log('no stats entry found, creating now..');
+    var server = app.listen(app.get('port'), '::', function() {
-      db.create_stats(settings.coin.name, function() {});
+      debug('Express server listening on port ' + server.address().port);
-    } else {
-      db.get_stats(settings.coin.name, function (stats) {
-        app.locals.stats = stats;
-      });
-    }
-  });
-
-  // check markets/exchanges
-  if (settings.markets_page.enabled == true) {
-    // loop through and test all exchanges defined in the settings.json file
-    Object.keys(settings.markets_page.exchanges).forEach(function (key, index, map) {
-      // check if market is enabled via settings
-      if (settings.markets_page.exchanges[key].enabled == true) {
-        // check if exchange is installed/supported
-        if (db.fs.existsSync('./lib/markets/' + key + '.js')) {
-          // loop through all trading pairs
-          settings.markets_page.exchanges[key].trading_pairs.forEach(function (pair_key, pair_index, pair_map) {
-            // split the pair data
-            var split_pair = pair_key.split('/');
-            // check if this is a valid trading pair
-            if (split_pair.length == 2) {
-              // lookup the exchange in the market collection
-              db.check_market(key, split_pair[0], split_pair[1], function(market, exists) {
-                // check if exchange trading pair exists in the market collection
-                if (!exists) {
-                  // exchange doesn't exist in the market collection so add a default definition now
-                  console.log('no %s: %s entry found, creating now..', market, pair_key);
-                  db.create_market(split_pair[0], split_pair[1], market, function() {});
-                }
-              });
-            }
-          });
-        }
-      }
     });
-  }
 
-  // Add new field(s) to tx collection if missing
+    process.on('SIGINT', () => {
-  db.check_txes(function(exists) {});
+      server.close(() => {
+        var mongoose = require('mongoose');
 
-  // Add new field(s) to masternode collection if missing
+        mongoose.connection.close(false, () => {
-  db.check_masternodes(function(exists) {});
+          // close the main process now that all http and database connections have closed
-
+          process.exit(0);
-  db.check_richlist(settings.coin.name, function(exists) {
+        });
-    if (exists == false) {
-      console.log('no richlist entry found, creating now..');
-      db.create_richlist(settings.coin.name, function() {});
-    }
-  });
-
-  if (settings.blockchain_specific.heavycoin.enabled == true) {
-    db.check_heavy(settings.coin.name, function(exists) {
-      if (exists == false) {
-        console.log('no heavycoin entry found, creating now..');
-        db.create_heavy(settings.coin.name, function() {});
-      }
-    });
-  }
-
-  var server = app.listen(app.get('port'), '::', function() {
-    debug('Express server listening on port ' + server.address().port);
-  });
-
-  process.on('SIGINT', () => {
-    server.close(() => {
-      var mongoose = require('mongoose');
-
-      mongoose.connection.close(false, () => {
-        // close the main process now that all http and database connections have closed
-        process.exit(0);
       });
     });
   });

lib/database.js

@@ -212,13 +212,102 @@ function hex_to_ascii(hex) {
     return str;
 }
 
+function init_markets(cb) {
+  // check if markets/exchanges feature is enabled
+  if (settings.markets_page.enabled == true) {
+    var marketCounter = 0;
+
+    // loop through and test all exchanges defined in the settings.json file
+    Object.keys(settings.markets_page.exchanges).forEach(function (key, index, map) {
+      // check if market is enabled via settings
+      if (settings.markets_page.exchanges[key].enabled == true) {
+        // check if exchange is installed/supported
+        if (module.exports.fs.existsSync('./lib/markets/' + key + '.js')) {
+          var pairCounter = 0;
+
+          // loop through all trading pairs
+          settings.markets_page.exchanges[key].trading_pairs.forEach(function (pair_key, pair_index, pair_map) {
+            // split the pair data
+            var split_pair = pair_key.split('/');
+            // check if this is a valid trading pair
+            if (split_pair.length == 2) {
+              // lookup the exchange in the market collection
+              module.exports.check_market(key, split_pair[0], split_pair[1], function(market, exists) {
+                // check if exchange trading pair exists in the market collection
+                if (!exists) {
+                  // exchange doesn't exist in the market collection so add a default definition now
+                  console.log('No %s: %s entry found. Creating new entry now..', market, pair_key);
+                  module.exports.create_market(split_pair[0], split_pair[1], market, function() {
+                    pairCounter++;
+
+                    // check if all pairs have been tested
+                    if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length)
+                      marketCounter++;
+
+                    // check if all exchanges have been tested
+                    if (marketCounter == Object.keys(settings.markets_page.exchanges).length) {
+                      // finished initializing markets
+                      return cb();
+                    }
+                  });
+                } else {
+                  pairCounter++;
+
+                  // check if all pairs have been tested
+                  if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length)
+                    marketCounter++;
+                }
+
+                // check if all exchanges have been tested
+                if (marketCounter == Object.keys(settings.markets_page.exchanges).length) {
+                  // finished initializing markets
+                  return cb();
+                }
+              });
+            } else {
+              pairCounter++;
+
+              // check if all pairs have been tested
+              if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length)
+                marketCounter++;
+            }
+          });
+        } else
+          marketCounter++;
+      } else
+        marketCounter++;
+    });
+
+    // check if all exchanges have been tested
+    if (marketCounter == Object.keys(settings.markets_page.exchanges).length) {
+      // finished initializing markets
+      return cb();
+    }
+  } else
+    return cb();
+}
+
+function init_heavy(cb) {
+  if (settings.blockchain_specific.heavycoin.enabled == true) {
+    module.exports.check_heavy(settings.coin.name, function(exists) {
+      if (exists == false) {
+        console.log('No heavycoin entry found. Creating new entry now..');
+        module.exports.create_heavy(settings.coin.name, function() {
+          return cb();
+        });
+      } else
+        return cb();
+    });
+  } else
+    return cb();
+}
+
 module.exports = {
   // initialize DB
   connect: function(database, cb) {
     mongoose.connect(database, function(err) {
       if (err) {
-        console.log('Unable to connect to database: %s', database);
+        console.log('Error: Unable to connect to database: %s', database);
-        console.log('Aborting');
         process.exit(999);
       }
 
@@ -377,21 +466,25 @@ module.exports = {
     });
   },
 
-  create_stats: function(coin, cb) {
+  create_stats: function(coin, skip, cb) {
-    var newStats = new Stats({
+    // check if stats need to be created
-      coin: coin,
+    if (!skip) {
-      last: 0
+      var newStats = new Stats({
-    });
+        coin: coin,
+        last: 0
+      });
 
-    newStats.save(function(err) {
+      newStats.save(function(err) {
-      if (err) {
+        if (err) {
-        console.log(err);
+          console.log(err);
-        return cb();
+          return cb();
-      } else {
+        } else {
-        console.log("initial stats entry created for %s", coin);
+          console.log("Initial stats entry created for %s", coin);
-        return cb();
+          return cb();
-      }
+        }
-    });
+      });
+    } else
+      return cb();
   },
 
   get_address: function(hash, caseSensitive, cb) {
@@ -630,7 +723,7 @@ module.exports = {
         console.log(err);
         return cb();
       } else {
-        console.log("initial market entry created for %s: %s", market, coin_symbol +'/' + pair_symbol);
+        console.log("Initial market entry created for %s: %s", market, coin_symbol +'/' + pair_symbol);
         return cb();
       }
     });
@@ -653,21 +746,25 @@ module.exports = {
     });
   },
 
-  // creates initial richlist entry in database; called on first launch of explorer
+  // creates initial richlist entry in database; called on first launch of explorer + after restore or delete database
-  create_richlist: function(coin, cb) {
+  create_richlist: function(coin, skip, cb) {
-    var newRichlist = new Richlist({
+    // check if stats need to be created
-      coin: coin
+    if (!skip) {
-    });
+      var newRichlist = new Richlist({
+        coin: coin
+      });
 
-    newRichlist.save(function(err) {
+      newRichlist.save(function(err) {
-      if (err) {
+        if (err) {
-        console.log(err);
+          console.log(err);
-        return cb();
+          return cb();
-      } else {
+        } else {
-        console.log("initial richlist entry created for %s", coin);
+          console.log("Initial richlist entry created for %s", coin);
-        return cb();
+          return cb();
-      }
+        }
-    });
+      });
+    } else
+      return cb();
   },
 
   // drops richlist data for given coin
@@ -700,7 +797,7 @@ module.exports = {
         console.log(err);
         return cb();
       } else {
-        console.log("initial heavycoin entry created for %s", coin);
+        console.log("Initial heavycoin entry created for %s", coin);
         return cb();
       }
     });
@@ -812,7 +909,7 @@ module.exports = {
                       }, function() {
                         // update reward_last_updated value
                         module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (new_cb) {
-                          console.log('heavycoin update complete');
+                          console.log('Heavycoin update complete');
                           return cb();
                         });
                       });
@@ -849,7 +946,7 @@ module.exports = {
             newNetworkHistory.save(function(err) {
               // check for errors
               if (err) {
-                console.log('error updating network history: ' + err);
+                console.log('Error updating network history: ' + err);
                 return cb();
               } else {
                 // get the count of network history records
@@ -866,12 +963,12 @@ module.exports = {
 
                       // delete old network history records
                       NetworkHistory.deleteMany({blockindex: {$in: ids}}, function(err) {
-                        console.log('network history update complete');
+                        console.log('Network history update complete');
                         return cb();
                       });
                     });
                   } else {
-                    console.log('network history update complete');
+                    console.log('Network history update complete');
                     return cb();
                   }
                 });
@@ -977,7 +1074,7 @@ module.exports = {
     lib.get_blockcount( function (count) {
       // check to ensure count is a positive number
       if (!count || (count != null && typeof count === 'number' && count < 0)) {
-        console.log('Unable to connect to explorer API');
+        console.log('Error: Unable to connect to explorer API');
 
         return cb(false);
       }
@@ -993,7 +1090,7 @@ module.exports = {
                 connections: (connections ? connections : 0)
               }, function(err) {
                 if (err)
-                  console.log("Error during Stats Update: ", err);
+                  console.log("Error during stats update: %s", err);
 
                 return cb({
                   coin: coin,
@@ -1005,7 +1102,7 @@ module.exports = {
                 });
               });
             } else {
-              console.log("Error during Stats Update: ", (err ? err : 'cannot find stats collection'));
+              console.log("Error during stats update: %s", (err ? err : 'Cannot find stats collection'));
               return cb(false);
             }
           });
@@ -1038,7 +1135,7 @@ module.exports = {
           txes: txes
         }, function() {});
       } else if (check_only) {
-        console.log('checking block ' + block_height + '...');
+        console.log('Checking block ' + block_height + '...');
       }
 
       lib.get_blockhash(block_height, function(blockhash) {
@@ -1077,7 +1174,7 @@ module.exports = {
                 }, timeout);
               });
             } else {
-              console.log('block not found: %s', blockhash);
+              console.log('Block not found: %s', blockhash);
 
               setTimeout( function() {
                 next_block();
@@ -1203,7 +1300,7 @@ module.exports = {
   // add or update a single masternode
   add_update_masternode(masternode, add, cb) {
     if (masternode.proTxHash == null && masternode.txhash == null) {
-      console.log('Masternode Update - TXid is missing');
+      console.log('Masternode update error: TXid is missing');
 
       return cb(false);
     } else {
@@ -1444,5 +1541,67 @@ module.exports = {
     });
   },
 
+  initialize_data_startup: function(cb) {
+    console.log('Initializing database.. Please wait...');
+
+    // check if stats collection is initialized
+    module.exports.check_stats(settings.coin.name, function(stats_exists) {
+      var skip = true;
+
+      // determine if stats collection already exists
+      if (stats_exists == false) {
+        console.log('No stats entry found. Creating new entry now..');
+        skip = false;
+      }
+
+      // initialize the stats collection
+      module.exports.create_stats(settings.coin.name, skip, function() {
+        // check and initialize the markets collection
+        init_markets(function() {
+          // add new field(s) to tx collection if missing
+          module.exports.check_txes(function(txes_exists) {
+            // add new field(s) to masternode collection if missing
+            module.exports.check_masternodes(function(masternodes_exists) {
+              // check if richlist collection is initialized
+              module.exports.check_richlist(settings.coin.name, function(richlist_exists) {
+                skip = true;
+
+                // determine if richlist collection already exists
+                if (richlist_exists == false) {
+                  console.log('No richlist entry found. Creating new entry now..');
+                  skip = false;
+                }
+
+                // initialize the richlist collection
+                module.exports.create_richlist(settings.coin.name, skip, function() {
+                  // check and initialize the heavycoin collection
+                  init_heavy(function() {
+                    // finished initializing startup data
+                    console.log('Database initialization complete');
+                    return cb();
+                  });
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+  },
+
+  remove_sync_message: function() {
+    var filePath = './tmp/show_sync_message.tmp';
+
+    // Check if the show sync stub file exists
+    if (fs.existsSync(filePath)) {
+      // File exists, so delete it now
+      try {
+        fs.unlinkSync(filePath);
+      } catch (err) {
+        console.log(err);
+      }
+    }
+  },
+
   fs: fs
 };

lib/explorer.js

@@ -1315,5 +1315,93 @@ module.exports = {
     }, function() {
       return cb(arr_vin, tx_type);
     });
+  },
+
+  create_lock: function(lock) {
+    const fs = require('fs');
+    var fname = './tmp/' + lock + '.pid';
+
+    try {
+      fs.appendFileSync(fname, process.pid.toString());
+      return true;
+    } catch(err) {
+      console.log("Error: Unable to remove lock: %s", fname);
+      return false;
+    }
+  },
+
+  remove_lock: function(lock) {
+    const fs = require('fs');
+    var fname = './tmp/' + lock + '.pid';
+
+    try {
+      fs.unlinkSync(fname);
+      return true;
+    } catch(err) {
+      console.log("Error: Unable to remove lock: %s", fname);
+      return false;
+    }
+  },
+
+  is_locked: function(lock_array) {
+    const fs = require('fs');
+    const path = require('path');
+    var retVal = false;
+
+    // loop through all lock files that need to be checked
+    for (var i = 0; i < lock_array.length; i++) {
+      var pidFile = path.join(path.dirname(__dirname), 'tmp', `${lock_array[i]}.pid`);
+
+      // check if the script is already running (tmp/file.pid file already exists)
+      if (fs.existsSync(pidFile)) {
+        const { execSync } = require('child_process');
+        var deactivateLock = false;
+
+        // the pid file exists
+        // determine the operating system
+        switch (process.platform) {
+          case 'win32':
+            // windows
+            // run a cmd that will determine if the lock should still be active
+            var cmdResult = execSync(`tasklist /FI "PID eq ${fs.readFileSync(pidFile).toString()}"`);
+
+            // check if the process that created the lock is actually still running (crude check by testing for # of carriage returns or node.exe process running, but should work universally across different systems and languages)
+            if (cmdResult.toString().split('\n').length < 4 || cmdResult.toString().toLowerCase().indexOf('\nnode.exe') == -1) {
+              // lock should be deactivated
+              deactivateLock = true;
+            }
+
+            break;
+          default:
+            // linux or other
+            // run a cmd that will determine if the lock should still be active
+
+            try {
+              var cmdResult = execSync('ps -p `cat ' + pidFile + '` > /dev/null');
+            } catch (err) {
+              // if an error occurs, the process is NOT running and therefore the lock should be deactivated
+              deactivateLock = true;
+            }
+        }
+
+        // check if the lock should be deactivated
+        if (deactivateLock) {
+          // script is not actually running so the lock file can be deleted
+          try {
+            fs.rmSync(pidFile);
+          } catch(err) {
+            console.log(`Failed to delete lock file ${pidFile}: ${err}`);
+          }
+        } else {
+          // script is running
+          console.log(`${lock_array[i]} script is running..`);
+          retVal = true;
+
+          break;
+        }
+      }
+    }
+
+    return retVal;
   }
 };

scripts/benchmark.js

@@ -6,9 +6,9 @@ var mongoose = require('mongoose'),
 
 var COUNT = 5000; // number of blocks to index
 
-function exit() {
+function exit(exitCode) {
   mongoose.disconnect();
-  process.exit(0);
+  process.exit(exitCode);
 }
 
 var dbString = 'mongodb://' + settings.dbsettings.user;
@@ -19,9 +19,8 @@ dbString = dbString + "/IQUIDUS-BENCHMARK";
 
 mongoose.connect(dbString, function(err) {
   if (err) {
-    console.log('Unable to connect to database: %s', dbString);
+    console.log('Error: Unable to connect to database: %s', dbString);
-    console.log('Aborting');
+    exit(999);
-    exit();
   }
 
   Tx.deleteMany({}, function(err) {
@@ -40,7 +39,7 @@ mongoose.connect(dbString, function(err) {
             };
 
             console.log(stats);
-            exit();
+            exit(0);
           });
         });
       });

scripts/create_backup.js

@@ -1,8 +1,23 @@
 const fs = require('fs');
 const path = require('path');
+const lib = require('../lib/explorer');
 const archiveSuffix = '.bak';
+const backupLockName = 'backup';
 var backupPath = path.join(path.dirname(__dirname), 'backups');
 var backupFilename;
+var lockCreated = false;
+
+// exit function used to cleanup lock before finishing script
+function exit(exitCode) {
+  // only remove backup lock if it was created in this session
+  if (!lockCreated || lib.remove_lock(backupLockName) == true) {
+    // clean exit with previous exit code
+    process.exit(exitCode);
+  } else {
+    // error removing lock
+    process.exit(1);
+  }
+}
 
 // check if a backup filename was passed into the script
 if (process.argv[2] != null && process.argv[2] != '') {
@@ -38,39 +53,60 @@ if (!fs.existsSync(backupPath)) {
 
 // check if backup file already exists
 if (!fs.existsSync(path.join(backupPath, `${backupFilename}${archiveSuffix}`))) {
-    const { exec } = require('child_process');
+  // check if the "create backup" process is already running
-    const settings = require('../lib/settings');
+  if (lib.is_locked([backupLockName]) == false) {
-    const randomDirectoryName = Math.random().toString(36).substring(2, 15) + Math.random().toString(23).substring(2, 5);
+    // create a new backup lock before checking the rest of the locks to minimize problems with running scripts at the same time
+    lib.create_lock(backupLockName);
+    // ensure the lock will be deleted on exit
+    lockCreated = true;
+    // check all other possible locks since backups should not run at the same time that data is being changed
+    if (lib.is_locked(['restore', 'delete', 'index', 'markets', 'peers', 'masternodes']) == false) {
+      // all tests passed. OK to run backup
+      console.log("Script launched with pid: " + process.pid);
 
-    // execute backup
+      const { exec } = require('child_process');
-    const backupProcess = exec(`mongodump --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --db="${settings.dbsettings.database}" --archive="${path.join(backupPath, backupFilename + archiveSuffix)}" --gzip`);
+      const settings = require('../lib/settings');
+      const randomDirectoryName = Math.random().toString(36).substring(2, 15) + Math.random().toString(23).substring(2, 5);
 
-    backupProcess.stdout.on('data', (data) => {
+      // execute backup
-      console.log(data);
+      const backupProcess = exec(`mongodump --host="${settings.dbsettings.address}" --port="${settings.dbsettings.port}" --username="${settings.dbsettings.user}" --password="${settings.dbsettings.password}" --db="${settings.dbsettings.database}" --archive="${path.join(backupPath, backupFilename + archiveSuffix)}" --gzip`);
-    });
 
-    backupProcess.stderr.on('data', (data) => {
+      backupProcess.stdout.on('data', (data) => {
-      console.log(Buffer.from(data).toString());
+        console.log(data);
-    });
+      });
 
-    backupProcess.on('error', (error) => {
+      backupProcess.stderr.on('data', (data) => {
-      console.log(error);
+        console.log(Buffer.from(data).toString());
-    });
+      });
 
-    backupProcess.on('exit', (code, signal) => {
+      backupProcess.on('error', (error) => {
-      if (code) {
+        console.log(error);
-        console.log(`Process exit with code: ${code}`);
+      });
-        process.exit(1);
+
-      } else if (signal) {
+      backupProcess.on('exit', (code, signal) => {
-        console.log(`Process killed with signal: ${signal}`);
+        if (code) {
-        process.exit(1);
+          console.log(`Process exit with code: ${code}`);
-      } else {
+          exit(code);
-        console.log(`Backup saved successfully to ${path.join(backupPath, backupFilename + archiveSuffix)}`);
+        } else if (signal) {
-        process.exit(0);
+          console.log(`Process killed with signal: ${signal}`);
-      }
+          exit(1);
-    });
+        } else {
+          console.log(`Backup saved successfully to ${path.join(backupPath, backupFilename + archiveSuffix)}`);
+          exit(0);
+        }
+      });
+    } else {
+      // another script process is currently running
+      console.log("Backup aborted");
+      exit(2);
+    }
+  } else {
+    // backup process is already running
+    console.log("Backup aborted");
+    exit(2);
+  }
 } else {
   // backup already exists
   console.log(`A backup named ${backupFilename}${archiveSuffix} already exists`);
-  process.exit(1);
+  exit(2);
 }

scripts/delete_database.js

@@ -1,31 +1,66 @@
+const lib = require('../lib/explorer');
 const readline = require('readline');
 const rl = readline.createInterface({
   input: process.stdin,
   output: process.stdout
 });
+const deleteLockName = 'delete';
+var lockCreated = false;
+
+// exit function used to cleanup lock before finishing script
+function exit(mongoose, exitCode) {
+  const db = require('../lib/database');
+
+  // check if mongo was connected
+  if (mongoose != null) {
+    // check if this is a clean exit
+    if (exitCode == 0) {
+      // initialize the database
+      db.initialize_data_startup(function() {
+        // disconnect mongo connection
+        mongoose.disconnect();
+
+        // finish exit cleanup
+        finishExit(db, exitCode);
+      });
+    } else {
+      // disconnect mongo connection
+      mongoose.disconnect();
+
+      // finish exit cleanup
+      finishExit(db, exitCode);
+    }
+  } else {
+    // finish exit cleanup
+    finishExit(db, exitCode);
+  }
+}
+
+function finishExit(db, exitCode) {
+  // always check for and remove the sync msg if exists
+  db.remove_sync_message();
+
+  // only remove delete lock if it was created in this session
+  if (!lockCreated || lib.remove_lock(deleteLockName) == true) {
+    // clean exit with previous exit code
+    process.exit(exitCode);
+  } else {
+    // error removing lock
+    process.exit(1);
+  }  
+}
 
 function drop_collection(mongoose, colName, cb) {
   // attempt to delete the collection
   mongoose.connection.db.dropCollection(colName, function(err, result) {
     if (err || !result) {
-      console.log(`Unable to delete the ${colName} collection`);
+      console.log(`Error: Unable to delete the ${colName} collection`);
-      console.log('Aborting');
+      exit(mongoose, 1);
-      process.exit(1);
     } else
       return cb(true);
   });
 }
 
-function finished_deleting(mongoose) {
-  console.log('Finished deleting database');
-
-  // disconnect from mongo database
-  mongoose.disconnect();
-
-  // delete database complete
-  process.exit(0);
-}
-
 console.log('You are about to delete the entire eIquidus database.');
 
 // prompt for deleting explorer database
@@ -40,62 +75,82 @@ rl.question('Are you sure you want to do this? [y/n]: ', function (deleteAnswer)
     case 'yes':
     case 'YES':
     case 'Yes':
-      const settings = require('../lib/settings');
+      // check if the "delete database" process is already running
-      const mongoose = require('mongoose');
+      if (lib.is_locked([deleteLockName]) == false) {
-      const dbString = `mongodb://${settings.dbsettings.user}:${settings.dbsettings.password}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
+        // create a new delete lock before checking the rest of the locks to minimize problems with running scripts at the same time
+        lib.create_lock(deleteLockName);
+        // ensure the lock will be deleted on exit
+        lockCreated = true;
+        // check all other possible locks since database deletion should not run at the same time that data is being changed
+        if (lib.is_locked(['backup', 'restore', 'index', 'markets', 'peers', 'masternodes']) == false) {
+          // all tests passed. OK to run delete
+          console.log("Script launched with pid: " + process.pid);
 
-      console.log('Connecting to database..');
+          const settings = require('../lib/settings');
+          const mongoose = require('mongoose');
+          const dbString = `mongodb://${settings.dbsettings.user}:${settings.dbsettings.password}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
 
-      // connect to mongo database
+          console.log('Connecting to database..');
-      mongoose.connect(dbString, function(err) {
+
-        if (err) {
+          // connect to mongo database
-          console.log('Unable to connect to database: %s', dbString);
+          mongoose.connect(dbString, function(err) {
-          console.log('Aborting');
-          process.exit(1);
-        } else {
-          // get the list of collections
-          mongoose.connection.db.listCollections().toArray(function (err, collections) {
             if (err) {
-              console.log('Unable to list collections in database: %s', err);
+              console.log('Error: Unable to connect to database: %s', dbString);
-              console.log('Aborting');
+              exit(mongoose, 999);
-              process.exit(1);
             } else {
-              // check if there are any collections
+              // get the list of collections
-              if (collections.length > 0) {
+              mongoose.connection.db.listCollections().toArray(function (err, collections) {
-                var counter = 0;
+                if (err) {
+                  console.log('Error: Unable to list collections in database: %s', err);
+                  exit(mongoose, 1);
+                } else {
+                  // check if there are any collections
+                  if (collections.length > 0) {
+                    var counter = 0;
 
-                // loop through all collections
+                    // loop through all collections
-                collections.forEach((collection) => {
+                    collections.forEach((collection) => {
-                  console.log(`Deleting ${collection.name}..`);
+                      console.log(`Deleting ${collection.name}..`);
 
-                  // delete this collection
+                      // delete this collection
-                  drop_collection(mongoose, collection.name, function(retVal) {
+                      drop_collection(mongoose, collection.name, function(retVal) {
-                    // check if the collection was successfully deleted
+                        // check if the collection was successfully deleted
-                    if (retVal)
+                        if (retVal)
-                      counter++;
+                          counter++;
 
-                    // check if the last collection was deleted
+                        // check if the last collection was deleted
-                    if (counter == collections.length) {
+                        if (counter == collections.length) {
-                      // finish the delete process
+                          // finish the delete process
-                      finished_deleting(mongoose);
+                          console.log('Finished deleting database');
-                    }
+                          exit(mongoose, 0);
-                  });
+                        }
-                });
+                      });
-              } else {
+                    });
-                // nothing to delete
+                  } else {
-                console.log('Nothing to delete, the database is already empty..');
+                    // nothing to delete
+                    console.log('Nothing to delete, the database is already empty..');
 
-                // finish the delete process
+                    // finish the delete process
-                finished_deleting(mongoose);
+                    exit(mongoose, 0);
-              }
+                  }
+                }
+              });
             }
           });
+        } else {
+          // another script process is currently running
+          console.log("Delete aborted");
+          exit(null, 2);
         }
-      });
+      } else {
+        // delete process is already running
+        console.log("Delete aborted");
+        exit(null, 2);
+      }
 
       break;
     default:
       console.log('Process aborted. Nothing was deleted.');
-      process.exit(1);
+      exit(null, 2);
   }
 });

scripts/restore_backup.js

@@ -1,8 +1,54 @@
 const fs = require('fs');
 const path = require('path');
+const lib = require('../lib/explorer');
 const archiveSuffix = '.bak';
 const oldArchiveSuffix = '.tar.gz';
+const restoreLockName = 'restore';
 const defaultBackupPath = path.join(path.dirname(__dirname), 'backups');
+var lockCreated = false;
+
+// exit function used to cleanup lock before finishing script
+function exit(mongoose, exitCode) {
+  const db = require('../lib/database');
+
+  // check if mongo was connected
+  if (mongoose != null) {
+    // check if this is a clean exit
+    if (exitCode == 0) {
+      // initialize the database
+      db.initialize_data_startup(function() {
+        // disconnect mongo connection
+        mongoose.disconnect();
+
+        // finish exit cleanup
+        finishExit(db, exitCode);
+      });
+    } else {
+      // disconnect mongo connection
+      mongoose.disconnect();
+
+      // finish exit cleanup
+      finishExit(db, exitCode);
+    }
+  } else {
+    // finish exit cleanup
+    finishExit(db, exitCode);
+  }
+}
+
+function finishExit(db, exitCode) {
+  // always check for and remove the sync msg if exists
+  db.remove_sync_message();
+
+  // only remove restore lock if it was created in this session
+  if (!lockCreated || lib.remove_lock(restoreLockName) == true) {
+    // clean exit with previous exit code
+    process.exit(exitCode);
+  } else {
+    // error removing lock
+    process.exit(1);
+  }
+}
 
 function check_module_directory_exists(dirName, cb) {
   // check if module directory exists
@@ -24,16 +70,14 @@ function drop_collection(mongoose, colName, cb) {
   // attempt to delete the collection
   mongoose.connection.db.dropCollection(colName, function(err, result) {
     if (err || !result) {
-      console.log(`Unable to delete the ${colName} collection`);
+      console.log(`Error: Unable to delete the ${colName} collection`);
-      console.log('Aborting');
+      exit(mongoose, 1);
-      process.exit(1);
     } else
       return cb(true);
   });
 }
 
-function delete_database(settings, cb) {
+function delete_database(mongoose, settings, cb) {
-  const mongoose = require('mongoose');
   const dbString = `mongodb://${settings.dbsettings.user}:${settings.dbsettings.password}@${settings.dbsettings.address}:${settings.dbsettings.port}/${settings.dbsettings.database}`;
 
   console.log('Connecting to database..');
@@ -41,16 +85,14 @@ function delete_database(settings, cb) {
   // connect to mongo database
   mongoose.connect(dbString, function(err) {
     if (err) {
-      console.log('Unable to connect to database: %s', dbString);
+      console.log('Error: Unable to connect to database: %s', dbString);
-      console.log('Aborting');
+      exit(mongoose, 999);
-      process.exit(1);
     } else {
       // get the list of collections
       mongoose.connection.db.listCollections().toArray(function (err, collections) {
         if (err) {
-          console.log('Unable to list collections in database: %s', err);
+          console.log('Error: Unable to list collections in database: %s', err);
-          console.log('Aborting');
+          exit(mongoose, 1);
-          process.exit(1);
         } else {
           // check if there are any collections
           if (collections.length > 0) {
@@ -68,9 +110,6 @@ function delete_database(settings, cb) {
 
                 // check if the last collection was deleted
                 if (counter == collections.length) {
-                  // disconnect from mongo database
-                  mongoose.disconnect();
-
                   // finished the delete process
                   return cb(true);
                 }
@@ -78,9 +117,6 @@ function delete_database(settings, cb) {
             });
           } else {
             // nothing to delete
-            // disconnect from mongo database
-            mongoose.disconnect();
-
             return cb(true);
           }
         }
@@ -89,7 +125,7 @@ function delete_database(settings, cb) {
   });
 }
 
-function restore_backup(settings, backupPath, extractedPath, gZip) {
+function restore_backup(mongoose, settings, backupPath, extractedPath, gZip) {
   const { exec } = require('child_process');
 
   console.log('Restoring backup.. Please wait..');
@@ -112,10 +148,10 @@ function restore_backup(settings, backupPath, extractedPath, gZip) {
   restoreProcess.on('exit', (code, signal) => {
     if (code) {
       console.log(`Process exit with code: ${code}`);
-      process.exit(1);
+      exit(mongoose, code);
     } else if (signal) {
       console.log(`Process killed with signal: ${signal}`);
-      process.exit(1);
+      exit(mongoose, 1);
     } else {
       // check if gZip is enabled
       if (!gZip) {
@@ -129,7 +165,7 @@ function restore_backup(settings, backupPath, extractedPath, gZip) {
 
       // restore backup complete
       console.log(`Backup restored from ${path.basename(backupPath)} successfully`);
-      process.exit(0);
+      exit(mongoose, 0);
     }
   });
 }
@@ -178,67 +214,92 @@ if (process.argv[2] != null && process.argv[2] != '') {
         case 'yes':
         case 'YES':
         case 'Yes':
-          const settings = require('../lib/settings');
+          // check if the "restore backup" process is already running
+          if (lib.is_locked([restoreLockName]) == false) {
+            // create a new restore lock before checking the rest of the locks to minimize problems with running scripts at the same time
+            lib.create_lock(restoreLockName);
+            // ensure the lock will be deleted on exit
+            lockCreated = true;
+            // check all other possible locks since restoring backups should not run at the same time that data is being changed
+            if (lib.is_locked(['backup', 'delete', 'index', 'markets', 'peers', 'masternodes']) == false) {
+              // all tests passed. OK to run restore
+              console.log("Script launched with pid: " + process.pid);
 
-          // check if this is a tar.gz (older explorer backup format)
+              const settings = require('../lib/settings');
-          if (!backupPath.endsWith(oldArchiveSuffix)) {
-            // newer backup format (.bak)
-            // delete all collections from existing database
-            delete_database(settings, function(retVal) {
-              if (retVal) {
-                // move on to the restore process
-                restore_backup(settings, backupPath, backupPath, true);
-              }
-            });
-          } else {
-            // older backup format (.tar.gz)
-            // check if the tar module is already installed
-            check_module_directory_exists('tar', function(retVal) {
-              const tar = require('tar');
 
-              console.log('Extracting backup files.. Please wait..');
+              // check if this is a tar.gz (older explorer backup format)
+              if (!backupPath.endsWith(oldArchiveSuffix)) {
+                const mongoose = require('mongoose');
 
-              // extract the backup archive
+                // newer backup format (.bak)
-              tar.x({ file: backupPath, cwd: defaultBackupPath, gzip: true }, function() {
+                // delete all collections from existing database
-                var extractedPath = path.join(defaultBackupPath, path.basename(backupPath).replace(oldArchiveSuffix, ''));
+                delete_database(mongoose, settings, function(retVal) {
+                  if (retVal) {
+                    // move on to the restore process
+                    restore_backup(mongoose, settings, backupPath, backupPath, true);
+                  }
+                });
+              } else {
+                // older backup format (.tar.gz)
+                // check if the tar module is already installed
+                check_module_directory_exists('tar', function(retVal) {
+                  const tar = require('tar');
 
-                // check if this is a valid backup archive now that the files have been extracted
+                  console.log('Extracting backup files.. Please wait..');
-                if (fs.existsSync(`${path.join(extractedPath, settings.dbsettings.database)}`)) {
+
-                  // delete all collections from existing database
+                  // extract the backup archive
-                  delete_database(settings, function(retVal) {
+                  tar.x({ file: backupPath, cwd: defaultBackupPath, gzip: true }, function() {
-                    if (retVal) {
+                    var extractedPath = path.join(defaultBackupPath, path.basename(backupPath).replace(oldArchiveSuffix, ''));
-                      // move on to the restore process
+
-                      restore_backup(settings, backupPath, extractedPath, false);
+                    // check if this is a valid backup archive now that the files have been extracted
+                    if (fs.existsSync(`${path.join(extractedPath, settings.dbsettings.database)}`)) {
+                      const mongoose = require('mongoose');
+
+                      // delete all collections from existing database
+                      delete_database(mongoose, settings, function(retVal) {
+                        if (retVal) {
+                          // move on to the restore process
+                          restore_backup(mongoose, settings, backupPath, extractedPath, false);
+                        }
+                      });
+                    } else {
+                      // backup file is not a valid mongo database backup
+                      // try to remove the backup directory
+                      try {
+                        fs.rmSync(extractedPath, { recursive: true });
+                      } catch {
+                        // do nothing
+                      } finally {
+                        console.log(`${path.basename(backupPath)} is not a valid backup file`);
+                        exit(null, 1);
+                      }
                     }
                   });
-                } else {
+                });
-                  // backup file is not a valid mongo database backup
+              }
-                  // try to remove the backup directory
+            } else {
-                  try {
+              // another script process is currently running
-                    fs.rmSync(extractedPath, { recursive: true });
+              console.log("Restore aborted");
-                  } catch {
+              exit(null, 2);
-                    // do nothing
+            }
-                  } finally {
+          } else {
-                    console.log(`${path.basename(backupPath)} is not a valid backup file`);
+            // restore process is already running
-                    process.exit(1);
+            console.log("Restore aborted");
-                  }
+            exit(null, 2);
-                }
-              });
-            });
           }
 
           break;
         default:
           console.log('Process aborted. Nothing was restored.');
-          process.exit(1);
+          exit(null, 2);
       }
     });
   } else {
     // backup does not exist
     console.log(`${backupPath} cannot be found`);
-    process.exit(1);
+    exit(null, 2);
   }
 } else {
   console.log('No backup file specified');
-  process.exit(1);
+  exit(null, 2);
 }