davide/purple-explorer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved (sync/backup) scripts + misc updates

Browse Source 
-Added locks to all sync processes (blocks, markets, peers, masternodes) as well as "create backup", "restore backup" and "delete database" functions. This helps prevent problems with syncing data while a backup is in progress for example
-The code to initialize certain database collections on startup was moved into database.js and is now called from restore_backup.js and delete_database.js. This effectively allows the database to be deleted or restored to a completely different backup while the explorer is still running
-Lock functions (create_lock, remove_lock, is_locked) were moved into explorer.js for better reusability and rewriten to be synchronous
-is_locked function now accepts an array of lock files to be able to check for multiple locks in a single call
-remove_sync_message() function was moved into database.js so that restore_backup.js and delete_database.js can also check for and remove the sync msg if it exists
-Useful Scripts section updated in the README to make it clear that the explorer no longer needs to be stopped for these scripts to be run
-Most if not all log messages now start with a capitlal letter
This commit is contained in:
Joe Uhren
2022-04-30 20:53:10 -06:00
parent 8e32e294b7
commit d7c18a48f5
9 changed files with 1085 additions and 854 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/database.js
+201 -42
View File
@@ -212,13 +212,102 @@ function hex_to_ascii(hex) {
return str;
}
function init_markets(cb) {
// check if markets/exchanges feature is enabled
if (settings.markets_page.enabled == true) {
var marketCounter = 0;
// loop through and test all exchanges defined in the settings.json file
Object.keys(settings.markets_page.exchanges).forEach(function (key, index, map) {
// check if market is enabled via settings
if (settings.markets_page.exchanges[key].enabled == true) {
// check if exchange is installed/supported
if (module.exports.fs.existsSync('./lib/markets/' + key + '.js')) {
var pairCounter = 0;
// loop through all trading pairs
settings.markets_page.exchanges[key].trading_pairs.forEach(function (pair_key, pair_index, pair_map) {
// split the pair data
var split_pair = pair_key.split('/');
// check if this is a valid trading pair
if (split_pair.length == 2) {
// lookup the exchange in the market collection
module.exports.check_market(key, split_pair[0], split_pair[1], function(market, exists) {
// check if exchange trading pair exists in the market collection
if (!exists) {
// exchange doesn't exist in the market collection so add a default definition now
console.log('No %s: %s entry found. Creating new entry now..', market, pair_key);
module.exports.create_market(split_pair[0], split_pair[1], market, function() {
pairCounter++;
// check if all pairs have been tested
if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length)
marketCounter++;
// check if all exchanges have been tested
if (marketCounter == Object.keys(settings.markets_page.exchanges).length) {
// finished initializing markets
return cb();
}
});
} else {
pairCounter++;
// check if all pairs have been tested
if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length)
marketCounter++;
}
// check if all exchanges have been tested
if (marketCounter == Object.keys(settings.markets_page.exchanges).length) {
// finished initializing markets
return cb();
}
});
} else {
pairCounter++;
// check if all pairs have been tested
if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length)
marketCounter++;
}
});
} else
marketCounter++;
} else
marketCounter++;
});
// check if all exchanges have been tested
if (marketCounter == Object.keys(settings.markets_page.exchanges).length) {
// finished initializing markets
return cb();
}
} else
return cb();
}
function init_heavy(cb) {
if (settings.blockchain_specific.heavycoin.enabled == true) {
module.exports.check_heavy(settings.coin.name, function(exists) {
if (exists == false) {
console.log('No heavycoin entry found. Creating new entry now..');
module.exports.create_heavy(settings.coin.name, function() {
return cb();
});
} else
return cb();
});
} else
return cb();
}
module.exports = {
// initialize DB
connect: function(database, cb) {
mongoose.connect(database, function(err) {
if (err) {
console.log('Unable to connect to database: %s', database);
console.log('Aborting');
console.log('Error: Unable to connect to database: %s', database);
process.exit(999);
}
@@ -377,21 +466,25 @@ module.exports = {
});
},
create_stats: function(coin, cb) {
var newStats = new Stats({
coin: coin,
last: 0
});
create_stats: function(coin, skip, cb) {
// check if stats need to be created
if (!skip) {
var newStats = new Stats({
coin: coin,
last: 0
});
newStats.save(function(err) {
if (err) {
console.log(err);
return cb();
} else {
console.log("initial stats entry created for %s", coin);
return cb();
}
});
newStats.save(function(err) {
if (err) {
console.log(err);
return cb();
} else {
console.log("Initial stats entry created for %s", coin);
return cb();
}
});
} else
return cb();
},
get_address: function(hash, caseSensitive, cb) {
@@ -630,7 +723,7 @@ module.exports = {
console.log(err);
return cb();
} else {
console.log("initial market entry created for %s: %s", market, coin_symbol +'/' + pair_symbol);
console.log("Initial market entry created for %s: %s", market, coin_symbol +'/' + pair_symbol);
return cb();
}
});
@@ -653,21 +746,25 @@ module.exports = {
});
},
// creates initial richlist entry in database; called on first launch of explorer
create_richlist: function(coin, cb) {
var newRichlist = new Richlist({
coin: coin
});
// creates initial richlist entry in database; called on first launch of explorer + after restore or delete database
create_richlist: function(coin, skip, cb) {
// check if stats need to be created
if (!skip) {
var newRichlist = new Richlist({
coin: coin
});
newRichlist.save(function(err) {
if (err) {
console.log(err);
return cb();
} else {
console.log("initial richlist entry created for %s", coin);
return cb();
}
});
newRichlist.save(function(err) {
if (err) {
console.log(err);
return cb();
} else {
console.log("Initial richlist entry created for %s", coin);
return cb();
}
});
} else
return cb();
},
// drops richlist data for given coin
@@ -700,7 +797,7 @@ module.exports = {
console.log(err);
return cb();
} else {
console.log("initial heavycoin entry created for %s", coin);
console.log("Initial heavycoin entry created for %s", coin);
return cb();
}
});
@@ -812,7 +909,7 @@ module.exports = {
}, function() {
// update reward_last_updated value
module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (new_cb) {
console.log('heavycoin update complete');
console.log('Heavycoin update complete');
return cb();
});
});
@@ -849,7 +946,7 @@ module.exports = {
newNetworkHistory.save(function(err) {
// check for errors
if (err) {
console.log('error updating network history: ' + err);
console.log('Error updating network history: ' + err);
return cb();
} else {
// get the count of network history records
@@ -866,12 +963,12 @@ module.exports = {
// delete old network history records
NetworkHistory.deleteMany({blockindex: {$in: ids}}, function(err) {
console.log('network history update complete');
console.log('Network history update complete');
return cb();
});
});
} else {
console.log('network history update complete');
console.log('Network history update complete');
return cb();
}
});
@@ -977,7 +1074,7 @@ module.exports = {
lib.get_blockcount( function (count) {
// check to ensure count is a positive number
if (!count || (count != null && typeof count === 'number' && count < 0)) {
console.log('Unable to connect to explorer API');
console.log('Error: Unable to connect to explorer API');
return cb(false);
}
@@ -993,7 +1090,7 @@ module.exports = {
connections: (connections ? connections : 0)
}, function(err) {
if (err)
console.log("Error during Stats Update: ", err);
console.log("Error during stats update: %s", err);
return cb({
coin: coin,
@@ -1005,7 +1102,7 @@ module.exports = {
});
});
} else {
console.log("Error during Stats Update: ", (err ? err : 'cannot find stats collection'));
console.log("Error during stats update: %s", (err ? err : 'Cannot find stats collection'));
return cb(false);
}
});
@@ -1038,7 +1135,7 @@ module.exports = {
txes: txes
}, function() {});
} else if (check_only) {
console.log('checking block ' + block_height + '...');
console.log('Checking block ' + block_height + '...');
}
lib.get_blockhash(block_height, function(blockhash) {
@@ -1077,7 +1174,7 @@ module.exports = {
}, timeout);
});
} else {
console.log('block not found: %s', blockhash);
console.log('Block not found: %s', blockhash);
setTimeout( function() {
next_block();
@@ -1203,7 +1300,7 @@ module.exports = {
// add or update a single masternode
add_update_masternode(masternode, add, cb) {
if (masternode.proTxHash == null && masternode.txhash == null) {
console.log('Masternode Update - TXid is missing');
console.log('Masternode update error: TXid is missing');
return cb(false);
} else {
@@ -1444,5 +1541,67 @@ module.exports = {
});
},
initialize_data_startup: function(cb) {
console.log('Initializing database.. Please wait...');
// check if stats collection is initialized
module.exports.check_stats(settings.coin.name, function(stats_exists) {
var skip = true;
// determine if stats collection already exists
if (stats_exists == false) {
console.log('No stats entry found. Creating new entry now..');
skip = false;
}
// initialize the stats collection
module.exports.create_stats(settings.coin.name, skip, function() {
// check and initialize the markets collection
init_markets(function() {
// add new field(s) to tx collection if missing
module.exports.check_txes(function(txes_exists) {
// add new field(s) to masternode collection if missing
module.exports.check_masternodes(function(masternodes_exists) {
// check if richlist collection is initialized
module.exports.check_richlist(settings.coin.name, function(richlist_exists) {
skip = true;
// determine if richlist collection already exists
if (richlist_exists == false) {
console.log('No richlist entry found. Creating new entry now..');
skip = false;
}
// initialize the richlist collection
module.exports.create_richlist(settings.coin.name, skip, function() {
// check and initialize the heavycoin collection
init_heavy(function() {
// finished initializing startup data
console.log('Database initialization complete');
return cb();
});
});
});
});
});
});
});
});
},
remove_sync_message: function() {
var filePath = './tmp/show_sync_message.tmp';
// Check if the show sync stub file exists
if (fs.existsSync(filePath)) {
// File exists, so delete it now
try {
fs.unlinkSync(filePath);
} catch (err) {
console.log(err);
}
}
},
fs: fs
};
lib/explorer.js
+88
View File
@@ -1315,5 +1315,93 @@ module.exports = {
}, function() {
return cb(arr_vin, tx_type);
});
},
create_lock: function(lock) {
const fs = require('fs');
var fname = './tmp/' + lock + '.pid';
try {
fs.appendFileSync(fname, process.pid.toString());
return true;
} catch(err) {
console.log("Error: Unable to remove lock: %s", fname);
return false;
}
},
remove_lock: function(lock) {
const fs = require('fs');
var fname = './tmp/' + lock + '.pid';
try {
fs.unlinkSync(fname);
return true;
} catch(err) {
console.log("Error: Unable to remove lock: %s", fname);
return false;
}
},
is_locked: function(lock_array) {
const fs = require('fs');
const path = require('path');
var retVal = false;
// loop through all lock files that need to be checked
for (var i = 0; i < lock_array.length; i++) {
var pidFile = path.join(path.dirname(__dirname), 'tmp', `${lock_array[i]}.pid`);
// check if the script is already running (tmp/file.pid file already exists)
if (fs.existsSync(pidFile)) {
const { execSync } = require('child_process');
var deactivateLock = false;
// the pid file exists
// determine the operating system
switch (process.platform) {
case 'win32':
// windows
// run a cmd that will determine if the lock should still be active
var cmdResult = execSync(`tasklist /FI "PID eq ${fs.readFileSync(pidFile).toString()}"`);
// check if the process that created the lock is actually still running (crude check by testing for # of carriage returns or node.exe process running, but should work universally across different systems and languages)
if (cmdResult.toString().split('\n').length < 4 || cmdResult.toString().toLowerCase().indexOf('\nnode.exe') == -1) {
// lock should be deactivated
deactivateLock = true;
}
break;
default:
// linux or other
// run a cmd that will determine if the lock should still be active
try {
var cmdResult = execSync('ps -p `cat ' + pidFile + '` > /dev/null');
} catch (err) {
// if an error occurs, the process is NOT running and therefore the lock should be deactivated
deactivateLock = true;
}
}
// check if the lock should be deactivated
if (deactivateLock) {
// script is not actually running so the lock file can be deleted
try {
fs.rmSync(pidFile);
} catch(err) {
console.log(`Failed to delete lock file ${pidFile}: ${err}`);
}
} else {
// script is running
console.log(`${lock_array[i]} script is running..`);
retVal = true;
break;
}
}
}
return retVal;
}
};