Initial release

This commit is contained in:
Joe Uhren
2019-05-27 10:33:22 -07:00
commit 60fb8e29b4
506 changed files with 88789 additions and 0 deletions
+46
View File
@@ -0,0 +1,46 @@
var mongoose = require('mongoose')
, db = require('../lib/database')
, Tx = require('../models/tx')
, Address = require('../models/address')
, settings = require('../lib/settings');
var COUNT = 5000; //number of blocks to index
function exit() {
mongoose.disconnect();
process.exit(0);
}
var dbString = 'mongodb://' + settings.dbsettings.user;
dbString = dbString + ':' + settings.dbsettings.password;
dbString = dbString + '@' + settings.dbsettings.address;
dbString = dbString + ':' + settings.dbsettings.port;
dbString = dbString + "/IQUIDUS-BENCHMARK";
mongoose.connect(dbString, { useNewUrlParser: true, useCreateIndex: true }, function(err) {
if (err) {
console.log('Unable to connect to database: %s', dbString);
console.log('Aborting');
exit();
}
Tx.remove({}, function(err) {
Address.remove({}, function(err2) {
var s_timer = new Date().getTime();
db.update_tx_db(settings.coin, 1, COUNT, settings.update_timeout, function(){
var e_timer = new Date().getTime();
Tx.count({}, function(txerr, txcount){
Address.count({}, function(aerr, acount){
var stats = {
tx_count: txcount,
address_count: acount,
seconds: (e_timer - s_timer)/1000,
};
console.log(stats);
exit();
});
});
});
});
});
});
+41
View File
@@ -0,0 +1,41 @@
#!/bin/bash
ends_with() { case $2 in *"$1") true;; *) false;; esac; }
BACKUP_PATH="$(dirname $(dirname $(readlink -f "$0")))/backups"
ARCHIVE_SUFFIX=".tar.gz"
if [ -n "${1}" ]; then
# use the backup filename passed into this script
BACKUP_FILENAME="${1}"
else
# no backup filename passed in, use todays date as the backup filename
BACKUP_FILENAME=$(date +"%Y-%b-%d")
fi
if ends_with "${ARCHIVE_SUFFIX}" "${BACKUP_FILENAME}"; then
# remove the archive suffix from the backup filename
BACKUP_FILENAME=${BACKUP_FILENAME%"${ARCHIVE_SUFFIX}"}
fi
if [ $(dirname "${BACKUP_FILENAME}") != "." ]; then
# The backup filename is a full path
# Split out the path and filename
BACKUP_DIR="$(dirname ${BACKUP_FILENAME})"
TEMP_FILENAME=${BACKUP_FILENAME#"${BACKUP_DIR}/"}
BACKUP_PATH=${BACKUP_FILENAME%"/${TEMP_FILENAME}"}
BACKUP_FILENAME="${TEMP_FILENAME}"
fi
if [ ! -f "${BACKUP_PATH}/${BACKUP_FILENAME}${ARCHIVE_SUFFIX}" ]; then
# execute backup
eval "mongodump -d explorerdb -o ${BACKUP_PATH}/${BACKUP_FILENAME}"
# archive the backup
cd "${BACKUP_PATH}" && tar -cvzf "${BACKUP_PATH}/${BACKUP_FILENAME}${ARCHIVE_SUFFIX}" "${BACKUP_FILENAME}"
# delete the uncompressed backup directory
rm -rf "${BACKUP_PATH}/${BACKUP_FILENAME}"
# finished msg
echo "Backup saved successfully to ${BACKUP_PATH}/${BACKUP_FILENAME}${ARCHIVE_SUFFIX}."
else
# backup already exists
echo "A backup named ${BACKUP_FILENAME}${ARCHIVE_SUFFIX} already exists."
fi
+44
View File
@@ -0,0 +1,44 @@
#!/bin/bash
# Find the absolute path to this script file
SCRIPT_PATH=`dirname "$0"`
SCRIPT_PATH=`( cd "$SCRIPT_PATH" && pwd )`
# Remove temp file if it exists from last time
if [ -f "${SCRIPT_PATH}/del.tmp" ]; then
rm -f "${SCRIPT_PATH}/del.tmp"
fi
# Prompt for deleting database
echo "You are about to delete the entire eIquidus database."
echo "Are you sure you want to do this? [y/n]: ";
read -p "" DELETE_ANSWER
# Determine if the database should be deleted
case "$DELETE_ANSWER" in
y|Y|yes|Yes|YES) ;;
*) echo "Process aborted. Nothing was deleted." && exit ;;
esac
# Erase entire database
sudo touch "${SCRIPT_PATH}/del.tmp" && mongo <<EOF
use explorerdb
db.addresses.remove({})
db.addresses.drop()
db.coinstats.remove({})
db.coinstats.drop()
db.heavies.remove({})
db.heavies.drop()
db.markets.remove({})
db.markets.drop()
db.peers.remove({})
db.peers.drop()
db.richlists.remove({})
db.richlists.drop()
db.txes.remove({})
db.txes.drop()
exit
EOF
# Check if the temp file exists to determine if the delete was successful or not
if [ -f "${SCRIPT_PATH}/del.tmp" ]; then
rm -f "${SCRIPT_PATH}/del.tmp"
echo "Finished deleting database."
else
echo "Process aborted. Nothing was deleted."
fi
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
# this super hack will sync the explorer within the specified block height range
forcesync() {
blockcount=$1
echo "╒══════════════════<<"
echo "| height : $blockcount"
blockhash=`curl -s https://explorer.exor.io/api/getblockhash?height=$blockcount`
echo "| ଓ hash : $blockhash"
curl -s https://explorer.exor.io/block/$blockhash > /dev/null
echo "╘═══════════════════════════════>>"
}
main() {
if [ $currentblockcount -ne $endingblockcount ]; then
forcesync $currentblockcount
currentblockcount=$((currentblockcount + 1))
else exit; fi
main
}
startingblockcount=1213133
endingblockcount=1213143
echo "Syncing..."
currentblockcount=$startingblockcount
main
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
# this super hack will sync the explorer from the newest block as they occur
forcesync() {
blockcount=$1
echo "╒══════════════════<<"
echo "| height : $blockcount"
blockhash=`curl -s https://explorer.exor.io/api/getblockhash?height=$blockcount`
echo "| ଓ hash : $blockhash"
curl -s https://explorer.exor.io/block/$blockhash > /dev/null
echo "╘═══════════════════════════════>>"
}
main() {
echo "Checking for new block..."
previousblockcount=$currentblockcount
currentblockcount=`curl -s https://explorer.exor.io/api/getblockcount`
if [ $currentblockcount -ne $previousblockcount ]; then
echo "New block found. Syncing..."
forcesync $currentblockcount
else echo "No new block found. Sleeping..."; fi
sleep 20
main
}
currentblockcount=0
main
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
EXPLORER_PATH=$(dirname $(dirname $(readlink -f "$0")))
if [ -f "${EXPLORER_PATH}/tmp/index.pid" ];
then
ps -p `cat ${EXPLORER_PATH}/tmp/index.pid` > /dev/null
if [ $? -eq 0 ]; then
exit 1
else
rm "${EXPLORER_PATH}/tmp/index.pid"
fi
fi
if [ -z "${1}" ]; then
eval "cd ${EXPLORER_PATH} && $(which node) scripts/sync.js index update > /dev/null 2>&1"
else
eval "cd ${EXPLORER_PATH} && ${1} scripts/sync.js index update > /dev/null 2>&1"
fi
+104
View File
@@ -0,0 +1,104 @@
var mongoose = require('mongoose')
, lib = require('../lib/explorer')
, db = require('../lib/database')
, settings = require('../lib/settings')
, request = require('request');
var COUNT = 5000; //number of blocks to index
function exit() {
mongoose.disconnect();
process.exit(0);
}
var dbString = 'mongodb://' + settings.dbsettings.user;
dbString = dbString + ':' + settings.dbsettings.password;
dbString = dbString + '@' + settings.dbsettings.address;
dbString = dbString + ':' + settings.dbsettings.port;
dbString = dbString + '/' + settings.dbsettings.database;
mongoose.connect(dbString, { useNewUrlParser: true, useCreateIndex: true }, function(err) {
if (err) {
console.log('Unable to connect to database: %s', dbString);
console.log('Aborting');
exit();
} else {
request({uri: 'http://127.0.0.1:' + settings.port + '/api/getpeerinfo', json: true, headers: {'User-Agent': 'eiquidus'}}, function (error, response, body) {
lib.syncLoop(body.length, function (loop) {
var i = loop.iteration();
var address = body[i].addr.substring(0, body[i].addr.lastIndexOf(":")).replace("[","").replace("]","");
var rateLimit = new RateLimit(1, 2000, false);
db.find_peer(address, function(peer) {
if (peer) {
// peer already exists
loop.next();
} else {
rateLimit.schedule(function() {
request({uri: 'http://ip-api.com/json/' + address + '?fields=country', json: true, headers: {'User-Agent': 'eiquidus'}}, function (error, response, geo) {
db.create_peer({
address: address,
protocol: body[i].version,
version: body[i].subver.replace('/', '').replace('/', ''),
country: geo.country
}, function(){
loop.next();
});
});
});
}
});
}, function() {
exit();
});
});
}
});
// rate limiting class from Matteo Agosti via https://www.matteoagosti.com/blog/2013/01/22/rate-limiting-function-calls-in-javascript/
var RateLimit = (function() {
var RateLimit = function(maxOps, interval, allowBursts) {
this._maxRate = allowBursts ? maxOps : maxOps / interval;
this._interval = interval;
this._allowBursts = allowBursts;
this._numOps = 0;
this._start = new Date().getTime();
this._queue = [];
};
RateLimit.prototype.schedule = function(fn) {
var that = this,
rate = 0,
now = new Date().getTime(),
elapsed = now - this._start;
if (elapsed > this._interval) {
this._numOps = 0;
this._start = now;
}
rate = this._numOps / (this._allowBursts ? 1 : elapsed);
if (rate < this._maxRate) {
if (this._queue.length === 0) {
this._numOps++;
fn();
}
else {
if (fn) this._queue.push(fn);
this._numOps++;
this._queue.shift()();
}
}
else {
if (fn) this._queue.push(fn);
setTimeout(function() {
that.schedule();
}, 1 / this._maxRate);
}
};
return RateLimit;
})();
+67
View File
@@ -0,0 +1,67 @@
#!/bin/bash
ARCHIVE_SUFFIX=".tar.gz"
# Check if a backup file was specified
if [ -n "${1}" ]; then
BACKUP_PATH="${1}"
# Check if the backup file exists as a full path
if [ ! -f "${BACKUP_PATH}" ]; then
# Check if the backup is valid by adding the archive suffix
if [ -f "${BACKUP_PATH}${ARCHIVE_SUFFIX}" ]; then
# The backup file is valid after adding the archive suffix
BACKUP_PATH="${BACKUP_PATH}${ARCHIVE_SUFFIX}"
else
# Prepend the default backup path
BACKUP_PATH="$(dirname $(dirname $(readlink -f "$0")))/backups/${BACKUP_PATH}"
fi
fi
# Check for the backup file (again)
if [ ! -f "${BACKUP_PATH}" ]; then
# Append the default archive suffix
BACKUP_PATH="${BACKUP_PATH}${ARCHIVE_SUFFIX}"
fi
# Check for the backup file (last time)
if [ -f "${BACKUP_PATH}" ]; then
# Extract the backup archive
DIR_NAME=$(dirname "${BACKUP_PATH}")
tar -zxvf "${BACKUP_PATH}" -C "${DIR_NAME}"
# Check if this is a valid backup archive now that the files have been extracted
if [ -d ${BACKUP_PATH%"${ARCHIVE_SUFFIX}"}/explorerdb ]; then
BACKUP_DIR=${BACKUP_PATH%"${ARCHIVE_SUFFIX}"}
# Erase entire database
sudo mongo <<EOF
use explorerdb
db.addresses.remove({})
db.addresses.drop()
db.coinstats.remove({})
db.coinstats.drop()
db.heavies.remove({})
db.heavies.drop()
db.markets.remove({})
db.markets.drop()
db.peers.remove({})
db.peers.drop()
db.richlists.remove({})
db.richlists.drop()
db.txes.remove({})
db.txes.drop()
exit
EOF
# Restore mongo database from the backup directory
eval "mongorestore -d explorerdb ${BACKUP_DIR}/explorerdb"
# Remove the backup directory
rm -rf "${BACKUP_DIR}"
# Finished msg
echo "Backup restored from ${BACKUP_PATH} successfully."
else
# Backup file is not a valid mongo database backup
echo "${BACKUP_PATH} is not a valid backup file."
fi
else
# Backup does not exist
echo "${BACKUP_PATH} cannot be found."
fi
else
echo "no backup file specified."
fi
+223
View File
@@ -0,0 +1,223 @@
var mongoose = require('mongoose')
, db = require('../lib/database')
, Tx = require('../models/tx')
, Address = require('../models/address')
, Richlist = require('../models/richlist')
, Stats = require('../models/stats')
, settings = require('../lib/settings')
, fs = require('fs');
var mode = 'update';
var database = 'index';
// displays usage and exits
function usage() {
console.log('Usage: node scripts/sync.js [database] [mode]');
console.log('');
console.log('database: (required)');
console.log('index [mode] Main index: coin info/stats, transactions & addresses');
console.log('market Market data: summaries, orderbooks, trade history & chartdata')
console.log('');
console.log('mode: (required for index database only)');
console.log('update Updates index from last sync to current block');
console.log('check checks index for (and adds) any missing transactions/addresses');
console.log('reindex Clears index then resyncs from genesis to current block');
console.log('');
console.log('notes:');
console.log('* \'current block\' is the latest created block when script is executed.');
console.log('* The market database only supports (& defaults to) reindex mode.');
console.log('* If check mode finds missing data(ignoring new data since last sync),');
console.log(' index_timeout in settings.json is set too low.')
console.log('');
process.exit(0);
}
// check options
if (process.argv[2] == 'index') {
if (process.argv.length <3) {
usage();
} else {
switch(process.argv[3])
{
case 'update':
mode = 'update';
break;
case 'check':
mode = 'check';
break;
case 'reindex':
mode = 'reindex';
break;
default:
usage();
}
}
} else if (process.argv[2] == 'market'){
database = 'market';
} else {
usage();
}
function create_lock(cb) {
if ( database == 'index' ) {
var fname = './tmp/' + database + '.pid';
fs.appendFile(fname, process.pid, function (err) {
if (err) {
console.log("Error: unable to create %s", fname);
process.exit(1);
} else {
return cb();
}
});
} else {
return cb();
}
}
function remove_lock(cb) {
if ( database == 'index' ) {
var fname = './tmp/' + database + '.pid';
fs.unlink(fname, function (err){
if(err) {
console.log("unable to remove lock: %s", fname);
process.exit(1);
} else {
return cb();
}
});
} else {
return cb();
}
}
function is_locked(cb) {
if ( database == 'index' ) {
var fname = './tmp/' + database + '.pid';
fs.exists(fname, function (exists){
if(exists) {
return cb(true);
} else {
return cb(false);
}
});
} else {
return cb();
}
}
function exit() {
remove_lock(function(){
mongoose.disconnect();
process.exit(0);
});
}
var dbString = 'mongodb://' + settings.dbsettings.user;
dbString = dbString + ':' + settings.dbsettings.password;
dbString = dbString + '@' + settings.dbsettings.address;
dbString = dbString + ':' + settings.dbsettings.port;
dbString = dbString + '/' + settings.dbsettings.database;
is_locked(function (exists) {
if (exists) {
console.log("Script already running..");
process.exit(0);
} else {
create_lock(function (){
console.log("script launched with pid: " + process.pid);
mongoose.connect(dbString, { useNewUrlParser: true, useCreateIndex: true }, function(err) {
if (err) {
console.log('Unable to connect to database: %s', dbString);
console.log('Aborting');
exit();
} else if (database == 'index') {
db.check_stats(settings.coin, function(exists) {
if (exists == false) {
console.log('Run \'npm start\' to create database structures before running this script.');
exit();
} else {
db.update_db(settings.coin, function(){
db.get_stats(settings.coin, function(stats){
if (settings.heavy == true) {
db.update_heavy(settings.coin, stats.count, 20, function(){
});
}
if (mode == 'reindex') {
Tx.remove({}, function(err) {
Address.remove({}, function(err2) {
Richlist.update({coin: settings.coin}, {
received: [],
balance: [],
}, function(err3) {
Stats.update({coin: settings.coin}, {
last: 0,
}, function() {
console.log('index cleared (reindex)');
});
db.update_tx_db(settings.coin, 1, stats.count, settings.update_timeout, function(){
db.update_richlist('received', function(){
db.update_richlist('balance', function(){
db.get_stats(settings.coin, function(nstats){
console.log('reindex complete (block: %s)', nstats.last);
exit();
});
});
});
});
});
});
});
} else if (mode == 'check') {
db.update_tx_db(settings.coin, 1, stats.count, settings.check_timeout, function(){
db.get_stats(settings.coin, function(nstats){
console.log('check complete (block: %s)', nstats.last);
exit();
});
});
} else if (mode == 'update') {
db.update_tx_db(settings.coin, stats.last, stats.count, settings.update_timeout, function(){
db.update_richlist('received', function(){
db.update_richlist('balance', function(){
db.get_stats(settings.coin, function(nstats){
console.log('update complete (block: %s)', nstats.last);
exit();
});
});
});
});
}
});
});
}
});
} else {
//update markets
var markets = settings.markets.enabled;
var complete = 0;
for (var x = 0; x < markets.length; x++) {
var market = markets[x];
db.check_market(market, function(mkt, exists) {
if (exists) {
db.update_markets_db(mkt, function(err) {
if (!err) {
console.log('%s market data updated successfully.', mkt);
complete++;
} else {
console.log('%s: %s', mkt, err);
complete++;
}
});
} else {
console.log('error: entry for %s does not exists in markets db.', mkt);
complete++;
}
});
}
// Get the last usd price for coinstats
db.get_last_usd_price(function(retVal) { exit(); });
}
});
});
}
});