davide/purple-explorer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved block sync speed

Browse Source 
-A number of functions have been rewritten to be more optimized and faster: calculate_total, is_unique, convert_to_satoshi, get_input_addresses, processVoutAddresses, prepare_vout, prepare_vin
-Txes are now written to database via bulk writes which helps improve the sync speed and also controls memory usage with batching to write data once a certain threshold is reached
-update_address function changed to update_addresses since it now bulk writes the addresses in batches to improve sync speed and also controls memory usage with batching to write data once a certain threshold is reached
-The syncLoop function has been completely removed from the project and replaced with async library loops or even normal "for" loops in some cases which greatly improves sync speeds over large batches of data
-Fixed an issue with the flattened count of txes that is saved to the coinstats collection which could save incorrectly when using more than 1 thread
-Fixed an issue with the block sync which caused an unwanted delay when syncing less blocks than the amount of threads used to sync the data
-Fixed an issue with vout data processing that could sometimes populate data out of order
-Added a new sync.batch_size setting used to determine how many records (txes, addresses, addresstxes) should be saved in a single database transaction
-Added a new wait_for_bulk_database_save setting used to increase the block sync speed at the cost of not returning any error msgs for data that failed to save
-get_input_addresses function no longer returns in the exports section of the explorer.js file since it is only referenced in that file
-Updated explorerspec tests to use the newest function changes for any tests that needed to be updated

Special thanks to Karzo from Pepecoin for help with the bulkwrite code changes!
This commit is contained in:
Joe Uhren
2025-02-02 19:10:17 -07:00
parent 0b0ef817f1
commit 3a2f679201
10 changed files with 966 additions and 867 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/database.js
+92 -110
View File
@@ -1,19 +1,20 @@
var mongoose = require('mongoose'),
Stats = require('../models/stats'),
Markets = require('../models/markets'),
Masternode = require('../models/masternode'),
Address = require('../models/address'),
AddressTx = require('../models/addresstx'),
Tx = require('../models/tx'),
Orphans = require('../models/orphans'),
Richlist = require('../models/richlist'),
Peers = require('../models/peers'),
Heavy = require('../models/heavy'),
NetworkHistory = require('../models/networkhistory'),
ClaimAddress = require('../models/claimaddress'),
lib = require('./explorer'),
settings = require('./settings'),
fs = require('fs');
const mongoose = require('mongoose');
const Stats = require('../models/stats');
const Markets = require('../models/markets');
const Masternode = require('../models/masternode');
const Address = require('../models/address');
const AddressTx = require('../models/addresstx');
const Tx = require('../models/tx');
const Orphans = require('../models/orphans');
const Richlist = require('../models/richlist');
const Peers = require('../models/peers');
const Heavy = require('../models/heavy');
const NetworkHistory = require('../models/networkhistory');
const ClaimAddress = require('../models/claimaddress');
const lib = require('./explorer');
const settings = require('./settings');
const fs = require('fs');
const async = require('async');
function find_address(hash, caseSensitive, cb) {
if (caseSensitive) {
@@ -250,24 +251,22 @@ function remove_inactive_markets(installed_markets, cb) {
// check if the database has any markets installed
if (db_markets != null && db_markets.length > 0) {
// loop through the list of markets in the database
lib.syncLoop(db_markets.length, function(market_loop) {
let m = market_loop.iteration();
async.eachSeries(db_markets, function(current_market, market_loop) {
// check if this market is installed
if (installed_markets.findIndex(x => x.market.toUpperCase() == db_markets[m].market.toUpperCase() && x.coin_symbol.toUpperCase() == db_markets[m].coin_symbol.toUpperCase() && x.pair_symbol.toUpperCase() == db_markets[m].pair_symbol.toUpperCase()) == -1) {
if (installed_markets.findIndex(x => x.market.toUpperCase() == current_market.market.toUpperCase() && x.coin_symbol.toUpperCase() == current_market.coin_symbol.toUpperCase() && x.pair_symbol.toUpperCase() == current_market.pair_symbol.toUpperCase()) == -1) {
// remove this market from the database because it is not installed or active
Markets.deleteOne({market: db_markets[m].market, coin_symbol: db_markets[m].coin_symbol, pair_symbol: db_markets[m].pair_symbol}).then(() => {
Markets.deleteOne({market: current_market.market, coin_symbol: current_market.coin_symbol, pair_symbol: current_market.pair_symbol}).then(() => {
// move to the next market record
market_loop.next();
market_loop();
}).catch((err) => {
console.log(err);
// move to the next market record
market_loop.next();
market_loop();
});
} else {
// move to the next market record
market_loop.next();
market_loop();
}
}, function() {
// finished removing inactive markets
@@ -299,35 +298,30 @@ function init_heavy(cb) {
function init_claimaddress(coin, cb) {
// first, get the stats data
Stats.findOne({coin: coin}).then((stats) => {
var newer_claim_address = false;
let newer_claim_address = false;
// check if stats were found
if (stats) {
// check if the claim address data was already moved to the new collection
if (stats.newer_claim_address != null && stats.newer_claim_address == true)
newer_claim_address = true;
}
// check if stats were found and the claim address data was already moved to the new collection
if (stats && stats.newer_claim_address != null && stats.newer_claim_address == true)
newer_claim_address = true;
// check if the claim address data should be moved to a new collection
if (!newer_claim_address) {
// find all addresses with a custom claim address name
Address.find({$and: [{"name": {$ne: ""}}, {"name": {$ne: null}}]}).exec().then((addresses) => {
Address.find({$and: [{'name': {$ne: ''}}, {'name': {$ne: null}}]}).exec().then((addresses) => {
// loop through the claimed addresses
lib.syncLoop(addresses.length, function(address_loop) {
var a = address_loop.iteration();
async.eachSeries(addresses, function(current_address, address_loop) {
// create a new claimaddress record
var claim_address = new ClaimAddress({
a_id: addresses[a].a_id,
claim_name: addresses[a].name
const claim_address = new ClaimAddress({
a_id: current_address.a_id,
claim_name: current_address.name
});
// add new claim address to collection
claim_address.save().then(() => {
address_loop.next();
address_loop();
}).catch((err) => {
console.log(err);
address_loop.next();
address_loop();
});
}, function() {
// finished moving all claimed address data to the new collection
@@ -864,17 +858,17 @@ module.exports = {
},
get_txs: function(block, cb) {
var txs = [];
let txs = [];
lib.syncLoop(block.tx.length, function (loop) {
var i = loop.iteration();
find_tx(block.tx[i], function(tx) {
async.eachSeries(block.tx, function(current_tx, loop) {
find_tx(current_tx[i], function(tx) {
if (tx) {
// add tx to the list
txs.push(tx);
loop.next();
} else
loop.next();
}
// move to next tx
loop();
});
}, function() {
return cb(txs);
@@ -951,14 +945,10 @@ module.exports = {
},
get_address_txs_ajax: function(hash, start, length, cb) {
var totalCount = 0;
AddressTx.find({a_id: hash}).countDocuments().then((count) => {
totalCount = count;
AddressTx.find({a_id: hash}).countDocuments().then((totalCount) => {
AddressTx.aggregate([
{ $match: { a_id: hash } },
{ $sort: {blockindex: -1} },
{ $sort: { blockindex: -1 } },
{ $skip: Number(start) },
{
$group: {
@@ -972,29 +962,27 @@ module.exports = {
balance: '$balance'
}
},
{ $sort: {blockindex: -1} }
{ $sort: { blockindex: -1 } }
]).then((balance_sum) => {
AddressTx.find({a_id: hash}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec().then((address_tx) => {
var txs = [];
var count = address_tx.length;
var running_balance = balance_sum.length > 0 ? balance_sum[0].balance : 0;
var txs = [];
let txs = [];
let running_balance = (balance_sum.length > 0 ? balance_sum[0].balance : 0);
lib.syncLoop(count, function (loop) {
var i = loop.iteration();
find_tx(address_tx[i].txid, function (tx) {
if (tx && !txs.includes(tx)) {
async.eachSeries(address_tx, function(current_addresstx, loop) {
find_tx(current_addresstx.txid, function(tx) {
if (tx) {
// set the balance for this tx
tx.balance = running_balance;
txs.push(tx);
loop.next();
} else if (!txs.includes(tx)) {
txs.push("1. Not found");
loop.next();
} else
loop.next();
running_balance = running_balance - address_tx[i].amount;
// add tx to list of txes
txs.push(tx);
// subtract from the running balance
running_balance -= current_addresstx.amount;
}
// move to next address tx
loop();
});
}, function () {
return cb(txs, totalCount);
@@ -1133,7 +1121,7 @@ module.exports = {
},
get_distribution: function(richlist, stats, cb) {
var distribution = {
const distribution = {
supply: stats.supply,
t_1_25: {percent: 0, total: 0 },
t_26_50: {percent: 0, total: 0 },
@@ -1142,32 +1130,30 @@ module.exports = {
t_101plus: {percent: 0, total: 0 }
};
lib.syncLoop(richlist.balance.length, function (loop) {
var i = loop.iteration();
var count = i + 1;
var percentage = ((richlist.balance[i].balance / 100000000) / stats.supply) * 100;
async.timesSeries(richlist.balance.length, function(i, loop) {
const percentage = ((richlist.balance[i].balance / 100000000) / stats.supply) * 100;
if (count <= 25 ) {
if ((i + 1) <= 25) {
distribution.t_1_25.percent = distribution.t_1_25.percent + percentage;
distribution.t_1_25.total = distribution.t_1_25.total + (richlist.balance[i].balance / 100000000);
}
if (count <= 50 && count > 25) {
if ((i + 1) <= 50 && (i + 1) > 25) {
distribution.t_26_50.percent = distribution.t_26_50.percent + percentage;
distribution.t_26_50.total = distribution.t_26_50.total + (richlist.balance[i].balance / 100000000);
}
if (count <= 75 && count > 50) {
if ((i + 1) <= 75 && (i + 1) > 50) {
distribution.t_51_75.percent = distribution.t_51_75.percent + percentage;
distribution.t_51_75.total = distribution.t_51_75.total + (richlist.balance[i].balance / 100000000);
}
if (count <= 100 && count > 75) {
if ((i + 1) <= 100 && (i + 1) > 75) {
distribution.t_76_100.percent = distribution.t_76_100.percent + percentage;
distribution.t_76_100.total = distribution.t_76_100.total + (richlist.balance[i].balance / 100000000);
}
loop.next();
loop();
}, function() {
distribution.t_101plus.percent = parseFloat(100 - distribution.t_76_100.percent - distribution.t_51_75.percent - distribution.t_26_50.percent - distribution.t_1_25.percent - (settings.richlist_page.burned_coins.include_burned_coins_in_distribution == true && richlist.burned > 0 ? ((richlist.burned / 100000000) / stats.supply) * 100 : 0)).toFixed(2);
distribution.t_101plus.total = parseFloat(distribution.supply - distribution.t_76_100.total - distribution.t_51_75.total - distribution.t_26_50.total - distribution.t_1_25.total - (settings.richlist_page.burned_coins.include_burned_coins_in_distribution == true && richlist.burned > 0 ? (richlist.burned / 100000000) : 0)).toFixed(8);
@@ -1187,23 +1173,21 @@ module.exports = {
// updates heavycoin stats
// height: current block height, count: amount of votes to store
update_heavy: function(coin, height, count, cb) {
var newVotes = [];
lib.get_maxmoney(function(maxmoney) {
lib.get_maxvote(function(maxvote) {
lib.get_vote(function(vote) {
lib.get_phase(function(phase) {
lib.get_reward(function(reward) {
module.exports.get_stats(settings.coin.name, function(stats) {
lib.get_estnext(function(estnext) {
lib.get_nextin(function(nextin) {
let newVotes = [];
lib.get_maxmoney( function (maxmoney) {
lib.get_maxvote( function (maxvote) {
lib.get_vote( function (vote) {
lib.get_phase( function (phase) {
lib.get_reward( function (reward) {
module.exports.get_stats(settings.coin.name, function (stats) {
lib.get_estnext( function (estnext) {
lib.get_nextin( function (nextin) {
lib.syncLoop(count, function (loop) {
var i = loop.iteration();
lib.get_blockhash(height - i, function (hash) {
lib.get_block(hash, function (block) {
async.timesSeries(count, function(i, loop) {
lib.get_blockhash(height - i, function(hash) {
lib.get_block(hash, function(block) {
newVotes.push({ count: height - i, reward: block.reward, vote: (block && block.vote ? block.vote : 0) });
loop.next();
loop();
});
});
}, function() {
@@ -1219,7 +1203,7 @@ module.exports = {
votes: newVotes
}).then(() => {
// update reward_last_updated value
module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (new_cb) {
module.exports.update_last_updated_stats(settings.coin.name, { reward_last_updated: Math.floor(new Date() / 1000) }, function (update_success) {
console.log('Heavycoin update complete');
return cb();
});
@@ -1830,10 +1814,10 @@ module.exports = {
},
populate_claim_address_names: function(tx, cb) {
var addresses = [];
const addresses = [];
// loop through vin addresses
tx.vin.forEach(function (vin) {
tx.vin.forEach(function(vin) {
// check if this address already exists
if (addresses.indexOf(vin.addresses) == -1) {
// add address to array
@@ -1842,7 +1826,7 @@ module.exports = {
});
// loop through vout addresses
tx.vout.forEach(function (vout) {
tx.vout.forEach(function(vout) {
// check if this address already exists
if (addresses.indexOf(vout.addresses) == -1) {
// add address to array
@@ -1851,31 +1835,29 @@ module.exports = {
});
// loop through address array
lib.syncLoop(addresses.length, function(loop) {
var a = loop.iteration();
module.exports.get_claim_name(addresses[a], function(claim_name) {
async.eachSeries(addresses, function(current_address, loop) {
module.exports.get_claim_name(current_address, function(claim_name) {
if (claim_name != null && claim_name != '') {
// look for address in vin
for (v = 0; v < tx.vin.length; v++) {
for (let v = 0; v < tx.vin.length; v++) {
// check if this is the correct address
if (tx.vin[v].addresses == addresses[a]) {
if (tx.vin[v].addresses == current_address) {
// add claim name to array
tx.vin[v]['claim_name'] = claim_name;
}
}
// look for address in vout
for (v = 0; v < tx.vout.length; v++) {
for (let v = 0; v < tx.vout.length; v++) {
// check if this is the correct address
if (tx.vout[v].addresses == addresses[a]) {
if (tx.vout[v].addresses == current_address) {
// add claim name to array
tx.vout[v]['claim_name'] = claim_name;
}
}
}
loop.next();
loop();
});
}, function() {
// return modified tx object