From 09fa91968645b63321813d190749d6502a6fbaa8 Mon Sep 17 00:00:00 2001 From: Joe Uhren Date: Sat, 6 May 2023 12:36:35 -0600 Subject: [PATCH] Orphan block fix + other misc improvements -The block sync will now remove orphaned block data from the txes collecton, address balances/sent/received data as well as addresstx data and now stores limited info in a new orphans collection -Added a new optional page for viewing orphaned block data -The coinbase address now updates sent totals from POS rewards and other transactions that have vout but no vin addresses -Block and transaction pages now display a warning when viewing an orphaned block or tx -Added a couple new fields to the coinstats collection for tracking orphaned blocks -Added new locale strings for the orphaned block feature --- README.md | 1 + app.js | 34 ++ lib/database.js | 35 +- lib/explorer.js | 22 +- lib/locale.js | 11 + lib/settings.js | 32 ++ locale/en.json | 11 + models/orphans.js | 12 + models/stats.js | 4 +- routes/index.js | 216 ++++++---- scripts/sync.js | 894 +++++++++++++++++++++++++++++++++++++---- settings.json.template | 32 ++ views/block.pug | 8 + views/layout.pug | 17 + views/orphans.pug | 123 ++++++ views/tx.pug | 8 + 16 files changed, 1301 insertions(+), 159 deletions(-) create mode 100644 models/orphans.js create mode 100644 views/orphans.pug diff --git a/README.md b/README.md index 5a1e2f4..b1d65db 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Table of Contents - **getmasternoderewards:** Returns a list of masternode reward transactions for a specific address that arrived after a specific block height *\*only applicable to masternode coins* - **getmasternoderewardstotal:** Returns the total number of coins earned in masternode rewards for a specific address that arrived after a specific block height *\*only applicable to masternode coins* - **Claim Address:** Allows anyone to set custom display names for wallet addresses that they own using the **Sign Message** feature from their local wallet. Includes *bad word* filter support. + - **Orphaned Blocks:** Displays a list of orphaned blocks with links to the next and previous "good" blocks - **Block Info:** Displays block summary and list of transactions for a specific block height - **Transaction Info:** Displays transaction summary, optional OP_RETURN value, list of input addresses and output addresses for a specific transaction - **Address Info:** Displays wallet address summary (balance, total sent, total received, QR code) and a list of latest transactions for a specific wallet address diff --git a/app.js b/app.js index 31efd78..939a6f0 100644 --- a/app.js +++ b/app.js @@ -592,6 +592,39 @@ app.use('/ext/getmasternoderewardstotal/:hash/:since', function(req, res) { res.end('This method is disabled'); }); +// get the list of orphans from local collection +app.use('/ext/getorphanlist/:start/:length', function(req, res) { + // check the headers to see if it matches an internal ajax request from the explorer itself (TODO: come up with a more secure method of whitelisting ajax calls from the explorer) + if (req.headers['x-requested-with'] != null && req.headers['x-requested-with'].toLowerCase() == 'xmlhttprequest' && req.headers.referer != null && req.headers.accept.indexOf('text/javascript') > -1 && req.headers.accept.indexOf('application/json') > -1) { + // fix parameters + if (typeof req.params.start === 'undefined' || isNaN(req.params.start) || req.params.start < 0) + req.params.start = 0; + if (typeof req.params.length === 'undefined' || isNaN(req.params.length)) + req.params.length = 10; + + // get the orphan list from local collection + db.get_orphans(req.params.start, req.params.length, function(orphans, count) { + var data = []; + + for (i = 0; i < orphans.length; i++) { + var row = []; + + row.push(orphans[i].blockindex); + row.push(orphans[i].orphan_blockhash); + row.push(orphans[i].good_blockhash); + row.push(orphans[i].prev_blockhash); + row.push(orphans[i].next_blockhash); + + data.push(row); + } + + // display data formatted for internal datatable + res.json({"data": data, "recordsTotal": count, "recordsFiltered": count}); + }); + } else + res.end('This method is disabled'); +}); + app.use('/ext/getnetworkchartdata', function(req, res) { db.get_network_chart_data(function(data) { if (data) @@ -766,6 +799,7 @@ app.set('richlist_page', settings.richlist_page); app.set('markets_page', settings.markets_page); app.set('api_page', settings.api_page); app.set('claim_address_page', settings.claim_address_page); +app.set('orphans_page', settings.orphans_page); app.set('labels', settings.labels); app.set('api_cmds', settings.api_cmds); app.set('blockchain_specific', settings.blockchain_specific); diff --git a/lib/database.js b/lib/database.js index 04732f7..e7e7a4c 100644 --- a/lib/database.js +++ b/lib/database.js @@ -5,6 +5,7 @@ var mongoose = require('mongoose'), Address = require('../models/address'), AddressTx = require('../models/addresstx'), Tx = require('../models/tx'), + Orphans = require('../models/orphans'), Richlist = require('../models/richlist'), Peers = require('../models/peers'), Heavy = require('../models/heavy'), @@ -403,7 +404,13 @@ module.exports = { // collection has data // determine if last_usd_price field exists check_add_db_field(Stats, 'last_usd_price', 0, function(exists) { - return cb(true); + // determine if orphan_index field exists + check_add_db_field(Stats, 'orphan_index', 0, function(exists) { + // determine if orphan_current field exists + check_add_db_field(Stats, 'orphan_current', 0, function(exists) { + return cb(true); + }); + }); }); } else return cb(false); @@ -424,7 +431,9 @@ module.exports = { if (!skip) { var newStats = new Stats({ coin: coin, - last: 0 + last: 0, + orphan_index: 0, + orphan_current: 0 }); newStats.save(function(err) { @@ -1068,7 +1077,9 @@ module.exports = { supply: (supply ? supply : 0), connections: (connections ? connections : 0), last: (stats.last ? stats.last : 0), - txes: (stats.txes ? stats.txes : 0) + txes: (stats.txes ? stats.txes : 0), + orphan_index: (stats.orphan_index ? stats.orphan_index : 0), + orphan_current: (stats.orphan_current ? stats.orphan_current : 0) }); }); } else { @@ -1511,7 +1522,7 @@ module.exports = { if (tx && tx != 'There was an error. Check your console.') { lib.prepare_vin(tx, function(vin, tx_type_vin) { lib.prepare_vout(tx.vout, txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) { - lib.syncLoop(vin.length, function (loop) { + lib.syncLoop(nvin.length, function (loop) { var i = loop.iteration(); // check if address is inside an array @@ -1556,7 +1567,7 @@ module.exports = { var newTx = new Tx({ txid: tx.txid, - vin: nvin, + vin: (vin == null || vin.length == 0 ? [] : nvin), vout: vout, total: total.toFixed(8), timestamp: tx.time, @@ -1582,5 +1593,19 @@ module.exports = { }); }, + // get the list of orphans from local collection + get_orphans: function(start, length, cb) { + // get the count of orphaned blocks + Orphans.find({}).countDocuments(function(err, count) { + // get the actual orphaned block data + Orphans.find({}).sort({blockindex: -1}).skip(Number(start)).limit(Number(length)).exec(function(err, orphans) { + if (err) + return cb([], count); + else + return cb(orphans, count); + }); + }); + }, + fs: fs }; \ No newline at end of file diff --git a/lib/explorer.js b/lib/explorer.js index cbce5b2..8bb1170 100644 --- a/lib/explorer.js +++ b/lib/explorer.js @@ -1202,7 +1202,27 @@ module.exports = { arr_vout[0].amount = arr_vout[0].amount - arr_vin[0].amount; arr_vin.shift(); - return cb(arr_vout, arr_vin, tx_type); + // check if any vin remains + if (arr_vin == null || arr_vin.length == 0) { + // empty vin should be linked to coinbase + arr_vin = [{coinbase: "coinbase"}]; + + var new_vout = []; + + // loop through the arr_vout to create a copy of the data with coin amounts only for use with prepare_vin() + for (i = 0; i < arr_vout.length; i++) { + new_vout.push({ + value: arr_vout[i].amount / 100000000 + }); + } + + // call the prepare_vin again to populate the vin data correctly + module.exports.prepare_vin({txid: txid, vin: arr_vin, vout: new_vout}, function(return_vin, return_tx_type_vin) { + return cb(arr_vout, return_vin, return_tx_type_vin); + }); + } else { + return cb(arr_vout, arr_vin, tx_type); + } } else return cb(arr_vout, arr_vin, tx_type); } else diff --git a/lib/locale.js b/lib/locale.js index 7c5a088..92da9d7 100644 --- a/lib/locale.js +++ b/lib/locale.js @@ -16,6 +16,7 @@ exports.menu_movement = "Movement", exports.menu_node = "Nodes", exports.menu_network = "Network", exports.menu_claim_address = "Claim Address", +exports.menu_orphans = "Orphaned Blocks", exports.ex_title = "{1} Block Explorer", exports.ex_description = "A listing of all verified {1} transactions", @@ -166,6 +167,16 @@ exports.mkt_select = "Market Select", exports.claim_title = "{1} Wallet Address Claim", exports.claim_description = "Verify ownership of your {1} wallet address and set a custom display name in the explorer", +// Orphans view +exports.orphan_title = "{1} Orphaned Blocks", +exports.orphan_description = "A listing of valid blocks that have been orphaned and do not belong to the main blockchain", +exports.orphan_block_list = "Orphaned Block List", +exports.orphan_block_hash = "Orphaned Block Hash", +exports.orphan_actual_block = "Actual Block", +exports.orphan_prev_block = "Previous Block", +exports.orphan_next_block = "Next Block", +exports.view_orphan = "View Orphaned Block", + // Heavycoin exports.heavy_vote = "Vote", // Heavycoin rewards view diff --git a/lib/settings.js b/lib/settings.js index 87a66d3..132563b 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -1184,6 +1184,38 @@ exports.claim_address_page = { "enable_bad_word_filter": true }; +// orphans_page: a collection of settings that pertain to the orphans page +exports.orphans_page = { + // enabled: Enable/disable the orphans page (true/false) + // If set to false, the orphans page will be completely inaccessible + "enabled": false, + // show_panels: Determine whether to show the panels configured in the shared_pages.page_header section across the top of this page (true/false) + "show_panels": false, + // show_nethash_chart: Determine whether to show the network hashrate chart configured in the shared_pages.network_charts.nethash_chart section across the top of this page (true/false) + "show_nethash_chart": false, + // show_difficulty_chart: Determine whether to show the network difficulty chart configured in the shared_pages.network_charts.difficulty_chart section across the top of this page (true/false) + "show_difficulty_chart": false, + // page_header: a collection of settings that pertain to the orphans page header + "page_header": { + // show_img: Determine whether to show the page title image defined in the "shared_pages.page_header.page_title_image" setting (true/false) + "show_img": true, + // show_title: Determine whether to show the page title as defined in "locale.orphan_title" (true/false) + "show_title": true, + // show_last_updated: Determine whether to show a label below the page title with the last updated date (true/false) + "show_last_updated": true, + // show_description: Determine whether to show the page description as defined in "locale.orphan_description" (true/false) + "show_description": true + }, + // orphans_table: a collection of settings that pertain to the orphans table on the orphans page + // Table data is populated via the /ext/getorphanlist api + "orphans_table": { + // page_length_options: An array of page length options that determine how many items/records to display in the table at any given time + "page_length_options": [ 10, 25, 50, 75, 100, 250, 500, 1000 ], + // items_per_page: The default amount of items/records to display in the table at any given time + "items_per_page": 10 + } +}; + // sync: a collection of settings that pertain to the data synchronization process exports.sync = { // block_parallel_tasks: Use multiple threads to do blockchain syncing which greatly improves the initial sync speed, but there is a drawback. diff --git a/locale/en.json b/locale/en.json index 97a7a82..aafdfb5 100644 --- a/locale/en.json +++ b/locale/en.json @@ -10,6 +10,7 @@ "menu_node": "Nodes", "menu_network": "Network", "menu_claim_address": "Claim Address", + "menu_orphans": "Orphaned Blocks", // explorer view "ex_title": "{1} Block Explorer", @@ -169,6 +170,16 @@ "claim_title": "{1} Wallet Address Claim", "claim_description": "Verify ownership of your {1} wallet address and set a custom display name in the explorer", + // orphans view + "orphan_title": "{1} Orphaned Blocks", + "orphan_description": "A listing of valid blocks that have been orphaned and do not belong to the main blockchain", + "orphan_block_list": "Orphaned Block List", + "orphan_block_hash": "Orphaned Block Hash", + "orphan_actual_block": "Actual Block", + "orphan_prev_block": "Previous Block", + "orphan_next_block": "Next Block", + "view_orphan": "View Orphaned Block", + // heavycoin rewards view "heavy_title": "{1} Reward/Voting Details", "heavy_description": "Viewing {1} voting data and coin reward change details", diff --git a/models/orphans.js b/models/orphans.js new file mode 100644 index 0000000..08838b1 --- /dev/null +++ b/models/orphans.js @@ -0,0 +1,12 @@ +var mongoose = require('mongoose'), + Schema = mongoose.Schema; + +var OrphanSchema = new Schema({ + blockindex: {type: Number, default: 0, index: true}, + orphan_blockhash: {type: String, unique: true, index: true}, + good_blockhash: {type: String, index: true}, + prev_blockhash: {type: String, index: true}, + next_blockhash: {type: String, index: true} +}, {id: false}); + +module.exports = mongoose.model('Orphan', OrphanSchema); \ No newline at end of file diff --git a/models/stats.js b/models/stats.js index 60b9b0c..0b5a700 100644 --- a/models/stats.js +++ b/models/stats.js @@ -15,7 +15,9 @@ var StatsSchema = new Schema({ masternodes_last_updated: { type: Number, default: 0 }, network_last_updated: { type: Number, default: 0 }, richlist_last_updated: { type: Number, default: 0 }, - markets_last_updated: { type: Number, default: 0 } + markets_last_updated: { type: Number, default: 0 }, + orphan_index: { type: Number, default: 0 }, + orphan_current: { type: Number, default: 0 } }); module.exports = mongoose.model('coinstats', StatsSchema); \ No newline at end of file diff --git a/routes/index.js b/routes/index.js index c7005ae..237b078 100644 --- a/routes/index.js +++ b/routes/index.js @@ -24,7 +24,10 @@ function route_get_block(res, blockhash) { page_title_prefix: settings.coin.name + ' Genesis Block' } ); - else { + else if (block.confirmations == -1) { + // this is an orphaned block, so get the data from the wallet directly + get_block_data_from_wallet(block, res, true); + } else { db.get_txs(block, function(txs) { if (txs.length > 0) res.render( @@ -43,45 +46,7 @@ function route_get_block(res, blockhash) { ); else { // cannot find block in local database so get the data from the wallet directly - var ntxs = []; - - lib.syncLoop(block.tx.length, function (loop) { - var i = loop.iteration(); - - lib.get_rawtransaction(block.tx[i], function(tx) { - if (tx && tx != 'There was an error. Check your console.') { - lib.prepare_vin(tx, function(vin, tx_type_vin) { - lib.prepare_vout(tx.vout, block.tx[i], vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) { - lib.calculate_total(vout, function(total) { - ntxs.push({ - txid: block.tx[i], - vout: vout, - total: total.toFixed(8) - }); - - loop.next(); - }); - }); - }); - } else - loop.next(); - }); - }, function() { - res.render( - 'block', - { - active: 'block', - block: block, - confirmations: settings.shared_pages.confirmations, - txs: ntxs, - showSync: db.check_show_sync_message(), - customHash: get_file_timestamp('./public/css/custom.scss'), - styleHash: get_file_timestamp('./public/css/style.scss'), - themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), - page_title_prefix: settings.coin.name + ' Block ' + block.height - } - ); - }); + get_block_data_from_wallet(block, res, false); } }); } @@ -157,48 +122,57 @@ function route_get_tx(res, txid) { lib.prepare_vout(rtx.vout, rtx.txid, vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof rtx.vjoinsplit === 'undefined' || rtx.vjoinsplit == null) ? [] : rtx.vjoinsplit), function(rvout, rvin, tx_type_vout) { lib.calculate_total(rvout, function(total) { if (!rtx.confirmations > 0) { - var utx = { - txid: rtx.txid, - vin: rvin, - vout: rvout, - total: total.toFixed(8), - timestamp: rtx.time, - blockhash: '-', - blockindex: -1 - }; + lib.get_block(rtx.blockhash, function(block) { + if (block && block != 'There was an error. Check your console.') { + var utx = { + txid: rtx.txid, + vin: rvin, + vout: rvout, + total: total.toFixed(8), + timestamp: (rtx.time == null ? block.time : rtx.time), + blockhash: (rtx.blockhash == null ? '-' : rtx.blockhash), + blockindex: block.height + }; - if (settings.claim_address_page.enabled == true) { - db.populate_claim_address_names(utx, function(utx) { - res.render( - 'tx', - { - active: 'tx', - tx: utx, - confirmations: settings.shared_pages.confirmations, - blockcount: -1, - showSync: db.check_show_sync_message(), - customHash: get_file_timestamp('./public/css/custom.scss'), - styleHash: get_file_timestamp('./public/css/style.scss'), - themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), - page_title_prefix: settings.coin.name + ' Transaction ' + utx.txid - } - ); - }); - } else - res.render( - 'tx', - { - active: 'tx', - tx: utx, - confirmations: settings.shared_pages.confirmations, - blockcount: -1, - showSync: db.check_show_sync_message(), - customHash: get_file_timestamp('./public/css/custom.scss'), - styleHash: get_file_timestamp('./public/css/style.scss'), - themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), - page_title_prefix: settings.coin.name + ' Transaction ' + utx.txid - } - ); + if (settings.claim_address_page.enabled == true) { + db.populate_claim_address_names(utx, function(utx) { + res.render( + 'tx', + { + orphan: true, + active: 'tx', + tx: utx, + confirmations: settings.shared_pages.confirmations, + blockcount: (block.height - 1), + showSync: db.check_show_sync_message(), + customHash: get_file_timestamp('./public/css/custom.scss'), + styleHash: get_file_timestamp('./public/css/style.scss'), + themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), + page_title_prefix: settings.coin.name + ' Transaction ' + utx.txid + } + ); + }); + } else + res.render( + 'tx', + { + orphan: true, + active: 'tx', + tx: utx, + confirmations: settings.shared_pages.confirmations, + blockcount: (block.height - 1), + showSync: db.check_show_sync_message(), + customHash: get_file_timestamp('./public/css/custom.scss'), + styleHash: get_file_timestamp('./public/css/style.scss'), + themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), + page_title_prefix: settings.coin.name + ' Transaction ' + utx.txid + } + ); + } else { + // cannot load tx + route_get_index(res, null); + } + }); } else { // check if blockheight exists if (!rtx.blockheight && rtx.blockhash) { @@ -418,6 +392,62 @@ function route_get_claim_form(res, hash) { route_get_address(res, hash); } +function get_last_updated_date(show_last_updated, last_updated_field, cb) { + // check if the last updated date is needed + if (show_last_updated == true) { + // lookup the stats record + db.get_stats(settings.coin.name, function (stats) { + // return the last updated date + return cb(stats[last_updated_field]); + }); + } else { + return cb(null); + } +} + +function get_block_data_from_wallet(block, res, orphan) { + var ntxs = []; + + lib.syncLoop(block.tx.length, function (loop) { + var i = loop.iteration(); + + lib.get_rawtransaction(block.tx[i], function(tx) { + if (tx && tx != 'There was an error. Check your console.') { + lib.prepare_vin(tx, function(vin, tx_type_vin) { + lib.prepare_vout(tx.vout, block.tx[i], vin, ((!settings.blockchain_specific.zksnarks.enabled || typeof tx.vjoinsplit === 'undefined' || tx.vjoinsplit == null) ? [] : tx.vjoinsplit), function(vout, nvin, tx_type_vout) { + lib.calculate_total(vout, function(total) { + ntxs.push({ + txid: block.tx[i], + vout: vout, + total: total.toFixed(8) + }); + + loop.next(); + }); + }); + }); + } else + loop.next(); + }); + }, function() { + res.render( + 'block', + { + orphan: orphan, + active: 'block', + block: block, + confirmations: settings.shared_pages.confirmations, + txs: ntxs, + showSync: db.check_show_sync_message(), + customHash: get_file_timestamp('./public/css/custom.scss'), + styleHash: get_file_timestamp('./public/css/style.scss'), + themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), + page_title_prefix: settings.coin.name + ' Block ' + block.height + } + ); + }); +} + /* GET home page. */ router.get('/', function(req, res) { @@ -770,6 +800,30 @@ router.get('/address/:hash', function(req, res) { route_get_address(res, req.params.hash); }); +router.get('/orphans', function(req, res) { + // ensure orphans page is enabled + if (settings.orphans_page.enabled == true) { + // lookup the last updated date if necessary + get_last_updated_date(settings.orphans_page.page_header.show_last_updated, 'blockchain_last_updated', function(last_updated_date) { + res.render( + 'orphans', + { + active: 'orphans', + last_updated: last_updated_date, + showSync: db.check_show_sync_message(), + customHash: get_file_timestamp('./public/css/custom.scss'), + styleHash: get_file_timestamp('./public/css/style.scss'), + themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), + page_title_prefix: locale.orphan_title.replace('{1}', settings.coin.name) + } + ); + }); + } else { + // orphans page is not enabled so default to the index page + route_get_index(res, null); + } +}); + router.post('/search', function(req, res) { if (settings.shared_pages.page_header.search.enabled == true) { var query = req.body.search.trim(); diff --git a/scripts/sync.js b/scripts/sync.js index ed3552b..f01631a 100644 --- a/scripts/sync.js +++ b/scripts/sync.js @@ -4,6 +4,7 @@ var mongoose = require('mongoose'), Tx = require('../models/tx'), Address = require('../models/address'), AddressTx = require('../models/addresstx'), + Orphans = require('../models/orphans'), Richlist = require('../models/richlist'), Stats = require('../models/stats'), settings = require('../lib/settings'), @@ -66,7 +67,7 @@ function exit(exitCode) { } } -// updates tx, address & richlist db's +// updates tx & address balances function update_tx_db(coin, start, end, txes, timeout, check_only, cb) { var complete = false; var blocks_to_scan = []; @@ -84,12 +85,12 @@ function update_tx_db(coin, start, end, txes, timeout, check_only, cb) { blocks_to_scan.push(i); async.eachLimit(blocks_to_scan, task_limit_blocks, function(block_height, next_block) { - if (!check_only && block_height % settings.sync.save_stats_after_sync_blocks === 0) { + if (check_only == 0 && block_height % settings.sync.save_stats_after_sync_blocks === 0) { Stats.updateOne({coin: coin}, { last: block_height - 1, txes: txes }, function() {}); - } else if (check_only) { + } else if (check_only == 1) { console.log('Checking block ' + block_height + '...'); } @@ -99,47 +100,69 @@ function update_tx_db(coin, start, end, txes, timeout, check_only, cb) { if (block) { async.eachLimit(block.tx, task_limit_txs, function(txid, next_tx) { Tx.findOne({txid: txid}, function(err, tx) { - if (tx) { - setTimeout( function() { + if (tx && check_only != 2) { + setTimeout(function() { tx = null; // check if the script is stopping - if (stopSync) { + if (stopSync && check_only != 2) { // stop the loop next_tx({}); } else next_tx(); }, timeout); } else { - db.save_tx(txid, block_height, function(err, tx_has_vout) { - if (err) - console.log(err); - else - console.log('%s: %s', block_height, txid); + // check if the transaction exists but doesn't match the current block height + check_delete_tx(tx, block_height, txes, timeout, function(updated_txes, tx_deleted) { + // update the running tx count + txes = updated_txes; - if (tx_has_vout) - txes++; + // check if this tx should be added to the local database + if (tx_deleted || !tx) { + // save the transaction to local database + db.save_tx(txid, block_height, function(err, tx_has_vout) { + if (err) + console.log(err); + else + console.log('%s: %s', block_height, txid); - setTimeout( function() { - tx = null; + if (tx_has_vout) + txes++; - // check if the script is stopping - if (stopSync) { - // stop the loop - next_tx({}); - } else - next_tx(); - }, timeout); + setTimeout(function() { + tx = null; + + // check if the script is stopping + if (stopSync && check_only != 2) { + // stop the loop + next_tx({}); + } else + next_tx(); + }, timeout); + }); + } else { + // skip adding the current tx + setTimeout(function() { + tx = null; + + // check if the script is stopping + if (stopSync && check_only != 2) { + // stop the loop + next_tx({}); + } else + next_tx(); + }, timeout); + } }); } }); }, function() { - setTimeout( function() { + setTimeout(function() { blockhash = null; block = null; // check if the script is stopping - if (stopSync) { + if (stopSync && check_only != 2) { // stop the loop next_block({}); } else @@ -149,9 +172,9 @@ function update_tx_db(coin, start, end, txes, timeout, check_only, cb) { } else { console.log('Block not found: %s', blockhash); - setTimeout( function() { + setTimeout(function() { // check if the script is stopping - if (stopSync) { + if (stopSync && check_only != 2) { // stop the loop next_block({}); } else @@ -160,9 +183,9 @@ function update_tx_db(coin, start, end, txes, timeout, check_only, cb) { } }); } else { - setTimeout( function() { + setTimeout(function() { // check if the script is stopping - if (stopSync) { + if (stopSync && check_only != 2) { // stop the loop next_block({}); } else @@ -171,19 +194,729 @@ function update_tx_db(coin, start, end, txes, timeout, check_only, cb) { } }); }, function() { - // check if the script stopped prematurely - if (!stopSync) { - Stats.updateOne({coin: coin}, { - last: end, - txes: txes - }, function() { - return cb(); - }); - } else - return cb(); + var statUpdateObject = {}; + + // check what stats data should be updated + if (stopSync || check_only == 2) { + // only update txes when fixing invalid and missing blocks or when a "normal" sync was stopped prematurely + statUpdateObject.txes = txes; + } else { + // update last and txes values for "normal" sync that finishes without being stopped prematurely + statUpdateObject = { + txes: txes, + last: end + }; + } + + // update local stats + Stats.updateOne({coin: coin}, statUpdateObject, function() { + return cb(txes); + }); }); } +// fixes data belonging to orphaned blocks +function update_orphans(orphan_index, orphan_current, last_blockindex, timeout, cb) { + // lookup the earliest orphaned block if this is the first time that orphans are being checked + get_earliest_orphan_block(orphan_index, orphan_current, last_blockindex, function(orphan_data, err) { + if (err != null) { + console.log(err); + return cb(); + } else { + // re-populate the orphan data in case it has changed + orphan_index = orphan_data.orphan_index; + orphan_current = orphan_data.orphan_current; + + // Check if the sync msg should be shown + check_show_sync_message(last_blockindex - orphan_index); + + // start from the current orphan (if exists) or else use the last orphan index + let current_block = (orphan_current == 0 ? orphan_index : orphan_current); + let unresolved_forks = []; + let correct_block_data = null; + + // loop infinitely until finished iterating through all known blocks + async.forever(function(next) { + // check if the script is stopping or if the last block has been reached + if (stopSync || current_block >= last_blockindex) { + // stop the main loop + next('stop'); + } else { + if (unresolved_forks.length == 0 && current_block % settings.sync.save_stats_after_sync_blocks === 0) { + Stats.updateOne({coin: settings.coin.name}, { + orphan_index: current_block - 1, + orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0]) + }, function() {}); + } + + // do not show the 'checking...' msg if this block is about to be resolved + if (correct_block_data == null) + console.log(current_block.toString() + ': Checking for forks...'); + + // check if there is a fork in this block + check_block_height_for_fork(current_block, function(blockhashes, fork_err) { + if (fork_err != null) { + // an error occurred + // stop looking for orphans + next(fork_err); + } else if (blockhashes == null) { + // no forks found in this block + // check if there are previously unresolved forks + if (unresolved_forks.length == 0) { + // move to the next block + current_block++; + + setTimeout(function() { + // process next block + next(null); + }, timeout); + } else { + // one or more forks still need to be resolved + // set the current block to the block after the next orphaned block + current_block = unresolved_forks[unresolved_forks.length - 1] + 1; + + // lookup the block hash using the block height + lib.get_blockhash(current_block, function(block_hash) { + // check if the block hash was found + if (block_hash) { + // lookup the current block data from the wallet + lib.get_block(block_hash, function (block_data) { + // remember the correct block hashes + correct_block_data = { + prev_hash: block_data.previousblockhash, + next_hash: block_data.hash + }; + + console.log('Good ' + current_block.toString() + ' block found. Returning to fix block ' + unresolved_forks[unresolved_forks.length -1].toString()); + + // go back to the last unresolved fork + current_block = unresolved_forks.pop(); + + setTimeout(function() { + // process next block + next(null); + }, timeout); + }); + } else { + // an error occurred + // stop looking for orphans + next('cannot find block ' + current_block.toString()); + } + }); + } + } else { + // there is at least one fork in this block height + // check if the correct block hash is known + if (correct_block_data != null) { + // this fork needs to be resolved + // lookup the current block data from the wallet + lib.get_block(correct_block_data.prev_hash, function (block_data) { + var tx_count = 0; + + // check if the good block hash is in the list of blockhashes + if (blockhashes.indexOf(correct_block_data.prev_hash) > -1) { + // remove the good block hash + blockhashes.splice(blockhashes.indexOf(correct_block_data.prev_hash), 1); + } + + // loop through the remaining orphaned block hashes + lib.syncLoop(blockhashes.length, function(block_loop) { + var i = block_loop.iteration(); + + console.log('Resolving orphaned block [' + (i + 1).toString() + '/' + blockhashes.length.toString() + ']: ' + blockhashes[i]); + + // find all orphaned txid's from the current orphan block hash + get_orphaned_txids(blockhashes[i], function(txids) { + // save the orphan block data to the orphan collection + create_orphan(current_block, blockhashes[i], correct_block_data.prev_hash, block_data.previousblockhash, correct_block_data.next_hash, function() { + // loop through the remaining orphaned block hashes + lib.syncLoop(txids.length, function(tx_loop) { + var t = tx_loop.iteration(); + + // remove the orphaned tx and cleanup all associated data + delete_and_cleanup_tx(txids[t], current_block, tx_count, timeout, function(updated_tx_count) { + // update the running tx count + tx_count = updated_tx_count; + + // some blockchains will reuse the same orphaned transaction ids + // lookup the transaction that was just deleted to ensure it doesn't belong to another block + check_add_tx(txids[t], blockhashes[i], tx_count, function(updated_tx_count2) { + // update the running tx count + tx_count = updated_tx_count2; + + setTimeout(function() { + // move to the next tx record + tx_loop.next(); + }, timeout); + }); + }); + }, function() { + setTimeout(function() { + // move to the next block record + block_loop.next(); + }, timeout); + }); + }); + }); + }, function() { + // get the most recent stats + Stats.findOne({coin: settings.coin.name}, function(stat_err, stats) { + // add missing txes for the current block + update_tx_db(settings.coin.name, current_block, current_block, (stats.txes + tx_count), timeout, 2, function(updated_tx_count) { + // update the stats collection by removing the orphaned txes in this block from the tx count + // and setting the orphan_index and orphan_current values in case the sync is interrupted before finishing + Stats.updateOne({coin: settings.coin.name}, { + orphan_index: current_block, + orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0]) + }, function (stats_err) { + if (stats_err != null) + console.log(stats_err); + + // clear the saved block hash data + correct_block_data = null; + + // move to the next block + current_block++; + + setTimeout(function() { + // process next block + next(null); + }, timeout); + }); + }); + }); + }); + }); + } else { + // correct block hash is unknown + console.log('Block ' + current_block.toString() + ' has ' + (blockhashes.length - 1).toString() + ' unresolved fork' + (blockhashes.length - 1 == 1 ? '' : 's')); + + // add this block height to the list of unresolved forks + unresolved_forks.push(current_block); + + // move to the next block + current_block++; + + setTimeout(function() { + // process next block + next(null); + }, timeout); + } + } + }); + } + }, + function(err) { + // check if there is a msg to display + if (err != '' && err != 'stop') { + // display the msg + console.log(err); + + // stop fixing orphaned block data + return cb(); + } else { + // check if the script is stopping + if (!stopSync) + console.log('Finished looking for forks'); + + // get the list of orphans with a null next_blockhash + Orphans.find({next_blockhash: null}).exec(function(next_err, orphans) { + // loop through the list of orphans + lib.syncLoop(orphans.length, function(orphan_loop) { + var o = orphan_loop.iteration(); + + // lookup the block data from the wallet + lib.get_block(orphans[o].good_blockhash, function (good_block_data) { + // check if the next block hash is known + if (good_block_data.nextblockhash != null) { + // update the next blockhash for this orphan record + Orphans.updateOne({blockindex: orphans[o].blockindex, orphan_blockhash: orphans[o].orphan_blockhash}, { + next_blockhash: good_block_data.nextblockhash + }, function() { + setTimeout(function() { + // move to the next orphan record + orphan_loop.next(); + }, timeout); + }); + } else { + setTimeout(function() { + // move to the next orphan record + orphan_loop.next(); + }, timeout); + } + }); + }, function() { + // update the orphan stats + Stats.updateOne({coin: settings.coin.name}, { + orphan_index: current_block - 1, + orphan_current: (unresolved_forks.length == 0 ? 0 : unresolved_forks[0]) + }, function() { + // finished fixing orphaned block data + return cb(); + }); + }); + }); + } + }); + } + }); +} + +function get_earliest_orphan_block(orphan_index, orphan_current, last_blockindex, cb) { + // check if it is necessary to search for orphan data + if (orphan_index == null || orphan_index == 0) { + console.log('Finding the earliest orphaned blockindex.. Please wait..'); + + Tx.aggregate([ + { $match: { + "vout": { $exists: true, $ne: [] } + }}, + { $group: { + _id: "$blockindex", + blockhashes: { $addToSet: "$blockhash" } + }}, + { $match: { + "blockhashes.1": { "$exists": true } + }}, + { $sort: { + "_id": 1 + }}, + { $limit: 1 }, + { $project: { + "_id": 1 + }} + ], function(err, data) { + if (err) { + return cb(null, err); + } else if (data.length > 0) { + // found the first unprocessed orphaned block + orphan_current = data[0]._id; + orphan_index = (orphan_current - 1); + console.log('Found orphan block index: ' + orphan_current.toString()); + + Stats.updateOne({coin: settings.coin.name}, { + orphan_current: orphan_current, + orphan_index: orphan_index + }, function() { + return cb({orphan_index: orphan_index, orphan_current: orphan_current}, null); + }); + } else { + // no unprocessed orphaned blocks found + orphan_current = 0; + orphan_index = last_blockindex; + console.log('No orphaned blocks found'); + + Stats.updateOne({coin: settings.coin.name}, { + orphan_current: orphan_current, + orphan_index: orphan_index + }, function() { + return cb({orphan_index: orphan_index, orphan_current: orphan_current}, null); + }); + } + }).option({ allowDiskUse: true }); + } else + return cb({orphan_index: orphan_index, orphan_current: orphan_current}, null); +} + +function check_block_height_for_fork(block_height, cb) { + // find all unique blockhashes in the txes collections for this block height + Tx.aggregate([ + { $match: { + $and: [ + {"blockindex": { $eq: block_height }}, + {"vout": { $exists: true, $ne: [] }} + ] + }}, + { $group: { + _id: "$blockindex", + blockhashes: { $addToSet: "$blockhash" } + }} + ], function(err, data) { + if (err) { + // an error was returned + return cb(null, err); + } else if (data.length > 0) { + // lookup the "good" block hash using the block height + lib.get_blockhash(block_height, function(block_hash) { + // check if there is more than 1 block hash + if (data[0].blockhashes.length == 1) { + // 1 block found + // check if the found block is the good block + if (data[0].blockhashes[0] == block_hash) { + // no forks found + return cb(null, null); + } else { + // add the good block to the list of blockhashes + data[0].blockhashes.push(block_hash); + + // return the block hashes + return cb(data[0].blockhashes, null); + } + } else { + // more than 1 block found + // check if the good block is already in the list + if (data[0].blockhashes.indexOf(block_hash) == -1) { + // add the good block to the list of blockhashes + data[0].blockhashes.push(block_hash); + } + + // return the block hashes + return cb(data[0].blockhashes, null); + } + }); + } else { + // no blocks found + return cb(null, null); + } + }); +} + +function create_orphan(blockindex, orphan_blockhash, good_blockhash, prev_blockhash, next_blockhash, cb) { + var newOrphan = new Orphans({ + blockindex: blockindex, + orphan_blockhash: orphan_blockhash, + good_blockhash: good_blockhash, + prev_blockhash: prev_blockhash, + next_blockhash: next_blockhash + }); + + // create a new orphan record in the local database + newOrphan.save(function(err) { + if (err) { + // check if this is a duplicate key error which can be ignored + if (!(err.toString().indexOf('E11000') > -1 || err.toString().indexOf('duplicate key error') > -1)) + console.log(err); + + return cb(); + } else { + // new orphan record saved successfully + return cb(); + } + }); +} + +function get_orphaned_txids(block_hash, cb) { + // get all transactions by block hash + Tx.find({blockhash: block_hash}).exec(function(err, txes) { + if (err) { + // an error was returned + return cb(null, err); + } else if (txes.length > 0) { + // found at least one orphaned transaction + var txids = []; + + // populate an array of txids without the object data + for (t = 0; t < txes.length; t++) + txids.push(txes[t].txid); + + return cb(txids, null); + } else { + // no txes found + return cb(null, null); + } + }); +} + +function check_add_tx(txid, blockhash, tx_count, cb) { + // lookup the transaction to ensure it doesn't belong to another block + lib.get_rawtransaction(txid, function(tx) { + // check if this txid belongs to the main blockchain + if (tx && tx.txid && tx.blockhash != blockhash && tx.confirmations > 0) { + // lookup the correct block index in case it is not the same as the current block + lib.get_block(tx.blockhash, function(block) { + // check if the block was found + if (block) { + // save the tx to the local database + db.save_tx(txid, block.height, function(save_tx_err, tx_has_vout) { + // check if there were any save errors + if (save_tx_err) + console.log(save_tx_err); + else + console.log('%s: %s', block.height, txid); + + // check if the tx was saved correctly + if (tx_has_vout) { + // keep a running total of txes that were added + tx_count++; + } + + return cb(tx_count); + }); + } else { + // block not found so there is nothing to fix + return cb(tx_count); + } + }); + } else { + // block does not belong to main blockchain so there is nothing to fix + return cb(tx_count); + } + }); +} + +function delete_and_cleanup_tx(txid, block_height, tx_count, timeout, cb) { + // lookup all address tx records associated with the current tx + AddressTx.find({txid: txid}).exec(function (find_address_err, address_txes) { + if (find_address_err != null) { + console.log(find_address_err); + return cb(tx_count); + } else if (address_txes.length == 0) { + // no vouts for this tx, so just delete the tx without cleaning up addresses + delete_tx(txid, block_height, function(tx_err, tx_result) { + if (tx_err) { + console.log(tx_err); + return cb(tx_count); + } else { + // NOTE: do not subtract from the tx_count here because only txes with vouts are counted + return cb(tx_count); + } + }); + } else { + // lookup the current tx in the local database + Tx.findOne({txid: txid}, function(find_tx_err, tx) { + if (find_tx_err != null) { + console.log(find_tx_err); + return cb(tx_count); + } else { + var addressTxArray = []; + var has_vouts = (tx.vout != null && tx.vout.length > 0); + + // check if this is a coinbase tx + if (tx.vin == null || tx.vin.length == 0) { + // add a coinbase tx into the addressTxArray array + addressTxArray.push({ + txid: txid, + a_id: 'coinbase', + amount: tx.total + }); + } + + // check if there are any vin addresses + if (tx.vin != null && tx.vin.length > 0) { + // loop through the vin data + for (var vin_tx_counter = tx.vin.length - 1; vin_tx_counter >= 0; vin_tx_counter--) { + // loop through the addresstxe data + for (var vin_addresstx_counter = address_txes.length - 1; vin_addresstx_counter >= 0; vin_addresstx_counter--) { + // check if there is a tx record that exactly matches to the addresstx + if (tx.vin[vin_tx_counter].addresses == address_txes[vin_addresstx_counter].a_id && tx.vin[vin_tx_counter].amount == -address_txes[vin_addresstx_counter].amount) { + // add the address into the addressTxArray array + addressTxArray.push({ + txid: txid, + a_id: tx.vin[vin_tx_counter].addresses, + amount: address_txes[vin_addresstx_counter].amount + }); + + // remove the found records from both arrays + tx.vin.splice(vin_tx_counter, 1); + address_txes.splice(vin_addresstx_counter, 1); + + break; + } + } + } + } + + // check if there are any vout addresses + if (tx.vout != null && tx.vout.length > 0) { + // loop through the vout data + for (var vout_tx_counter = tx.vout.length - 1; vout_tx_counter >= 0; vout_tx_counter--) { + // loop through the addresstxe data + for (var vout_addresstx_counter = address_txes.length - 1; vout_addresstx_counter >= 0; vout_addresstx_counter--) { + // check if there is a tx record that exactly matches to the addresstx + if (tx.vout[vout_tx_counter].addresses == address_txes[vout_addresstx_counter].a_id && tx.vout[vout_tx_counter].amount == address_txes[vout_addresstx_counter].amount) { + // add the address into the addressTxArray array + addressTxArray.push({ + txid: txid, + a_id: tx.vout[vout_tx_counter].addresses, + amount: address_txes[vout_addresstx_counter].amount + }); + + // remove the found records from both arrays + tx.vout.splice(vout_tx_counter, 1); + address_txes.splice(vout_addresstx_counter, 1); + + break; + } + } + } + } + + // check if there are still more vin/vout records to process + if (tx.vin.length > 0 || tx.vout.length > 0 || address_txes.length > 0) { + // get all unique remaining addresses + var address_list = []; + + // get unique addresses from the tx vin + tx.vin.forEach(function(vin) { + if (address_list.indexOf(vin.addresses) == -1) + address_list.push(vin.addresses); + }); + + // get unique addresses from the tx vout + tx.vout.forEach(function(vout) { + if (address_list.indexOf(vout.addresses) == -1) + address_list.push(vout.addresses); + }); + + // get unique addresses from the addresstxes + address_txes.forEach(function(address_tx) { + if (address_list.indexOf(address_tx.a_id) == -1) + address_list.push(address_tx.a_id); + }); + + // loop through each unique address + address_list.forEach(function(address) { + var vin_total = 0; + var vout_total = 0; + var address_tx_total = 0; + + // add up all the vin amounts for this address + tx.vin.forEach(function(vin) { + // check if this is the correct address + if (vin.addresses == address) + vin_total += vin.amount; + }); + + // add up all the vout amounts for this address + tx.vout.forEach(function(vout) { + // check if this is the correct address + if (vout.addresses == address) + vout_total += vout.amount; + }); + + // add up all the addresstx amounts for this address + address_txes.forEach(function(address_tx) { + // check if this is the correct address + if (address_tx.a_id == address) + address_tx_total += address_tx.amount; + }); + + // check if the tx and addresstx totals match + if ((vout_total - vin_total) == address_tx_total) { + // the values match (this indicates that this address sent coins to themselves) + // add a vin record for this address into the addressTxArray array + addressTxArray.push({ + txid: txid, + a_id: address, + amount: -vin_total + }); + + // add a vout record for this address into the addressTxArray array + addressTxArray.push({ + txid: txid, + a_id: address, + amount: vout_total + }); + } else { + // the values do not match (this indicates there was a problem saving the data) + // output the data for this address as-is, using the addresstx values + address_txes.forEach(function(address_tx) { + // check if this is the correct address + if (address_tx.a_id == address) { + // add a record for this address into the addressTxArray array + addressTxArray.push({ + txid: txid, + a_id: address, + amount: address_tx.amount + }); + } + }); + } + }); + } + + // loop through the address txes + lib.syncLoop(addressTxArray.length, function(address_loop) { + var a = address_loop.iteration(); + + // fix the balance, sent and received data for the current address + fix_address_data(addressTxArray[a], function() { + setTimeout(function() { + // move to the next address record + address_loop.next(); + }, timeout); + }); + }, function() { + // delete all AddressTx records from the local collection for this tx + AddressTx.deleteMany({txid: txid}, function(address_tx_err, address_tx_result) { + if (address_tx_err) + console.log(address_tx_err); + + // delete the tx from the local database + delete_tx(txid, block_height, function(tx_err, tx_result) { + if (tx_err) { + console.log(tx_err); + return cb(tx_count); + } else { + // check if the deleted tx had vouts + if (has_vouts) { + // keep a running total of txes that were removed + tx_count -= tx_result.deletedCount; + } + + return cb(tx_count); + } + }); + }); + }); + } + }); + } + }); +} + +function fix_address_data(address_data, cb) { + var addr_inc = {}; + var amount = address_data.amount; + + // determine how to fix the address balances + if (address_data.a_id == 'coinbase') + addr_inc.sent = -amount; + else if (amount < 0) { + // vin + addr_inc.sent = amount; + addr_inc.balance = -amount; + } else { + // vout + addr_inc.received = -amount; + addr_inc.balance = -amount; + } + + // reverse the amount from the running totals in the Address collection for the current address + Address.findOneAndUpdate({a_id: address_data.a_id}, { + $inc: addr_inc + }, { + upsert: false + }, function (address_err, return_address) { + if (address_err != null) + console.log(address_err); + + // finished fixing the address balance data + return cb(); + }); +} + +function delete_tx(txid, block_height, cb) { + // delete the tx from the local database + Tx.deleteOne({txid: txid, blockindex: block_height}, function(tx_err, tx_result) { + return cb(tx_err, tx_result); + }); +} + +function check_delete_tx(tx, block_height, tx_count, timeout, cb) { + // check if the tx object exists and does not match the current block height + if (tx && tx.blockindex != block_height) { + // the transaction exists but does not match the correct block height, therefore it should be deleted + delete_and_cleanup_tx(tx.txid, tx.blockindex, tx_count, timeout, function(updated_tx_count) { + // finished removing the transaction + return cb(updated_tx_count, true); + }); + } else { + // tx dosn't exist or block heights match so nothing to do + return cb(tx_count, false); + } +} + function update_heavy(coin, height, count, heavycoin_enabled, cb) { if (heavycoin_enabled == true) { db.update_heavy(coin, height, count, function() { @@ -398,11 +1131,12 @@ if (lib.is_locked([database]) == false) { console.log('Deleting block index.. Please wait..'); Stats.updateOne({coin: settings.coin.name}, { last: 0, - count: 0, supply: 0, txes: 0, blockchain_last_updated: 0, - richlist_last_updated: 0 + richlist_last_updated: 0, + orphan_index: 0, + orphan_current: 0 }, function() { console.log('Block index deleted successfully'); @@ -410,7 +1144,7 @@ if (lib.is_locked([database]) == false) { check_show_sync_message(stats.count); console.log('Starting resync of blockchain data.. Please wait..'); - update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.update_timeout, false, function() { + update_tx_db(settings.coin.name, block_start, stats.count, 0, settings.sync.update_timeout, 0, function() { // check if the script stopped prematurely if (stopSync) { console.log('Reindex was stopped prematurely'); @@ -418,25 +1152,34 @@ if (lib.is_locked([database]) == false) { } else { // update blockchain_last_updated value db.update_last_updated_stats(settings.coin.name, { blockchain_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - db.update_richlist('received', function() { - db.update_richlist('balance', function() { - // update richlist_last_updated value - db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - db.get_stats(settings.coin.name, function(nstats) { - // check for and update heavycoin data if applicable - update_heavy(settings.coin.name, stats.count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) { - // check for and update network history data if applicable - update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) { - // always check for and remove the sync msg if exists - db.remove_sync_message(); + // fix data from orphaned blocks + update_orphans(0, 0, stats.count, settings.sync.update_timeout, function() { + // check if the script stopped prematurely + if (stopSync) { + console.log('Reindex was stopped prematurely'); + exit(1); + } else { + db.update_richlist('received', function() { + db.update_richlist('balance', function() { + // update richlist_last_updated value + db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) { + db.get_stats(settings.coin.name, function(nstats) { + // check for and update heavycoin data if applicable + update_heavy(settings.coin.name, stats.count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) { + // check for and update network history data if applicable + update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) { + // always check for and remove the sync msg if exists + db.remove_sync_message(); - console.log('Reindex complete (block: %s)', nstats.last); - exit(0); + console.log('Reindex complete (block: %s)', nstats.last); + exit(0); + }); + }); }); }); }); }); - }); + } }); }); } @@ -449,7 +1192,7 @@ if (lib.is_locked([database]) == false) { } else if (mode == 'check') { console.log('Checking blocks.. Please wait..'); - update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, true, function() { + update_tx_db(settings.coin.name, block_start, stats.count, stats.txes, settings.sync.check_timeout, 1, function() { // check if the script stopped prematurely if (stopSync) { console.log('Block check was stopped prematurely'); @@ -469,7 +1212,7 @@ if (lib.is_locked([database]) == false) { // Check if the sync msg should be shown check_show_sync_message(count - last); - update_tx_db(settings.coin.name, last, count, stats.txes, settings.sync.update_timeout, false, function() { + update_tx_db(settings.coin.name, last, count, stats.txes, settings.sync.update_timeout, 0, function() { // check if the script stopped prematurely if (stopSync) { console.log('Block sync was stopped prematurely'); @@ -477,25 +1220,34 @@ if (lib.is_locked([database]) == false) { } else { // update blockchain_last_updated value db.update_last_updated_stats(settings.coin.name, { blockchain_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - db.update_richlist('received', function() { - db.update_richlist('balance', function() { - // update richlist_last_updated value - db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - db.get_stats(settings.coin.name, function(nstats) { - // check for and update heavycoin data if applicable - update_heavy(settings.coin.name, stats.count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) { - // check for and update network history data if applicable - update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) { - // always check for and remove the sync msg if exists - db.remove_sync_message(); + // fix data from orphaned blocks + update_orphans(stats.orphan_index, stats.orphan_current, count, settings.sync.update_timeout, function() { + // check if the script stopped prematurely + if (stopSync) { + console.log('Block sync was stopped prematurely'); + exit(1); + } else { + db.update_richlist('received', function() { + db.update_richlist('balance', function() { + // update richlist_last_updated value + db.update_last_updated_stats(settings.coin.name, { richlist_last_updated: Math.floor(new Date() / 1000) }, function(cb) { + db.get_stats(settings.coin.name, function(nstats) { + // check for and update heavycoin data if applicable + update_heavy(settings.coin.name, count, 20, settings.blockchain_specific.heavycoin.enabled, function(heavy) { + // check for and update network history data if applicable + update_network_history(nstats.last, settings.network_history.enabled, function(network_hist) { + // always check for and remove the sync msg if exists + db.remove_sync_message(); - console.log('Block sync complete (block: %s)', nstats.last); - exit(0); + console.log('Block sync complete (block: %s)', nstats.last); + exit(0); + }); + }); }); }); }); }); - }); + } }); }); } diff --git a/settings.json.template b/settings.json.template index 27e1998..4a9ca99 100644 --- a/settings.json.template +++ b/settings.json.template @@ -1268,6 +1268,38 @@ "enable_bad_word_filter": true }, + // orphans_page: a collection of settings that pertain to the orphans page + "orphans_page": { + // enabled: Enable/disable the orphans page (true/false) + // If set to false, the orphans page will be completely inaccessible + "enabled": true, + // show_panels: Determine whether to show the panels configured in the shared_pages.page_header section across the top of this page (true/false) + "show_panels": false, + // show_nethash_chart: Determine whether to show the network hashrate chart configured in the shared_pages.network_charts.nethash_chart section across the top of this page (true/false) + "show_nethash_chart": false, + // show_difficulty_chart: Determine whether to show the network difficulty chart configured in the shared_pages.network_charts.difficulty_chart section across the top of this page (true/false) + "show_difficulty_chart": false, + // page_header: a collection of settings that pertain to the orphans page header + "page_header": { + // show_img: Determine whether to show the page title image defined in the "shared_pages.page_header.page_title_image" setting (true/false) + "show_img": true, + // show_title: Determine whether to show the page title as defined in "locale.orphan_title" (true/false) + "show_title": true, + // show_last_updated: Determine whether to show a label below the page title with the last updated date (true/false) + "show_last_updated": true, + // show_description: Determine whether to show the page description as defined in "locale.orphan_description" (true/false) + "show_description": true + }, + // orphans_table: a collection of settings that pertain to the orphans table on the orphans page + // Table data is populated via the /ext/getorphanlist api + "orphans_table": { + // page_length_options: An array of page length options that determine how many items/records to display in the table at any given time + "page_length_options": [ 10, 25, 50, 75, 100, 250, 500, 1000 ], + // items_per_page: The default amount of items/records to display in the table at any given time + "items_per_page": 10 + } + }, + // sync: a collection of settings that pertain to the data synchronization process "sync": { // block_parallel_tasks: Use multiple threads to do blockchain syncing which greatly improves the initial sync speed, but there is a drawback. diff --git a/views/block.pug b/views/block.pug index c889bd2..8c66e63 100644 --- a/views/block.pug +++ b/views/block.pug @@ -30,6 +30,14 @@ block content startRotateElement('img#header-img'); }); .col-xs-12.col-md-12 + if orphan != null && orphan == true + .col-12 + .alert.alert-warning.alert-dismissible.fade.show(role='alert') + button.btn-close(type='button', data-bs-dismiss='alert') + .cardSpacer + span.fas.fa-exclamation-triangle(style='margin-right:5px') + strong=settings.locale.ex_warning + div="This is an orphaned block" if settings.block_page.page_header.show_img == true || settings.block_page.page_header.show_title == true || settings.block_page.page_header.show_description == true #page-header-container(style='align-items:' + (settings.block_page.page_header.show_img == true && settings.block_page.page_header.show_title == true && settings.block_page.page_header.show_description == true ? 'flex-start' : 'center')) if settings.block_page.page_header.show_img == true diff --git a/views/layout.pug b/views/layout.pug index 3e92e40..49e2912 100644 --- a/views/layout.pug +++ b/views/layout.pug @@ -117,6 +117,13 @@ html(lang='en') - showNethashChart = true if settings.claim_address_page.show_difficulty_chart == true - showDifficultyChart = true + when 'orphans' + if settings.orphans_page.show_panels == true + - showPanels = true + if settings.orphans_page.show_nethash_chart == true + - showNethashChart = true + if settings.orphans_page.show_difficulty_chart == true + - showDifficultyChart = true when 'reward' if settings.blockchain_specific.heavycoin.reward_page.show_panels == true - showPanels = true @@ -1135,6 +1142,11 @@ html(lang='en') a.nav-link(href='/claim') span.far.fa-address-card span.margin-left-5 #{settings.locale.menu_claim_address} + if settings.orphans_page.enabled == true + li#orphans.nav-item + a.nav-link(href='/orphans') + span.far.fa-window-close + span.margin-left-5 #{settings.locale.menu_orphans} if settings.shared_pages.page_header.search.enabled == true && settings.shared_pages.page_header.search.position.toString().toLowerCase() == 'inside-header' span#search-header-span(alt='Search', title='Search', data-bs-toggle='tooltip', data-bs-placement='left') button#search-header-button(data-bs-toggle='collapse', data-bs-target='#search-navbar-collapse') @@ -1247,6 +1259,11 @@ html(lang='en') a.nav-link(href='/claim') span.nav-icon.far.fa-address-card span.margin-left-5 #{settings.locale.menu_claim_address} + if settings.orphans_page.enabled == true + li#orphans.nav-item(alt=settings.locale.menu_orphans, title=settings.locale.menu_orphans, data-bs-toggle='tooltip', data-bs-placement='right') + a.nav-link(href='/orphans') + span.nav-icon.far.fa-window-close + span.margin-left-5 #{settings.locale.menu_orphans} div#side-offcanvas.offcanvas.offcanvas-start(tabindex='-1') div.offcanvas-body.d-block.navbar(class=sideBarClasses) div#main-container(class=mainContainerClasses) diff --git a/views/orphans.pug b/views/orphans.pug new file mode 100644 index 0000000..90e6eaf --- /dev/null +++ b/views/orphans.pug @@ -0,0 +1,123 @@ +extends layout + +block content + include ./includes/common.pug + script. + var setting_txPerPage = parseInt("#{settings.orphans_page.orphans_table.items_per_page}"); + var lengthMenuOpts = !{JSON.stringify(settings.orphans_page.orphans_table.page_length_options)}; + var addedLength = false; + for (i = 0; i < lengthMenuOpts.length; i++) { + if (!addedLength) { + if (lengthMenuOpts[i] > setting_txPerPage) { + lengthMenuOpts.splice(i, 0, setting_txPerPage); + addedLength = true; + } else if (lengthMenuOpts[i] == setting_txPerPage) + addedLength = true; + } + } + if (!addedLength && setting_txPerPage != lengthMenuOpts[lengthMenuOpts.length - 1]) + lengthMenuOpts.push(setting_txPerPage); + $(document).ready(function() { + var otable = $('#orphans-table').dataTable({ + autoWidth: true, + searching: false, + ordering: false, + lengthChange: true, + processing: true, + serverSide: true, + iDisplayLength: setting_txPerPage, + lengthMenu: lengthMenuOpts, + scrollX: true, + ajax: { + url: '/ext/getorphanlist', + beforeSend: function(jqXHR, settings) { + settings.url = settings.url.substring(0, settings.url.indexOf('?')) + '/' + getParameterByName('start', settings.url) + '/' + getParameterByName('length', settings.url); + return true; + } + }, + language: { + paginate: { + previous: '<', + next: '>' + } + }, + rowCallback: function(row, data, index) { + var blockindex = data[0]; + var orphan_blockhash = data[1]; + var good_blockhash = data[2]; + var prev_blockhash = data[3]; + var next_blockhash = data[4]; + $("td:eq(0)", row).html('').addClass('text-center d-table-cell d-md-none'); + $("td:eq(1)", row).html('' + orphan_blockhash + '').addClass('breakWord d-none d-md-table-cell'); + $("td:eq(2)", row).html('' + blockindex.toString() + '').addClass('text-center'); + $("td:eq(3)", row).html('' + (blockindex - 1).toString() + '').addClass('text-center'); + + if (next_blockhash == null) + $("td:eq(4)", row).html(' '); + else + $("td:eq(4)", row).html('' + (blockindex + 1).toString() + '').addClass('text-center'); + }, + fnDrawCallback: function(settings) { + fixDataTableColumns(); + fixFooterHeightAndPosition(); + enableTooltips(); + } + }); + + if ('#{settings.orphans_page.page_header.show_last_updated}' == 'true') { + var lastUpdatedDate = #{(last_updated == null || last_updated == '0' ? 0 : last_updated)}; + + if (lastUpdatedDate != 0) { + $('span#lastUpdatedDate').html(' ' + format_unixtime(lastUpdatedDate)); + + if (#{settings.shared_pages.date_time.enable_alt_timezone_tooltips} == true) { + $('span#lastUpdatedDate').attr('data-bs-toggle', 'tooltip').attr('data-bs-placement', 'auto').attr('title', format_unixtime(lastUpdatedDate, true)); + enableTooltips(); + } + } else + $('span#lastUpdatedDate').html(' N/A'); + } + if (#{settings.shared_pages.page_header.page_title_image.enable_animation} == true && #{settings.orphans_page.page_header.show_img} == true) + startRotateElement('img#header-img'); + }); + .col-md-12 + if settings.orphans_page.page_header.show_img == true || settings.orphans_page.page_header.show_title == true || settings.orphans_page.page_header.show_last_updated == true || settings.orphans_page.page_header.show_description == true + #page-header-container(style='align-items:' + (settings.orphans_page.page_header.show_img == true && settings.orphans_page.page_header.show_title == true && settings.orphans_page.page_header.show_last_updated == true && settings.orphans_page.page_header.show_description == true ? 'flex-start' : 'center')) + if settings.orphans_page.page_header.show_img == true + #header-img-container + img#header-img(src=(settings.shared_pages.page_header.page_title_image == null || settings.shared_pages.page_header.page_title_image.image_path == null || settings.shared_pages.page_header.page_title_image.image_path == '' ? '/img/page-title-img.png' : settings.shared_pages.page_header.page_title_image.image_path)) + #page-title-container + if settings.orphans_page.page_header.show_title == true + h3#page-title #{settings.locale.orphan_title.replace('{1}', settings.coin.name)} + if settings.orphans_page.page_header.show_last_updated == true + if settings.orphans_page.page_header.show_title != true && settings.orphans_page.page_header.show_description != true + #page-title-container + .sub-page-header + span.fw-bold=settings.locale.last_updated + ':' + span.text-muted#lastUpdatedDate + else + .sub-page-header(style='margin-bottom:' + (settings.orphans_page.page_header.show_description == true ? '5' : '0') + 'px') + span.fw-bold=settings.locale.last_updated + ':' + span.text-muted#lastUpdatedDate + if settings.orphans_page.page_header.show_description == true + if settings.orphans_page.page_header.show_title != true && settings.orphans_page.page_header.show_last_updated != true + #page-title-container + .sub-page-header.text-muted=settings.locale.orphan_description + else + .sub-page-header.text-muted=settings.locale.orphan_description + .cardSpacer.clearfix + .card.card-default.border-0.cardSpacer + .card-header + strong=settings.locale.orphan_block_list + table#orphans-table.table.table-bordered.table-striped.table-paging.table-hover.mobile-border-right + - var theadClasses = []; + if settings.shared_pages.table_header_bgcolor != null && settings.shared_pages.table_header_bgcolor != '' + - theadClasses.push('table-' + settings.shared_pages.table_header_bgcolor); + thead + tr(class=theadClasses) + th.d-table-cell.d-md-none + th.d-none.d-md-table-cell #{settings.locale.orphan_block_hash} + th.text-center #{settings.locale.orphan_actual_block} + th.text-center #{settings.locale.orphan_prev_block} + th.text-center #{settings.locale.orphan_next_block} + tbody \ No newline at end of file diff --git a/views/tx.pug b/views/tx.pug index 482dd4c..01bac56 100644 --- a/views/tx.pug +++ b/views/tx.pug @@ -17,6 +17,14 @@ block content startRotateElement('img#header-img'); }); .col-xs-12.col-md-12 + if orphan != null && orphan == true + .col-12 + .alert.alert-warning.alert-dismissible.fade.show(role='alert') + button.btn-close(type='button', data-bs-dismiss='alert') + .cardSpacer + span.fas.fa-exclamation-triangle(style='margin-right:5px') + strong=settings.locale.ex_warning + div="This is an orphaned transaction" if settings.transaction_page.page_header.show_img == true || settings.transaction_page.page_header.show_title == true || settings.transaction_page.page_header.show_description == true #page-header-container(style='align-items:' + (settings.transaction_page.page_header.show_img == true && settings.transaction_page.page_header.show_title == true && settings.transaction_page.page_header.show_description == true ? 'flex-start' : 'center')) if settings.transaction_page.page_header.show_img == true