From 09cf47456229099b0c0b7d0d18dc3e2b5f456676 Mon Sep 17 00:00:00 2001 From: Joe Uhren Date: Tue, 26 Sep 2023 18:22:28 -0600 Subject: [PATCH] Moved claim address data into its own collection -Created a new claimaddress collection to hold claim address data to make it easier to work with and preserve that data if necessary in the future without being cluttered into the address collection -The database init function has been updated to move claim address data to the new collection and remove the data from the address collection. This process will update existing explorer databases automatically and is smart enough to only run this process one time to prevent from slowing down startup of the explorer on each run -The claim name field has been changed from "name" to "claim_name" wherever possible to be easier to find in the future. Searching for the keyword "name" brings back way too many matches and makes it difficult to find all the correct code snippets for future changes -Added a newer_claim_address field to the stats collection to determine if the claim address data needs to be moved to the new collection or not -All previous claim address code has been updated to pull from the new table and/or join to the address table if/when necessary --- app.js | 2 +- lib/database.js | 280 +++++++++++++++++++++++++++++++++++------ models/address.js | 2 +- models/claimaddress.js | 9 ++ models/stats.js | 3 +- routes/index.js | 38 +++--- views/address.pug | 6 +- views/richlist.pug | 8 +- 8 files changed, 284 insertions(+), 64 deletions(-) create mode 100644 models/claimaddress.js diff --git a/app.js b/app.js index 4c60078..f8fc76d 100644 --- a/app.js +++ b/app.js @@ -117,7 +117,7 @@ app.post('/claim', function(req, res) { if (body == false) res.json({'status': 'failed', 'error': true, 'message': 'Invalid signature'}); else if (body == true) { - db.update_label(req.body.address, req.body.message, function(val) { + db.update_claim_name(req.body.address, req.body.message, function(val) { // check if the update was successful if (val == '') res.json({'status': 'success'}); diff --git a/lib/database.js b/lib/database.js index ec7c17e..f3659da 100644 --- a/lib/database.js +++ b/lib/database.js @@ -10,6 +10,7 @@ var mongoose = require('mongoose'), Peers = require('../models/peers'), Heavy = require('../models/heavy'), NetworkHistory = require('../models/networkhistory'), + ClaimAddress = require('../models/claimaddress'), lib = require('./explorer'), settings = require('./settings'), locale = require('./locale'), @@ -42,6 +43,18 @@ function find_address(hash, caseSensitive, cb) { } } +function find_claim_name(hash, cb) { + ClaimAddress.findOne({a_id: hash}).then((claim_address) => { + if (claim_address) + return cb(claim_address.claim_name); + else + return cb(); + }).catch((err) => { + console.log(err); + return cb(); + }); +} + function find_richlist(coin, cb) { Richlist.findOne({coin: coin}).then((richlist) => { if (richlist) @@ -169,6 +182,28 @@ function check_rename_db_field(model_obj, old_field_name, new_field_name, cb) { }); } +function check_remove_db_field(model_obj, field_name, cb) { + // determine if a particular field exists in a db collection + model_obj.findOne({[field_name]: {$exists: true}}).then((model_data) => { + // check if field exists + if (model_data) { + // remove field + model_obj.updateMany({}, { + $unset: { [field_name]: 1 } + }, { multi: true, strict: false }).then(() => { + return cb(true); + }).catch((err) => { + console.log(err); + return cb(false); + }); + } else + return cb(false); + }).catch((err) => { + console.log(err); + return cb(false); + }); +} + function hex_to_ascii(hex) { var str = ''; @@ -268,6 +303,64 @@ function init_heavy(cb) { return cb(); } +function init_claimaddress(coin, cb) { + // first, get the stats data + Stats.findOne({coin: coin}).then((stats) => { + var newer_claim_address = false; + + // check if stats were found + if (stats) { + // check if the claim address data was already moved to the new collection + if (stats.newer_claim_address != null && stats.newer_claim_address == true) + newer_claim_address = true; + } + + // check if the claim address data should be moved to a new collection + if (!newer_claim_address) { + // find all addresses with a custom claim address name + Address.find({$and: [{"name": {$ne: ""}}, {"name": {$ne: null}}]}).exec().then((addresses) => { + // loop through the claimed addresses + lib.syncLoop(addresses.length, function(address_loop) { + var a = address_loop.iteration(); + + // create a new claimaddress record + var claim_address = new ClaimAddress({ + a_id: addresses[a].a_id, + claim_name: addresses[a].name + }); + + // add new claim address to collection + claim_address.save().then(() => { + address_loop.next(); + }).catch((err) => { + console.log(err); + address_loop.next(); + }); + }, function() { + // finished moving all claimed address data to the new collection + // remove the name field from the address collection to reclaim disk space + check_remove_db_field(Address, 'name', function(removed) { + // update the stats data to prevent this one-time process from happening again in the future + Stats.updateOne({coin: coin}, {newer_claim_address: true}).then(() => { + return cb(true); + }).catch((err) => { + console.log(err); + return cb(false); + }); + }); + }); + }).catch((err) => { + console.log(err); + return cb(false); + }); + } else + return cb(true); + }).catch((err) => { + console.log(err); + return cb(false); + }); +} + // find masternode by txid and function find_masternode(txhash, addr, cb) { Masternode.findOne({ txhash: txhash, addr: addr }).then((masternode) => { @@ -281,6 +374,16 @@ function find_masternode(txhash, addr, cb) { }); } +function after_update_claim_name(hash, claim_name, cb) { + // update claim name in richlist + module.exports.update_richlist_claim_name(hash, claim_name, function() { + // update claim name in masternode list + module.exports.update_masternode_claim_name(hash, claim_name, function() { + return cb(); + }); + }); +} + module.exports = { // initialize DB connect: function(database, cb) { @@ -305,27 +408,36 @@ module.exports = { return fs.existsSync('./tmp/show_sync_message.tmp'); }, - update_label: function(hash, claim_name, cb) { - find_address(hash, false, function(address) { - if (address) { - Address.updateOne({a_id: hash}, { - name: claim_name - }).then(() => { - // update claim name in richlist - module.exports.update_richlist_claim_name(hash, claim_name, function() { - // update claim name in masternode list - module.exports.update_masternode_claim_name(hash, claim_name, function() { - return cb(''); - }); - }); - }).catch((err) => { - return cb(err); + update_claim_name: function(hash, claim_name, cb) { + // check if the claim name is being removed + if (claim_name == null || claim_name == '') { + // remove the claim name + ClaimAddress.findOneAndRemove({a_id: hash}).then(() => { + // run processes after the claim name has been updated + after_update_claim_name(hash, claim_name, function() { + return cb(''); }); - } else { - // address is not valid or does not have any transactions - return cb('no_address'); - } - }); + }).catch((err) => { + console.log(err); + return cb(err); + }); + } else { + // add or update the claim name + ClaimAddress.updateOne({a_id: hash}, { + a_id: hash, + claim_name: claim_name + }, { + upsert: true + }).then(() => { + // run processes after the claim name has been updated + after_update_claim_name(hash, claim_name, function() { + return cb(''); + }); + }).catch((err) => { + console.log(err); + return cb(err); + }); + } }, update_richlist_claim_name: function(hash, claim_name, cb) { @@ -482,7 +594,8 @@ module.exports = { coin: coin, last: 0, orphan_index: 0, - orphan_current: 0 + orphan_current: 0, + newer_claim_address: true }); newStats.save().then(() => { @@ -502,6 +615,12 @@ module.exports = { }); }, + get_claim_name: function(hash, cb) { + find_claim_name(hash, function(claim_name) { + return cb(claim_name); + }); + }, + get_richlist: function(coin, cb) { find_richlist(coin, function(richlist) { return cb(richlist); @@ -522,7 +641,36 @@ module.exports = { if (list == 'received') { // update 'received' richlist data - Address.find({a_id: { $nin: burn_addresses }}, 'a_id name balance received').sort({received: 'desc'}).limit(total_addresses).exec().then((addresses) => { + Address.aggregate([ + { $match: { + a_id: { $nin: burn_addresses } + }}, + { $sort: {received: -1} }, + { $limit: total_addresses }, + { $lookup: + { + from: "claimaddresses", + localField: "a_id", + foreignField: "a_id", + as: "claim_name" + } + }, + { $unwind: { path: "$claim_name", preserveNullAndEmptyArrays: true } }, + { + $addFields: + { + 'claim_name': '$claim_name.claim_name' + } + }, + { + $project: + { + "_id": 0, + "__v": 0, + "sent": 0 + } + } + ]).then((addresses) => { Richlist.updateOne({coin: settings.coin.name}, { received: addresses }).then(() => { @@ -540,7 +688,36 @@ module.exports = { // check if burned addresses are in use and if it is necessary to track burned balances if (settings.richlist_page.burned_coins.addresses == null || settings.richlist_page.burned_coins.addresses.length == 0 || !settings.richlist_page.burned_coins.include_burned_coins_in_distribution) { // update 'balance' richlist data by filtering burned coin addresses immidiately - Address.find({a_id: { $nin: burn_addresses }}, 'a_id name balance received').sort({balance: 'desc'}).limit(total_addresses).exec().then((addresses) => { + Address.aggregate([ + { $match: { + a_id: { $nin: burn_addresses } + }}, + { $sort: {balance: -1} }, + { $limit: total_addresses }, + { $lookup: + { + from: "claimaddresses", + localField: "a_id", + foreignField: "a_id", + as: "claim_name" + } + }, + { $unwind: { path: "$claim_name", preserveNullAndEmptyArrays: true } }, + { + $addFields: + { + 'claim_name': '$claim_name.claim_name' + } + }, + { + $project: + { + "_id": 0, + "__v": 0, + "sent": 0 + } + } + ]).then((addresses) => { Richlist.updateOne({coin: settings.coin.name}, { balance: addresses }).then(() => { @@ -555,7 +732,33 @@ module.exports = { }); } else { // do not omit burned addresses from database query. instead, increase the limit of returned addresses and manually remove each burned address that made it into the rich list after recording the burned balance - Address.find({}, 'a_id name balance received').sort({balance: 'desc'}).limit(total_addresses + burn_addresses.length).exec().then((addresses) => { + Address.aggregate([ + { $sort: {balance: -1} }, + { $limit: total_addresses + burn_addresses.length }, + { $lookup: + { + from: "claimaddresses", + localField: "a_id", + foreignField: "a_id", + as: "claim_name" + } + }, + { $unwind: { path: "$claim_name", preserveNullAndEmptyArrays: true } }, + { + $addFields: + { + 'claim_name': '$claim_name.claim_name' + } + }, + { + $project: + { + "_id": 0, + "__v": 0, + "sent": 0 + } + } + ]).then((addresses) => { var return_addresses = []; var burned_balance = 0.0; @@ -1271,10 +1474,10 @@ module.exports = { // determine if the claim address feature is enabled if (settings.claim_address_page.enabled == true) { // claim address is enabled so lookup the address claim name - find_address(addr, false, function(address) { - if (address) { + find_claim_name(addr, function(claim_name) { + if (claim_name != null && claim_name != '') { // save claim name to masternode obejct - raw_masternode.claim_name = address.name; + raw_masternode.claim_name = claim_name; } else { // save blank claim name to masternode obejct raw_masternode.claim_name = ''; @@ -1498,26 +1701,26 @@ module.exports = { }); // loop through address array - lib.syncLoop(addresses.length, function (loop) { + lib.syncLoop(addresses.length, function(loop) { var a = loop.iteration(); - module.exports.get_address(addresses[a], false, function(address) { - if (address && address.name != null && address.name != '') { + module.exports.get_claim_name(addresses[a], function(claim_name) { + if (claim_name != null && claim_name != '') { // look for address in vin for (v = 0; v < tx.vin.length; v++) { // check if this is the correct address - if (tx.vin[v].addresses == address.a_id) { + if (tx.vin[v].addresses == addresses[a]) { // add claim name to array - tx.vin[v]['claim_name'] = address.name; + tx.vin[v]['claim_name'] = claim_name; } } // look for address in vout for (v = 0; v < tx.vout.length; v++) { // check if this is the correct address - if (tx.vout[v].addresses == address.a_id) { + if (tx.vout[v].addresses == addresses[a]) { // add claim name to array - tx.vout[v]['claim_name'] = address.name; + tx.vout[v]['claim_name'] = claim_name; } } } @@ -1596,9 +1799,12 @@ module.exports = { module.exports.create_richlist(settings.coin.name, skip, function() { // check and initialize the heavycoin collection init_heavy(function() { - // finished initializing startup data - console.log('Database initialization complete'); - return cb(); + // check and initialize the claimaddress collection + init_claimaddress(settings.coin.name, function() { + // finished initializing startup data + console.log('Database initialization complete'); + return cb(); + }); }); }); }); diff --git a/models/address.js b/models/address.js index a196954..200db42 100644 --- a/models/address.js +++ b/models/address.js @@ -3,7 +3,7 @@ var mongoose = require('mongoose'), var AddressSchema = new Schema({ a_id: { type: String, unique: true, index: true}, - name: { type: String, default: '', index: true}, + name: { type: String }, // no longer used but cannot be removed or else older versions that have data here will not be able to auto move claim name data to the new claimaddress collection received: { type: Number, default: 0, index: true }, sent: { type: Number, default: 0, index: true }, balance: {type: Number, default: 0, index: true}, diff --git a/models/claimaddress.js b/models/claimaddress.js new file mode 100644 index 0000000..310cdc1 --- /dev/null +++ b/models/claimaddress.js @@ -0,0 +1,9 @@ +var mongoose = require('mongoose'), + Schema = mongoose.Schema; + +var ClaimAddressSchema = new Schema({ + a_id: {type: String, unique: true, index: true}, + claim_name: {type: String, default: '', index: true} +}, {id: false}); + +module.exports = mongoose.model('ClaimAddress', ClaimAddressSchema); \ No newline at end of file diff --git a/models/stats.js b/models/stats.js index 0b5a700..0831f7c 100644 --- a/models/stats.js +++ b/models/stats.js @@ -17,7 +17,8 @@ var StatsSchema = new Schema({ richlist_last_updated: { type: Number, default: 0 }, markets_last_updated: { type: Number, default: 0 }, orphan_index: { type: Number, default: 0 }, - orphan_current: { type: Number, default: 0 } + orphan_current: { type: Number, default: 0 }, + newer_claim_address: { type: Boolean, default: false } }); module.exports = mongoose.model('coinstats', StatsSchema); \ No newline at end of file diff --git a/routes/index.js b/routes/index.js index 237b078..b5fd3fd 100644 --- a/routes/index.js +++ b/routes/index.js @@ -330,21 +330,25 @@ function route_get_address(res, hash) { if (hash != null && hash.toLowerCase() != 'coinbase' && ((hash.toLowerCase() == 'hidden_address' && settings.address_page.enable_hidden_address_view == true) || (hash.toLowerCase() == 'unknown_address' && settings.address_page.enable_unknown_address_view == true) || (hash.toLowerCase() != 'hidden_address' && hash.toLowerCase() != 'unknown_address'))) { // lookup address in local collection db.get_address(hash, false, function(address) { - if (address) - res.render( - 'address', - { - active: 'address', - address: address, - showSync: db.check_show_sync_message(), - customHash: get_file_timestamp('./public/css/custom.scss'), - styleHash: get_file_timestamp('./public/css/style.scss'), - themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), - page_title_prefix: settings.coin.name + ' Address ' + (address['name'] == null || address['name'] == '' ? address.a_id : address['name']) - } - ); - else - route_get_index(res, hash + ' not found'); + // lookup claim_name for this address if exists + db.get_claim_name(hash, function(claim_name) { + if (address) + res.render( + 'address', + { + active: 'address', + address: address, + claim_name: claim_name, + showSync: db.check_show_sync_message(), + customHash: get_file_timestamp('./public/css/custom.scss'), + styleHash: get_file_timestamp('./public/css/style.scss'), + themeHash: get_file_timestamp('./public/css/themes/' + settings.shared_pages.theme.toLowerCase() + '/bootstrap.min.css'), + page_title_prefix: settings.coin.name + ' Address ' + (claim_name == null || claim_name == '' ? address.a_id : claim_name) + } + ); + else + route_get_index(res, hash + ' not found'); + }); }); } else route_get_index(res, hash + ' not found'); @@ -371,14 +375,14 @@ function route_get_claim_form(res, hash) { ); } else { // lookup hash in the address collection - db.get_address(hash, false, function(address) { + db.get_claim_name(hash, function(claim_name) { // load the claim page regardless of whether the address exists or not res.render( 'claim_address', { active: 'claim-address', hash: hash, - claim_name: (address == null || address.name == null ? '' : address.name), + claim_name: (claim_name == null ? '' : claim_name), showSync: db.check_show_sync_message(), customHash: get_file_timestamp('./public/css/custom.scss'), styleHash: get_file_timestamp('./public/css/style.scss'), diff --git a/views/address.pug b/views/address.pug index d2a544e..c9a89a1 100644 --- a/views/address.pug +++ b/views/address.pug @@ -147,7 +147,7 @@ block content .cardSpacer.clearfix .card.card-default.border-0.card-address-summary.cardSpacer .card-header(style='position:relative;') - if settings.claim_address_page.enabled == false || address.name == null || address.name == '' + if settings.claim_address_page.enabled == false || claim_name == null || claim_name == '' if address.a_id == 'hidden_address' strong #{settings.locale.hidden_address} else if address.a_id == 'unknown_address' @@ -155,12 +155,12 @@ block content else strong #{address.a_id} else - strong #{address.name} + strong #{claim_name} | ​ include ./includes/rl_labels.pug if (!settings.labels[address.a_id] || !settings.labels[address.a_id].enabled) && settings.claim_address_page.enabled == true a#claim-address.fw-bold(href='/claim/' + address.a_id, style='font-size:smaller;padding-bottom:0;') - if address.name == null || address.name == '' + if claim_name == null || claim_name == '' =" Is this yours? Claim it now for free!" else =" Update claimed address" diff --git a/views/richlist.pug b/views/richlist.pug index 7c603eb..f215720 100644 --- a/views/richlist.pug +++ b/views/richlist.pug @@ -132,10 +132,10 @@ block content td.text-center =count td - if settings.claim_address_page.enabled == false || address.name == null || address.name == '' + if settings.claim_address_page.enabled == false || address.claim_name == null || address.claim_name == '' a.breakWord(href='/address/' + address.a_id) #{address.a_id} else - a.breakWord(href='/address/' + address.a_id) #{address.name} + a.breakWord(href='/address/' + address.a_id) #{address.claim_name} include ./includes/rl_labels.pug td.text-center #{itemFixedParts[0]}. span.decimal #{itemFixedParts[1]} @@ -162,10 +162,10 @@ block content td.text-center =count td - if settings.claim_address_page.enabled == false || address.name == null || address.name == '' + if settings.claim_address_page.enabled == false || address.claim_name == null || address.claim_name == '' a.breakWord(href='/address/' + address.a_id) #{address.a_id} else - a.breakWord(href='/address/' + address.a_id) #{address.name} + a.breakWord(href='/address/' + address.a_id) #{address.claim_name} include ./includes/rl_labels.pug td.text-center #{itemFixedParts[0]}. span.decimal #{itemFixedParts[1]}