From 65c48ea829d8c17f63a836586335f8df094a7109 Mon Sep 17 00:00:00 2001 From: Joe Uhren Date: Fri, 5 Jan 2024 00:47:22 -0700 Subject: [PATCH] Added market sync options: average and coingecko -The previous market price calculation setting was hardcoded to only display market and USD prices for a single exchange and trading pair which was not very accurate for coins listed on multiple exchanges or with multiple trading pairs. The new default is to average the market prices for all supported exchanges and trading pairs -The coingecko market price option was added to allow fetching the market price directly from the coingecko api instead of calculating it via supported exchanges known to the explorer -Added a new root setting option for default_coingecko_ids which allows presetting symbols to their associated internal coingecko id to help prevent matching to the wrong currency with same symbol via coingecko api calls -Fixed an issue where the explorer would fail to start with an enabled exchange that had no defined trading pairs --- README.md | 2 +- app.js | 25 +++- lib/apis/coingecko.js | 78 ++++++++++--- lib/database.js | 189 ++++++++++-------------------- lib/explorer.js | 13 +++ lib/settings.js | 33 +++++- scripts/sync.js | 255 ++++++++++++++++++++++++++++++++++++----- settings.json.template | 33 +++++- views/info.pug | 5 +- views/layout.pug | 5 +- 10 files changed, 443 insertions(+), 195 deletions(-) diff --git a/README.md b/README.md index a453b39..759712b 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Table of Contents - **Movement:** Displays latest blockchain transactions that are greater than a certain configurable amount - **Network:** Displays a list of peers that have connected to the coind wallet in the past 24 hours, along with useful addnode data that can be used to connect your own wallets to the network easier - **Top 100:** Displays the top 100 richest wallet addresses, the top 100 wallet addresses that have the highest total number of coins received based on adding up all received transactions, as well as a table and pie chart breakdown of wealth distribution. Additional support for omitting burned coins from top 100 lists - - **Markets:** Displays a number of exchange-related metrics including market summary, 24 hour chart, most recent buy/sell orders and latest trade history. The last known default exchange price is automatically converted to USD using the coingecko api from [https://www.coingecko.com/en/api](https://www.coingecko.com/en/api). The following 8 cryptocurrency exchanges are supported: + - **Markets:** Displays a number of exchange-related metrics including market summary, 24 hour chart, most recent buy/sell orders and latest trade history. Has the ability to integrate directly with exchange apis and/or the coingecko api from [https://www.coingecko.com/en/api](https://www.coingecko.com/en/api) to retrieve current market prices and convert to USD. The following 8 cryptocurrency exchanges are supported: - [AltMarkets](https://altmarkets.io) - [Bittrex](https://bittrex.com) - [Dex-Trade](https://dex-trade.com) diff --git a/app.js b/app.js index c536b69..8c87b67 100644 --- a/app.js +++ b/app.js @@ -286,7 +286,9 @@ app.use('/ext/getcurrentprice', function(req, res) { // check if the getcurrentprice api is enabled if (settings.api_page.enabled == true && settings.api_page.public_apis.ext.getcurrentprice.enabled == true) { db.get_stats(settings.coin.name, function (stats) { - eval('var p_ext = { "last_price_' + settings.markets_page.default_exchange.trading_pair.split('/')[1].toLowerCase() + '": stats.last_price, "last_price_usd": stats.last_usd_price, }'); + const currency = lib.get_market_currency_code(); + + eval('var p_ext = { "last_price_' + currency.toLowerCase() + '": stats.last_price, "last_price_usd": stats.last_usd_price, }'); res.send(p_ext); }); } else @@ -298,16 +300,18 @@ app.use('/ext/getbasicstats', function(req, res) { if (settings.api_page.enabled == true && settings.api_page.public_apis.ext.getbasicstats.enabled == true) { // lookup stats db.get_stats(settings.coin.name, function (stats) { + const currency = lib.get_market_currency_code(); + // check if the masternode count api is enabled if (settings.api_page.public_apis.rpc.getmasternodecount.enabled == true && settings.api_cmds['getmasternodecount'] != null && settings.api_cmds['getmasternodecount'] != '') { // masternode count api is available lib.get_masternodecount(function(masternodestotal) { - eval('var p_ext = { "block_count": (stats.count ? stats.count : 0), "money_supply": (stats.supply ? stats.supply : 0), "last_price_' + settings.markets_page.default_exchange.trading_pair.split('/')[1].toLowerCase() + '": stats.last_price, "last_price_usd": stats.last_usd_price, "masternode_count": masternodestotal.total }'); + eval('var p_ext = { "block_count": (stats.count ? stats.count : 0), "money_supply": (stats.supply ? stats.supply : 0), "last_price_' + currency.toLowerCase() + '": stats.last_price, "last_price_usd": stats.last_usd_price, "masternode_count": masternodestotal.total }'); res.send(p_ext); }); } else { // masternode count api is not available - eval('var p_ext = { "block_count": (stats.count ? stats.count : 0), "money_supply": (stats.supply ? stats.supply : 0), "last_price_' + settings.markets_page.default_exchange.trading_pair.split('/')[1].toLowerCase() + '": stats.last_price, "last_price_usd": stats.last_usd_price }'); + eval('var p_ext = { "block_count": (stats.count ? stats.count : 0), "money_supply": (stats.supply ? stats.supply : 0), "last_price_' + currency.toLowerCase() + '": stats.last_price, "last_price_usd": stats.last_usd_price }'); res.send(p_ext); } }); @@ -768,9 +772,17 @@ if (settings.markets_page.enabled == true) { return 0; }); - // Fix default exchange case - settings.markets_page.default_exchange.exchange_name = settings.markets_page.default_exchange.exchange_name.toLowerCase(); - settings.markets_page.default_exchange.trading_pair = settings.markets_page.default_exchange.trading_pair.toUpperCase(); + // fix default exchange name case + if (settings.markets_page.default_exchange.exchange_name != null) + settings.markets_page.default_exchange.exchange_name = settings.markets_page.default_exchange.exchange_name.toLowerCase(); + else + settings.markets_page.default_exchange.exchange_name = ''; + + // fix default exchange trading pair case + if (settings.markets_page.default_exchange.trading_pair != null) + settings.markets_page.default_exchange.trading_pair = settings.markets_page.default_exchange.trading_pair.toUpperCase(); + else + settings.markets_page.default_exchange.trading_pair = ''; var ex = settings.markets_page.exchanges; var ex_name = settings.markets_page.default_exchange.exchange_name; @@ -848,6 +860,7 @@ app.set('api_page', settings.api_page); app.set('claim_address_page', settings.claim_address_page); app.set('orphans_page', settings.orphans_page); app.set('labels', settings.labels); +app.set('default_coingecko_ids', settings.default_coingecko_ids); app.set('api_cmds', settings.api_cmds); app.set('blockchain_specific', settings.blockchain_specific); diff --git a/lib/apis/coingecko.js b/lib/apis/coingecko.js index 1bbeb16..5666170 100644 --- a/lib/apis/coingecko.js +++ b/lib/apis/coingecko.js @@ -1,43 +1,83 @@ -var request = require('postman-request'); -var base_url = 'https://api.coingecko.com/api/v3/'; +const request = require('postman-request'); +const base_url = 'https://api.coingecko.com/api/v3/'; function get_coin_list(cb) { request({ uri: base_url + 'coins/list?include_platform=false', json: true}, function (error, response, body) { if (error) return cb(error, []); + else if (body == null || body == '' || typeof body !== 'object') + return cb('No data returned', []); else return cb(null, body); }); } -function get_usd_value(id, cb) { - request({ uri: base_url + 'simple/price?ids=' + id + '&vs_currencies=usd', json: true}, function (error, response, body) { +function get_simple_price(id, currency, market_array, cb) { + request({ uri: base_url + `simple/price?ids=${id.toLowerCase()}&vs_currencies=usd${currency == null || currency == '' ? '' : `,${currency}`}`, json: true}, function (error, response, body) { if (error) - return cb(error, 0); - else - return cb(null, body[id].usd); + return cb(error, 0, 0); + else if (body == null || body == '' || typeof body !== 'object') + return cb('No data returned', 0, 0); + else { + try { + if (market_array != null) { + // multiple currencies need to be combined before return + let last_price = 0; + let last_usd_price = 0; + let counter = 0; + + // loop through all api object keys + Object.keys(body).forEach(function(key, index, map) { + const market_index = market_array.findIndex(p => p.coingecko_id.toLowerCase() == key.toLowerCase()); + + // check if the currency is found in the market_array + if (market_index > -1) { + // calculate the market and usd prices + last_price += (market_array[market_index].last_price * (currency == null || currency == '' ? 0 : body[key][currency.toLowerCase()])); + last_usd_price += (market_array[market_index].last_price * body[key]['usd']); + counter++; + } + }); + + // check if the counter is greater than 0 + if (counter > 0) { + // average the market and usd prices + last_price = (last_price / counter); + last_usd_price = (last_usd_price / counter); + } + + return cb(null, last_price, last_usd_price); + } else { + // single currency + return cb(null, (currency == null || currency == '' ? 0 : body[id.toLowerCase()][currency.toLowerCase()]), body[id.toLowerCase()]['usd']); + } + } catch(err) { + return cb('Received unexpected API data response', 0, 0); + } + } }); } module.exports = { get_coin_data: function (cb) { - var error = null; - get_coin_list(function (err, coin_list) { - if (err) - error = err; - - return cb(error, coin_list); + return cb(err, coin_list); }); }, - get_data: function (id, cb) { - var error = null; + get_market_prices: function (id, currency, cb) { + get_simple_price(id, currency, null, function (err, last_price, last_usd) { + if (last_price == null) + console.log(`Error: "${currency}" is not a valid coingecko api currency`); - get_usd_value(id, function (err, last_usd) { - if (err) - error = err; + return cb(err, last_price, last_usd); + }); + }, + get_avg_market_prices: function (id, currency, market_array, cb) { + get_simple_price(id, currency, market_array, function (err, last_price, last_usd) { + if (last_price == null) + console.log(`Error: "${currency}" is not a valid coingecko api currency`); - return cb(error, last_usd); + return cb(err, last_price, last_usd); }); } }; \ No newline at end of file diff --git a/lib/database.js b/lib/database.js index d03888a..341fc3f 100644 --- a/lib/database.js +++ b/lib/database.js @@ -14,8 +14,7 @@ var mongoose = require('mongoose'), lib = require('./explorer'), settings = require('./settings'), locale = require('./locale'), - fs = require('fs'), - coingecko = require('./apis/coingecko'); + fs = require('fs'); function find_address(hash, caseSensitive, cb) { if (caseSensitive) { @@ -226,64 +225,68 @@ function init_markets(cb) { if (settings.markets_page.exchanges[key].enabled == true) { // check if exchange is installed/supported if (module.exports.fs.existsSync('./lib/markets/' + key + '.js')) { - let pairCounter = 0; + // check if there are any trading pairs + if (settings.markets_page.exchanges[key].trading_pairs.length > 0) { + let pairCounter = 0; - // loop through all trading pairs - settings.markets_page.exchanges[key].trading_pairs.forEach(function (pair_key, pair_index, pair_map) { - // split the pair data - let split_pair = pair_key.toUpperCase().split('/'); + // loop through all trading pairs + settings.markets_page.exchanges[key].trading_pairs.forEach(function (pair_key, pair_index, pair_map) { + // split the pair data + let split_pair = pair_key.toUpperCase().split('/'); - // check if this is a valid trading pair - if (split_pair.length == 2) { - // add this pair to the list of installed markets - installed_markets.push({ - market: key, - coin_symbol: split_pair[0], - pair_symbol: split_pair[1] - }); + // check if this is a valid trading pair + if (split_pair.length == 2) { + // add this pair to the list of installed markets + installed_markets.push({ + market: key, + coin_symbol: split_pair[0], + pair_symbol: split_pair[1] + }); - // lookup the exchange in the market collection - module.exports.check_market(key, split_pair[0], split_pair[1], function(market, exists) { - // check if exchange trading pair exists in the market collection - if (!exists) { - // exchange doesn't exist in the market collection so add a default definition now - console.log('No %s[%s] entry found. Creating new entry now..', market, split_pair[0] + '/' + split_pair[1]); + // lookup the exchange in the market collection + module.exports.check_market(key, split_pair[0], split_pair[1], function(market, exists) { + // check if exchange trading pair exists in the market collection + if (!exists) { + // exchange doesn't exist in the market collection so add a default definition now + console.log('No %s[%s] entry found. Creating new entry now..', market, split_pair[0] + '/' + split_pair[1]); - module.exports.create_market(split_pair[0], split_pair[1], market, function() { + module.exports.create_market(split_pair[0], split_pair[1], market, function() { + pairCounter++; + + // check if all pairs have been tested + if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length) + marketCounter++; + + // check if all exchanges have been tested + if (marketCounter == Object.keys(settings.markets_page.exchanges).length) { + // finished initializing markets + return cb(installed_markets); + } + }); + } else { pairCounter++; // check if all pairs have been tested if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length) marketCounter++; + } - // check if all exchanges have been tested - if (marketCounter == Object.keys(settings.markets_page.exchanges).length) { - // finished initializing markets - return cb(installed_markets); - } - }); - } else { - pairCounter++; + // check if all exchanges have been tested + if (marketCounter == Object.keys(settings.markets_page.exchanges).length) { + // finished initializing markets + return cb(installed_markets); + } + }); + } else { + pairCounter++; - // check if all pairs have been tested - if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length) - marketCounter++; - } - - // check if all exchanges have been tested - if (marketCounter == Object.keys(settings.markets_page.exchanges).length) { - // finished initializing markets - return cb(installed_markets); - } - }); - } else { - pairCounter++; - - // check if all pairs have been tested - if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length) - marketCounter++; - } - }); + // check if all pairs have been tested + if (pairCounter == settings.markets_page.exchanges[key].trading_pairs.length) + marketCounter++; + } + }); + } else + marketCounter++; } else marketCounter++; } else @@ -1302,9 +1305,9 @@ module.exports = { update_markets_db: function(market, coin_symbol, pair_symbol, cb) { // check if market exists if (fs.existsSync('./lib/markets/' + market + '.js')) { - get_market_data(market, coin_symbol, pair_symbol, function (err, obj) { + get_market_data(market, coin_symbol, pair_symbol, function (market_err, obj) { // check if there was an error with getting market data - if (err == null) { + if (market_err == null) { // update the market collection for the current market and trading pair combination Markets.updateOne({market: market, coin_symbol: coin_symbol, pair_symbol: pair_symbol}, { chartdata: JSON.stringify(obj.chartdata), @@ -1313,93 +1316,19 @@ module.exports = { history: obj.trades, summary: obj.stats }).then(() => { - // check if this is the default market and trading pair - if (market == settings.markets_page.default_exchange.exchange_name && settings.markets_page.default_exchange.trading_pair.toUpperCase() == coin_symbol.toUpperCase() + '/' + pair_symbol.toUpperCase()) { - // this is the default market so update the last price stats - Stats.updateOne({coin: settings.coin.name}, { - last_price: obj.stats.last - }).then(() => { - // finished updating market data - return cb(null); - }).catch((err) => { - console.log(err); - return cb(null); - }); - } else { - // this is not the default market so we are finished updating market data - return cb(null); - } + // finished updating market data + return cb(null, obj.stats.last); }).catch((err) => { - console.log(err); - return cb(null); + return cb(err, null); }); } else { // an error occurred with getting market data so return the error msg - return cb(err); + return cb(market_err, null); } }); } else { // market does not exist - return cb('market is not installed'); - } - }, - - get_last_usd_price: function(cb) { - // check if the default market is enabled - if (settings.markets_page.exchanges[settings.markets_page.default_exchange.exchange_name].enabled == true) { - // get the list of coins from coingecko - coingecko.get_coin_data(function (err, coin_list) { - // check for errors - if (err == null) { - var symbol = settings.markets_page.default_exchange.trading_pair.split('/')[1]; - var index = coin_list.findIndex(p => p.symbol.toLowerCase() == symbol.toLowerCase()); - - // check if the default market pair is found in the coin list - if (index > -1) { - // initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis - var rateLimitLib = require('./ratelimit'); - var rateLimit = new rateLimitLib.RateLimit(1, 2000, false); - - // automatically pause for 2 seconds in between requests - rateLimit.schedule(function() { - // get the usd value of the default market pair from coingecko - coingecko.get_data(coin_list[index].id, function (err, last_usd) { - // check for errors - if (err == null) { - // get current stats - Stats.findOne({coin: settings.coin.name}).then((stats) => { - // update the last usd price - Stats.updateOne({coin: settings.coin.name}, { - last_usd_price: (last_usd * stats.last_price) - }).then(() => { - // last usd price updated successfully - return cb(null); - }).catch((err) => { - // return error msg - return cb(err); - }); - }).catch((err) => { - // return error msg - return cb(err); - }); - } else { - // return error msg - return cb(err); - } - }); - }); - } else { - // return error msg - return cb('cannot find symbol ' + symbol + ' in the coingecko api'); - } - } else { - // return error msg - return cb(err); - } - }); - } else { - // default exchange is not enabled so just exit without updating last price for now - return cb(null); + return cb('market is not installed', null); } }, diff --git a/lib/explorer.js b/lib/explorer.js index ea597c2..696e5a2 100644 --- a/lib/explorer.js +++ b/lib/explorer.js @@ -1490,5 +1490,18 @@ module.exports = { } return retVal; + }, + + get_market_currency_code: function() { + let currency = ''; + + // check if the market price is being updated by coingecko api + if (settings.markets_page.market_price == 'COINGECKO') { + currency = (settings.markets_page.coingecko_currency == null || settings.markets_page.coingecko_currency == '' ? '' : settings.markets_page.coingecko_currency); + } else if (settings.markets_page.default_exchange.trading_pair != null && settings.markets_page.default_exchange.trading_pair.indexOf('/') > -1) { + currency = settings.markets_page.default_exchange.trading_pair.split('/')[1]; + } + + return currency; } }; \ No newline at end of file diff --git a/lib/settings.js b/lib/settings.js index 6d38f09..031db2b 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -233,7 +233,7 @@ exports.shared_pages = { // The panel will be disabled with a value of 0 "display_order": 1 }, - // price_panel: a collection of settings that pertain to the price panel which displays the current market price measured against the default market pair + // price_panel: a collection of settings that pertain to the price panel which displays the current market price as determined by the markets_page.market_price value "price_panel": { // enabled: Enable/disable the price panel (true/false) // If set to false, the price panel will be completely inaccessible @@ -253,7 +253,7 @@ exports.shared_pages = { // The panel will be disabled with a value of 0 "display_order": 0 }, - // market_cap_panel: a collection of settings that pertain to the market cap panel which displays the current market cap value measured against the default market pair + // market_cap_panel: a collection of settings that pertain to the market cap panel which displays the current market cap value as determined by the markets_page.market_price value "market_cap_panel": { // enabled: Enable/disable the market cap panel (true/false) // If set to false, the market cap panel will be completely inaccessible @@ -917,9 +917,22 @@ exports.markets_page = { "trading_pairs": [ "LTC/BTC" ] } }, + // market_price: Determine how to calculate the market price + // NOTE: The market price is always retrieved at the end of the market sync process + // Valid options: + // AVERAGE : Market price is calculated based on averaging all supported exchange trading pairs that are enabled in the explorer and measured in the default_exchange.trading_pair value + // COINGECKO : Market price is retrieved directly from the coingecko api. This option is somewhat special in that it does not require the markets_page option or any supported markets to be set up or enabled to function correctly + "market_price": "AVERAGE", + // coingecko_currency: Determine the default cryptocurrency value to measure your coin against when using the COINGECKO market_price option + // NOTE: This value is only necessary to fill out if you set market_price = "COINGECKO". + // This can be any cryptocurrency symbol value that the coingecko api supports such as BTC or ETH for example. + // Although the coingecko api supports multiple different currencies, the explorer only supports 1 cryptocurrency market price and therefore specifying multiple currencies separated by commas will not work here. + // The USD fiat currency is also built-in and automatically returned from the coingecko api so there is no need to specify USD here. + // For more information about which currency values are supported, you can review the vs_currencies parameter of the /simple/price api from here: https://www.coingecko.com/api/documentation + "coingecko_currency": "BTC", // default_exchange: a collection of settings that pertain to the default exchange // When the "show_market_dropdown_menu" setting is disabled, the market header menu will navigate directly to the default exchange page - // The default exchange is also used to determine the last market price + // The default_exchange.trading_pair is used to determine the last market price when the market_price value is set to AVERAGE // If left blank or filled out incorrectly, the first enabled exchange and trading pair will be used as the default exchange "default_exchange": { // exchange_name: The name of the default exchange must exactly match the name of an exchange in the "exchanges" setting above @@ -1263,6 +1276,20 @@ exports.sync = { // NOTE: You can add as many address labels as desired exports.labels = {}; +// default_coingecko_ids: a collection of settings that pertain to the list of coingecko api symbols and ids +// Adding entries to this section will force a particular coin symbol to the associated coingecko id when the coingecko api is used for USD lookups and when using the markets_page.market_price = "COINGECKO" option +// This is useful when there are multiple coins available in the coingecko api that have the same symbol, since by default, the explorer will match to the first one in the list which may not always be the correct coin +// Visit the coingecko coin list api to manually find the correct ids to plug in here for the symbols you use with the explorer: https://api.coingecko.com/api/v3/coins/list?include_platform=false +// You can add as many coingecko id defaults as necessary in the following format: [ { "symbol": "btc", "id": "bitcoin" }, { "symbol": "eth", "id": "ethereum" } ] +// NOTE: If all symbols that the explorer needs to look up via the coingecko coin list api are defaulted here, then the market sync will save an api call and skip making the call to the coingecko coin list api whenever it would usually be called +exports.default_coingecko_ids = [ + { "symbol": "btc", "id": "bitcoin" }, + { "symbol": "eth", "id": "ethereum" }, + { "symbol": "usdt", "id": "tether" }, + { "symbol": "ltc", "id": "litecoin" }, + { "symbol": "exor", "id": "exor" } +]; + //api_cmds: A collection of settings that pertain to the list of customizable rpc api commands // Not all blockchains utilize the same rpc cmds for accessing the internal daemon api. Use these settings to set alternate names for similar api cmds. // Leaving a cmd value blank ( "" ) will completely disable use of that cmd. diff --git a/scripts/sync.js b/scripts/sync.js index 48473b6..293e64d 100644 --- a/scripts/sync.js +++ b/scripts/sync.js @@ -1013,28 +1013,206 @@ function check_show_sync_message(blocks_to_sync) { return retVal; } -function get_last_usd_price() { - console.log('Calculating market price.. Please wait..'); +function get_market_price(market_array) { + // check how the market price should be updated + if (settings.markets_page.market_price == 'COINGECKO') { + // find the coingecko id + find_coingecko_id(settings.coin.symbol, function(coingecko_id) { + // check if the coingecko_id was found + if (coingecko_id != null && coingecko_id != '') { + const coingecko = require('../lib/apis/coingecko'); + const currency = lib.get_market_currency_code(); - // get the last usd price for coinstats - db.get_last_usd_price(function(err) { - // check for errors - if (err == null) { - // update markets_last_updated value - db.update_last_updated_stats(settings.coin.name, { markets_last_updated: Math.floor(new Date() / 1000) }, function(cb) { - // check if the script stopped prematurely - if (stopSync) { - console.log('Market sync was stopped prematurely'); - exit(1); - } else { - console.log('Market sync complete'); - exit(0); + console.log('Calculating market price.. Please wait..'); + + // get the market price from coingecko api + coingecko.get_market_prices(coingecko_id, currency, function (err, last_price, last_usd_price) { + // check for errors + if (err == null) { + // get current stats + Stats.findOne({coin: settings.coin.name}).then((stats) => { + // update market stat prices + Stats.updateOne({coin: settings.coin.name}, { + last_price: (last_price == null ? 0 : last_price), + last_usd_price: (last_usd_price == null ? 0 : last_usd_price) + }).then(() => { + // market prices updated successfully + finish_market_sync(); + }).catch((err) => { + // error saving stats + console.log(err); + exit(1); + }); + }).catch((err) => { + // error getting stats + console.log(err); + exit(1); + }); + } else { + // coingecko api returned an error + console.log(err); + exit(1); + } + }); + } else { + // coingecko_id is not set which should have already thrown an error, so just exit + exit(1); + } + }); + } else { + console.log('Calculating market price.. Please wait..'); + + // get the list of coins from coingecko + coingecko_coin_list_api(market_array, function (coin_err, coin_list) { + // check for errors + if (coin_err == null) { + let api_ids = ''; + + // loop through all unique currencies in the market_array + for (let m = 0; m < market_array.length; m++) { + const index = coin_list.findIndex(p => p.symbol.toLowerCase() == market_array[m].currency.toLowerCase()); + + // check if the market currency is found in the coin list + if (index > -1) { + // add to the list of api_ids + api_ids += (api_ids == '' ? '' : ',') + coin_list[index].id; + + // add the coingecko id back to the market_array + market_array[m].coingecko_id = coin_list[index].id; + } else { + // coin symbol not found in the api + console.log('Error: Cannot find symbol "' + market_array[m].currency + '" in the coingecko api'); + } } + + // check if any api_ids were found + if (api_ids != '') { + const coingecko = require('../lib/apis/coingecko'); + const currency = lib.get_market_currency_code(); + + // get the market price from coingecko api + coingecko.get_avg_market_prices(api_ids, currency, market_array, function (mkt_err, last_price, last_usd) { + // check for errors + if (mkt_err == null) { + // update the last usd price + Stats.updateOne({coin: settings.coin.name}, { + last_price: last_price, + last_usd_price: last_usd + }).then(() => { + // market price updated successfully + finish_market_sync(); + }).catch((err) => { + // error saving stat data + console.log(err); + exit(1); + }); + } else { + // coingecko api returned an error + console.log(mkt_err); + exit(1); + } + }); + } else { + // no api_ids found so cannot continue to getting the usd price and error msgs were already thrown, so just exit + exit(1); + } + } else { + // coingecko api returned an error + console.log(coin_err); + exit(1); + } + }); + } +} + +function finish_market_sync() { + // update markets_last_updated value + db.update_last_updated_stats(settings.coin.name, { markets_last_updated: Math.floor(new Date() / 1000) }, function() { + // check if the script stopped prematurely + if (stopSync) { + console.log('Market sync was stopped prematurely'); + exit(1); + } else { + console.log('Market sync complete'); + exit(0); + } + }); +} + +function coingecko_coin_list_api(market_symbols, cb) { + let coin_array = []; + let call_coin_list_api = false; + + // check if market_symbols is an array + if (!Array.isArray(market_symbols)) { + // add this symbol to an array + market_symbols = [{currency: market_symbols}]; + } + + // loop through all symbols + for (var symbol of market_symbols) { + // check if this symbol has a default coingecko id in the settings + const index = settings.default_coingecko_ids.findIndex(p => p.symbol.toLowerCase() == symbol.currency.toLowerCase()); + + // check if the coin symbol is found in settings + if (index > -1) { + // add this symbol and id to a new array + coin_array.push({ + id: settings.default_coingecko_ids[index].id.toLowerCase(), + symbol: symbol.currency.toLowerCase() }); } else { - // display error msg - console.log('Error: %s', err); - exit(1); + // missing at least 1 symbol, so the coingecko api must be called + call_coin_list_api = true; + break; + } + } + + // check if the coin list api needs to be called + if (call_coin_list_api) { + const coingecko = require('../lib/apis/coingecko'); + + // get the list of coins from coingecko + coingecko.get_coin_data(function (err, coin_list) { + // check if there was an error + if (err == null) { + // initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis + const rateLimitLib = require('../lib/ratelimit'); + const rateLimit = new rateLimitLib.RateLimit(1, 2000, false); + + // automatically pause for 2 seconds in between requests + rateLimit.schedule(function() { + return cb(err, coin_list); + }); + } else { + return cb(err, coin_list); + } + }); + } else { + // return the custom array of known symbols and ids + return cb(null, coin_array); + } +} + +function find_coingecko_id(symbol, cb) { + coingecko_coin_list_api(symbol, function (err, coin_list) { + // check for errors + if (err == null) { + // find the index of the first coin symbol match + const index = coin_list.findIndex(p => p.symbol.toLowerCase() == symbol.toLowerCase()); + + // check if the coin symbol is found in the api coin list + if (index > -1) + return cb(coin_list[index].id); + else { + // coin symbol not found in the api + console.log('Error: Cannot find symbol "' + symbol + '" in the coingecko api'); + return cb(''); + } + } else { + // failed to get the coingecko api list + console.log(err); + return cb(''); } }); } @@ -1514,15 +1692,16 @@ if (lib.is_locked([database]) == false) { } }); } else { - // check if market feature is enabled - if (settings.markets_page.enabled == true) { + // start market sync + // check if market feature is enabled or the market_price option is set to COINGECKO + if (settings.markets_page.enabled == true || settings.markets_page.market_price == 'COINGECKO') { var total_pairs = 0; var exchanges = Object.keys(settings.markets_page.exchanges); // loop through all exchanges to determine how many trading pairs must be updated exchanges.forEach(function(key, index, map) { // check if market is enabled via settings - if (settings.markets_page.exchanges[key].enabled == true) { + if (settings.markets_page.enabled == true && settings.markets_page.exchanges[key].enabled == true) { // check if market is installed/supported if (db.fs.existsSync('./lib/markets/' + key + '.js')) { // add trading pairs to total @@ -1539,6 +1718,8 @@ if (lib.is_locked([database]) == false) { // check if there are any trading pairs to update if (total_pairs > 0) { + let market_array = []; + // initialize the rate limiter to wait 2 seconds between requests to prevent abusing external apis var rateLimitLib = require('../lib/ratelimit'); var rateLimit = new rateLimitLib.RateLimit(1, 2000, false); @@ -1563,16 +1744,30 @@ if (lib.is_locked([database]) == false) { // automatically pause for 2 seconds in between requests rateLimit.schedule(function() { // update market data - db.update_markets_db(key, split_pair[0], split_pair[1], function(err) { - if (!err) + db.update_markets_db(key, split_pair[0], split_pair[1], function(err, last_price) { + if (!err) { console.log('%s[%s]: Market data updated successfully', key, pair_key); - else + + // only add to the market_array if market data is being averaged + if (settings.markets_page.market_price == 'AVERAGE') { + // check if the currency already exists in the market array + const index = market_array.findIndex(item => item.currency.toUpperCase() == split_pair[1].toUpperCase()); + + if (index != -1) { + // update the last_price + market_array[index].last_price = (market_array[index].last_price + last_price) / 2; + } else { + // add new object to the array + market_array.push({currency: split_pair[1], last_price: last_price}); + } + } + } else console.log('%s[%s] Error: %s', key, pair_key, err); complete++; if (complete == total_pairs || stopSync) - get_last_usd_price(); + get_market_price(market_array); }); }); } else { @@ -1580,7 +1775,7 @@ if (lib.is_locked([database]) == false) { complete++; if (complete == total_pairs || stopSync) - get_last_usd_price(); + get_market_price(market_array); } }); } else { @@ -1589,7 +1784,7 @@ if (lib.is_locked([database]) == false) { complete++; if (complete == total_pairs || stopSync) - get_last_usd_price(); + get_market_price(market_array); } }); } else { @@ -1598,11 +1793,13 @@ if (lib.is_locked([database]) == false) { complete++; if (complete == total_pairs || stopSync) - get_last_usd_price(); + get_market_price(market_array); } } }); - } else { + } else if (settings.markets_page.market_price == 'COINGECKO') + get_market_price([]); + else { // no market trading pairs are enabled console.log('Error: No market trading pairs are enabled in settings'); exit(1); diff --git a/settings.json.template b/settings.json.template index 33e1120..0997a25 100644 --- a/settings.json.template +++ b/settings.json.template @@ -232,7 +232,7 @@ // The panel will be disabled with a value of 0 "display_order": 1 }, - // price_panel: a collection of settings that pertain to the price panel which displays the current market price measured against the default market pair + // price_panel: a collection of settings that pertain to the price panel which displays the current market price as determined by the markets_page.market_price value "price_panel": { // enabled: Enable/disable the price panel (true/false) // If set to false, the price panel will be completely inaccessible @@ -252,7 +252,7 @@ // The panel will be disabled with a value of 0 "display_order": 4 }, - // market_cap_panel: a collection of settings that pertain to the market cap panel which displays the current market cap value measured against the default market pair + // market_cap_panel: a collection of settings that pertain to the market cap panel which displays the current market cap value as determined by the markets_page.market_price value "market_cap_panel": { // enabled: Enable/disable the market cap panel (true/false) // If set to false, the market cap panel will be completely inaccessible @@ -1001,9 +1001,22 @@ "trading_pairs": [ "LTC/BTC" ] } }, + // market_price: Determine how to calculate the market price + // NOTE: The market price is always retrieved at the end of the market sync process + // Valid options: + // AVERAGE : Market price is calculated based on averaging all supported exchange trading pairs that are enabled in the explorer and measured in the default_exchange.trading_pair value + // COINGECKO : Market price is retrieved directly from the coingecko api. This option is somewhat special in that it does not require the markets_page option or any supported markets to be set up or enabled to function correctly + "market_price": "AVERAGE", + // coingecko_currency: Determine the default cryptocurrency value to measure your coin against when using the COINGECKO market_price option + // NOTE: This value is only necessary to fill out if you set market_price = "COINGECKO". + // This can be any cryptocurrency symbol value that the coingecko api supports such as BTC or ETH for example. + // Although the coingecko api supports multiple different currencies, the explorer only supports 1 cryptocurrency market price and therefore specifying multiple currencies separated by commas will not work here. + // The USD fiat currency is also built-in and automatically returned from the coingecko api so there is no need to specify USD here. + // For more information about which currency values are supported, you can review the vs_currencies parameter of the /simple/price api from here: https://www.coingecko.com/api/documentation + "coingecko_currency": "BTC", // default_exchange: a collection of settings that pertain to the default exchange // When the "show_market_dropdown_menu" setting is disabled, the market header menu will navigate directly to the default exchange page - // The default exchange is also used to determine the last market price + // The default_exchange.trading_pair is used to determine the last market price when the market_price value is set to AVERAGE // If left blank or filled out incorrectly, the first enabled exchange and trading pair will be used as the default exchange "default_exchange": { // exchange_name: The name of the default exchange must exactly match the name of an exchange in the "exchanges" setting above @@ -1380,6 +1393,20 @@ } }, + // default_coingecko_ids: a collection of settings that pertain to the list of coingecko api symbols and ids + // Adding entries to this section will force a particular coin symbol to the associated coingecko id when the coingecko api is used for USD lookups and when using the markets_page.market_price = "COINGECKO" option + // This is useful when there are multiple coins available in the coingecko api that have the same symbol, since by default, the explorer will match to the first one in the list which may not always be the correct coin + // Visit the coingecko coin list api to manually find the correct ids to plug in here for the symbols you use with the explorer: https://api.coingecko.com/api/v3/coins/list?include_platform=false + // You can add as many coingecko id defaults as necessary in the following format: [ { "symbol": "btc", "id": "bitcoin" }, { "symbol": "eth", "id": "ethereum" } ] + // NOTE: If all symbols that the explorer needs to look up via the coingecko coin list api are defaulted here, then the market sync will save an api call and skip making the call to the coingecko coin list api whenever it would usually be called + "default_coingecko_ids": [ + { "symbol": "btc", "id": "bitcoin" }, + { "symbol": "eth", "id": "ethereum" }, + { "symbol": "usdt", "id": "tether" }, + { "symbol": "ltc", "id": "litecoin" }, + { "symbol": "exor", "id": "exor" } + ], + //api_cmds: A collection of settings that pertain to the list of customizable rpc api commands // Not all blockchains utilize the same rpc cmds for accessing the internal daemon api. Use these settings to set alternate names for similar api cmds. // Leaving a cmd value blank ( "" ) will completely disable use of that cmd. diff --git a/views/info.pug b/views/info.pug index 99e9b8a..11e4c97 100644 --- a/views/info.pug +++ b/views/info.pug @@ -31,6 +31,7 @@ block content hr - var hide_rpc_api_section = !(settings.api_page.public_apis.rpc.getdifficulty.enabled == true && settings.api_cmds['getdifficulty'] != null && settings.api_cmds['getdifficulty'] != '') && !(settings.api_page.public_apis.rpc.getconnectioncount.enabled == true && settings.api_cmds['getconnectioncount'] != null && settings.api_cmds['getconnectioncount'] != '') && !(settings.api_page.public_apis.rpc.getblockcount.enabled == true && settings.api_cmds['getblockcount'] != null && settings.api_cmds['getblockcount'] != '') && !(settings.api_page.public_apis.rpc.getblockhash.enabled == true && settings.api_cmds['getblockhash'] != null && settings.api_cmds['getblockhash'] != '') && !(settings.api_page.public_apis.rpc.getblock.enabled == true && settings.api_cmds['getblock'] != null && settings.api_cmds['getblock'] != '') && !(settings.api_page.public_apis.rpc.getrawtransaction.enabled == true && settings.api_cmds['getrawtransaction'] != null && settings.api_cmds['getrawtransaction'] != '') && !(settings.api_page.public_apis.rpc.getnetworkhashps.enabled == true && settings.shared_pages.show_hashrate == true && settings.api_cmds['getnetworkhashps'] != null && settings.api_cmds['getnetworkhashps'] != '') && !(settings.api_page.public_apis.rpc.getvotelist.enabled == true && settings.api_cmds['getvotelist'] != null && settings.api_cmds['getvotelist'] != '') && !(settings.api_page.public_apis.rpc.getmasternodecount.enabled == true && settings.api_cmds['getmasternodecount'] != null && settings.api_cmds['getmasternodecount'] != '') && (!settings.blockchain_specific.heavycoin.enabled || (!(settings.blockchain_specific.heavycoin.public_apis.getmaxmoney.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getmaxmoney'] != null && settings.blockchain_specific.heavycoin.api_cmds['getmaxmoney'] != '') && !(settings.blockchain_specific.heavycoin.public_apis.getmaxvote.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getmaxvote'] != null && settings.blockchain_specific.heavycoin.api_cmds['getmaxvote'] != '') && !(settings.blockchain_specific.heavycoin.public_apis.getvote.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getvote'] != null && settings.blockchain_specific.heavycoin.api_cmds['getvote'] != '') && !(settings.blockchain_specific.heavycoin.public_apis.getphase.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getphase'] != null && settings.blockchain_specific.heavycoin.api_cmds['getphase'] != '') && !(settings.blockchain_specific.heavycoin.public_apis.getreward.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getreward'] != null && settings.blockchain_specific.heavycoin.api_cmds['getreward'] != '') && !(settings.blockchain_specific.heavycoin.public_apis.getsupply.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getsupply'] != null && settings.blockchain_specific.heavycoin.api_cmds['getsupply'] != '') && !(settings.blockchain_specific.heavycoin.public_apis.getnextrewardestimate.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getnextrewardestimate'] != null && settings.blockchain_specific.heavycoin.api_cmds['getnextrewardestimate'] != '') && !(settings.blockchain_specific.heavycoin.public_apis.getnextrewardwhenstr.enabled == true && settings.blockchain_specific.heavycoin.api_cmds['getnextrewardwhenstr'] != null && settings.blockchain_specific.heavycoin.api_cmds['getnextrewardwhenstr'] != ''))); - var hide_ext_api_section = !settings.api_page.public_apis.ext.getmoneysupply.enabled && !settings.api_page.public_apis.ext.getdistribution.enabled && !settings.api_page.public_apis.ext.getaddress.enabled && !settings.api_page.public_apis.ext.getaddresstxs.enabled && !settings.api_page.public_apis.ext.gettx.enabled && !settings.api_page.public_apis.ext.getbalance.enabled && !settings.api_page.public_apis.ext.getlasttxs.enabled && !settings.api_page.public_apis.ext.getcurrentprice.enabled && !settings.api_page.public_apis.ext.getnetworkpeers.enabled && !settings.api_page.public_apis.ext.getbasicstats.enabled && !settings.api_page.public_apis.ext.getsummary.enabled && !(settings.api_page.public_apis.ext.getmasternodelist.enabled && settings.api_cmds['getmasternodelist'] != null && settings.api_cmds['getmasternodelist'] != '') && !settings.api_page.public_apis.ext.getmasternoderewards.enabled && !settings.api_page.public_apis.ext.getmasternoderewardstotal.enabled; + - var market_currency = (settings.markets_page.market_price == 'COINGECKO' ? (settings.markets_page.coingecko_currency == null || settings.markets_page.coingecko_currency == '' ? '' : settings.markets_page.coingecko_currency) : (settings.markets_page.default_exchange.trading_pair != null && settings.markets_page.default_exchange.trading_pair.indexOf('/') > -1 ? settings.markets_page.default_exchange.trading_pair.split('/')[1] : '')).toUpperCase(); if !hide_rpc_api_section h3 #{settings.locale.api_calls} p @@ -236,14 +237,14 @@ block content p div.fw-bold getbasicstats div - em="Returns basic statistics about the coin including: block count, circulating supply, USD price, " + settings.markets_page.default_exchange.trading_pair.split('/')[1].toUpperCase() + " price" + (settings.api_page.public_apis.rpc.getmasternodecount.enabled == true && settings.api_cmds['getmasternodecount'] != null && settings.api_cmds['getmasternodecount'] != '' ? ', ' + '# of masternodes' : '') + em="Returns basic statistics about the coin including: block count, circulating supply, USD price, " + market_currency + " price" + (settings.api_page.public_apis.rpc.getmasternodecount.enabled == true && settings.api_cmds['getmasternodecount'] != null && settings.api_cmds['getmasternodecount'] != '' ? ', ' + '# of masternodes' : '') a(href='/ext/getbasicstats') #{address}/ext/getbasicstats if settings.api_page.public_apis.ext.getsummary.enabled == true li p div.fw-bold getsummary div - em="Returns a summary of coin data including: difficulty, hybrid difficulty, circulating supply, hash rate, " + settings.markets_page.default_exchange.trading_pair.split('/')[1].toUpperCase() + " price, USD price, network connection count, block count" + (settings.api_page.public_apis.rpc.getmasternodecount.enabled == true && settings.api_cmds['getmasternodecount'] != null && settings.api_cmds['getmasternodecount'] != '' ? ', ' + 'count of online masternodes' + ', ' + 'count of offline masternodes' : '') + em="Returns a summary of coin data including: difficulty, hybrid difficulty, circulating supply, hash rate, " + market_currency + " price, USD price, network connection count, block count" + (settings.api_page.public_apis.rpc.getmasternodecount.enabled == true && settings.api_cmds['getmasternodecount'] != null && settings.api_cmds['getmasternodecount'] != '' ? ', ' + 'count of online masternodes' + ', ' + 'count of offline masternodes' : '') a(href='/ext/getsummary') #{address}/ext/getsummary if settings.api_page.public_apis.ext.getmasternodelist.enabled == true && settings.api_cmds['getmasternodelist'] != null && settings.api_cmds['getmasternodelist'] != '' li diff --git a/views/layout.pug b/views/layout.pug index 6b74035..c1e105f 100644 --- a/views/layout.pug +++ b/views/layout.pug @@ -143,6 +143,7 @@ html(lang='en') - sideBarClasses.push('bg-primary'); - sideBarClasses.push('navbar-dark'); - var showNetworkPanel = false + - var market_currency = (settings.markets_page.market_price == 'COINGECKO' ? (settings.markets_page.coingecko_currency == null || settings.markets_page.coingecko_currency == '' ? '' : settings.markets_page.coingecko_currency) : (settings.markets_page.default_exchange.trading_pair != null && settings.markets_page.default_exchange.trading_pair.indexOf('/') > -1 ? settings.markets_page.default_exchange.trading_pair.split('/')[1] : '')).toUpperCase(); if settings.panel1 == 'network_panel' || settings.panel2 == 'network_panel' || settings.panel3 == 'network_panel' || settings.panel4 == 'network_panel' || settings.panel5 == 'network_panel' - showNetworkPanel = true script. @@ -306,13 +307,13 @@ html(lang='en') return `
#{settings.locale.ex_supply} (${"#{settings.coin.symbol}".replace(/"/g, '"')})
`; } function getPricePanel() { - return '
Price (#{settings.markets_page.default_exchange.trading_pair.split('/')[1]})
'; + return '
Price (#{market_currency})
'; } function getUSDPricePanel() { return '
Price (USD)
'; } function getMarketCapPanel() { - return '
Market Cap (#{settings.markets_page.default_exchange.trading_pair.split('/')[1]})
'; + return '
Market Cap (#{market_currency})
'; } function getUSDMarketCapPanel() { return '
Market Cap (USD)
';