Merge pull request #15 from Adamant-im/dev

v3.5.0: Add ExchangeRate.host source

.gitignore

@@ -1,7 +1,7 @@
 node_modules/
 logs/
 .vscode/
+.idea/
 config.test
 config.test2
-package-lock.json
 tests.js

README.md

@@ -1,9 +1,10 @@
-ADAMANT InfoServices is a crypto and fiat currency rates service provider. It collects rates from MOEX, Currency-Api, Coinmarketcap, CryptoCompare and Coingecko and calculates cross-rates, and provides information via API.
+ADAMANT InfoServices is a crypto and fiat currency rates service provider. It collects rates from MOEX, Currency-Api, ExchangeRate, Coinmarketcap, CryptoCompare and Coingecko and calculates cross-rates, and provides information via API.
 
 Features:
 
 - Collects rates from MOEX for fiat tickers
 - Collects rates from Currency-Api for fiat tickers
+- Collects rates from ExchangeRate (Currency-Api2) for fiat tickers
 - Collects rates from Coinmarketcap for crypto tickers
 - Collects rates from CryptoCompare for crypto tickers
 - Collects rates from Coingecko for crypto tickers
@@ -21,8 +22,8 @@ Features:
 
 ## Requirements
 
-- Ubuntu 18 / 20 (we didn't test with others)
-- NodeJS v12+
+- Ubuntu 18+ (we didn't test with others)
+- NodeJS v16+
 - MongoDB ([installation instructions](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/))
 
 ## Setup
@@ -43,9 +44,10 @@ nano config.json
 Parameters:
 
 - `crypto_cmc` <array> List of coins to fetch rates from Coinmarketcap
+- `crypto_cmc_coinids` <object> List of Coinmarketcap Symbol-Id pairs like { "ADM": 3703 } to fetch rates from Coinmarketcap. Used when one coin symbol is used for different coins. Coin ids can be seen via /cryptocurrency/quotes/latest?slug=coinname request.
 - `crypto_cc` <array> List of coins to fetch rates from Cryptocompare
 - `crypto_cg` <array> List of coins to fetch rates from Coingecko. Better use `crypto_cg_coinids`.
-- `crypto_cg_coinids` <array> List of Coingecko coin Ids to fetch rates from Coingecko. Used when one coin symbol is used for different coins. Coin ids can be seen on https://api.coingecko.com/api/v3/coins/list.
+- `crypto_cg_coinids` <string, array> List of Coingecko coin Ids to fetch rates from Coingecko. Used when one coin symbol is used for different coins. Coin ids can be seen on https://api.coingecko.com/api/v3/coins/list.
 - `fiat` <object> List of fiat pairs and their codes to fetch from MOEX
 - `baseCoins` <array> List of coins to calculate all available pairs using `crypto` and `fiat`
 - `cmcApiKey` <string> Coinmarketcap API key. You must get yours at https://coinmarketcap.com/api/.

app.js

@@ -1,5 +1,6 @@
 const Moex = require('./helpers/getMoex');
-const CurrencyApi = require('./helpers/getCurrencyApi');
+const CurrencyApi1 = require('./helpers/getCurrencyApi1');
+const CurrencyApi2 = require('./helpers/getCurrencyApi2');
 const Cmc = require('./helpers/getCmc');
 const Cc = require('./helpers/getCc');
 const Cg = require('./helpers/getCg');
@@ -13,17 +14,16 @@ let fetchedAll;
 
 let tickers = {};
 let tickersInfo;
-const RATE_DIFFERENCE_PERCENT_THRESHOLD = 25;
 
 function refresh() {
 
   log.log('Updating rates…');
   fetchedAll = true;
   tickers = {};
 
-  CurrencyApi((data) => {
+  CurrencyApi1((data) => {
     if (data) {
-      tickersInfo = mergeData({}, data, 'Null', 'CurrencyApi');
+      tickersInfo = mergeData({}, data, 'Null', 'CurrencyApi1');
       if (tickersInfo.isAlert) {
         notify(`Error: rates from different sources significantly differs: ${tickersInfo.alertString}. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
         fetchedAll = false;
@@ -32,12 +32,27 @@ function refresh() {
       }
     } else {
       fetchedAll = false;
-      notify(`Error: Unable to get data from CurrencyApi. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
+      notify(`Error: Unable to get data from CurrencyApi1. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
     }
 
+    CurrencyApi2((data) => {
+      if (data) {
+        tickersInfo = mergeData({}, data, 'CurrencyApi1', 'CurrencyApi2');
+        if (tickersInfo.isAlert) {
+          notify(`Error: rates from different sources significantly differs: ${tickersInfo.alertString}. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
+          fetchedAll = false;
+        } else {
+          tickers = tickersInfo.merged;
+        }
+      } else {
+        fetchedAll = false;
+        notify(`Error: Unable to get data from CurrencyApi2. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
+      }
+    });
+
     Cmc('USD', (data) => {
       if (data) {
-        tickersInfo = mergeData(tickers, data, 'CurrencyApi', 'Cmc');
+        tickersInfo = mergeData(tickers, data, 'CurrencyApi1+CurrencyApi2', 'Cmc');
         if (tickersInfo.isAlert) {
           notify(`Error: rates from different sources significantly differs: ${tickersInfo.alertString}. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
           fetchedAll = false;
@@ -51,7 +66,7 @@ function refresh() {
 
       Moex((data) => {
         if (data) {
-          tickersInfo = mergeData(tickers, data, 'CurrencyApi+Cmc', 'Moex');
+          tickersInfo = mergeData(tickers, data, 'CurrencyApi1+CurrencyApi2+Cmc', 'Moex');
           if (tickersInfo.isAlert) {
             notify(`Error: rates from different sources significantly differs: ${tickersInfo.alertString}. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
             fetchedAll = false;
@@ -65,7 +80,7 @@ function refresh() {
 
         Cc('USD', (data) => {
           if (data) {
-            tickersInfo = mergeData(tickers, data, 'CurrencyApi+Cmc+Moex', 'Cc');
+            tickersInfo = mergeData(tickers, data, 'CurrencyApi1+CurrencyApi2+Cmc+Moex', 'Cc');
             if (tickersInfo.isAlert) {
               notify(`Error: rates from different sources significantly differs: ${tickersInfo.alertString}. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
               fetchedAll = false;
@@ -79,7 +94,7 @@ function refresh() {
 
           Cg('USD', (data) => {
             if (data) {
-              tickersInfo = mergeData(tickers, data, 'CurrencyApi+Cmc+Moex+Cc', 'Cg');
+              tickersInfo = mergeData(tickers, data, 'CurrencyApi1+CurrencyApi2+Cmc+Moex+Cc', 'Cg');
               if (tickersInfo.isAlert) {
                 notify(`Error: rates from different sources significantly differs: ${tickersInfo.alertString}. InfoService will provide previous rates; historical rates wouldn't be saved.`, 'error');
                 fetchedAll = false;
@@ -105,7 +120,7 @@ function refresh() {
         }); // Cryptocompare
       }); // Moex
     }); // Coinmarketcap
-  }); // CurrencyApi
+  }); // CurrencyApi1
 
 } // refresh
 
@@ -126,7 +141,7 @@ function mergeData(tickers1, tickers2, source1, source2) {
   Object.keys(merged).forEach((m) => {
     if (utils.isPositiveOrZeroNumber(tickers1[m]) && utils.isPositiveOrZeroNumber(tickers2[m])) {
       const diff = utils.numbersDifferencePercent(tickers1[m], tickers2[m]);
-      if (diff > RATE_DIFFERENCE_PERCENT_THRESHOLD) {
+      if (diff > config.rateDifferencePercentThreshold) {
         alertTickers.push(`**${m}** ${diff.toFixed(0)}%: ${+tickers1[m].toFixed(8)} (${source1}) — ${+tickers2[m].toFixed(8)} (${source2})`);
       }
     }

config.json

@@ -1,19 +1,21 @@
 {
   "crypto_cmc": ["BTC", "ETH", "DASH", "LSK", "DOGE", "BCH", "USDT", "LTC", "TRX", "ADM",
     "BSV", "BNB", "XMR", "XEM", "NEO", "ETC", "USDC", "ZEC", "TUSD", "KCS", "USDS"],
+  "crypto_cmc_coinids": [],
   "crypto_cc": ["BTC", "ETH", "DASH", "LSK", "DOGE", "BCH", "USDT", "LTC", "TRX", "ADM",
     "BSV", "BNB", "XMR", "XEM", "NEO", "ETC", "USDC", "ZEC", "TUSD", "KCS"],
   "crypto_cg": [],
   "crypto_cg_coinids": ["bitcoin", "ethereum", "dash", "lisk", "dogecoin", "tether", "litecoin", "adamant-messenger",
 		"binancecoin", "ethereum-classic", "usd-coin", "true-usd", "stableusd", "resfinex-token"],
-  "baseCoins": ["USD", "RUB", "EUR", "CNY", "JPY", "BTC", "ETH"],
+  "baseCoins": ["USD", "RUB", "EUR", "CNY", "JPY", "BTC", "ETH", "KRW"],
   "fiat": {
     "USD/RUB": "USDRUB_TOM",
     "EUR/RUB": "EURRUB_TOM",
     "GBR/RUB": "GBPRUB_TOM",
     "CNY/RUB": "CNYRUB_TOM",
-    "JPY/RUB": "JPYRUB_TOD"
+    "JPY/RUB": "JPYRUB_TOM"
   },
+  "rateDifferencePercentThreshold": 25,
   "cmcApiKey": "Put yours Coinmarketcap API key",
   "ccApiKey": "Put yours CryptoCompare API key",
   "cgApiKey": "No need for Coingecko API key",

helpers/configReader.js

@@ -11,13 +11,13 @@ try {
     config = JSON.parse(jsonminify(fs.readFileSync('./config.json', 'utf-8')));
   }
 
-  config.isCc = config.crypto_cc && config.crypto_cc.length !== 0 && config.ccApiKey;
-  config.isCg = (config.crypto_cg && config.crypto_cg.length !== 0) ||
-    (config.crypto_cg_coinids && config.crypto_cg_coinids.length !== 0);
-  config.isCmc = config.crypto_cmc && config.crypto_cmc.length !== 0 && config.cmcApiKey;
+  config.isCc = config.crypto_cc?.length && config.ccApiKey;
+  config.isCg = (config.crypto_cg?.length) || (config.crypto_cg_coinids?.length);
+  config.isCmc = (config.crypto_cmc?.length || config.crypto_cmc_coinids?.length) && config.cmcApiKey;
   config.version = require('../package.json').version;
   config.isDev = isDev;
-  config.crypto_all = config.crypto_cmc.concat(config.crypto_cc); // Also, Coingecko coins will be added in getCg module
+  // Also, Coingecko coins will be added in getCg module and Coinmarketcap in getCmc
+  config.crypto_all = [].concat(config.crypto_cc);
   console.info(`InfoService successfully read a config-file${isDev ? ' (dev)' : ''}.`);
 
 } catch (e) {

helpers/getCg.js

@@ -35,6 +35,7 @@ function getCgCoinIds() {
             config.crypto_cg_full.push(cg_crypto);
           });
           config.crypto_all = config.crypto_all.concat(config.crypto_cg_full.map((e) => e.symbol));
+          config.crypto_all = [...new Set(config.crypto_all)]; // Remove duplicates
           config.isCgFull = true;
           log.log(`Coingecko coin ids fetched successfully`);
         } catch (e) {

helpers/getCmc.js

@@ -4,16 +4,88 @@ const log = require('./log');
 const notify = require('./notify');
 const _ = require('underscore');
 
+/**
+ * https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyQuotesLatest
+ * id: One or more comma-separated cryptocurrency CoinMarketCap IDs. Example: 1,2
+ * slug: Alternatively pass a comma-separated list of cryptocurrency slugs. Example: "bitcoin,ethereum"
+ * symbol: Alternatively pass one or more comma-separated cryptocurrency symbols.
+ *    Example: "BTC,ETH". At least one "id" or "slug" or "symbol" is required for this request.
+ * convert: Optionally calculate market quotes in up to 120 currencies at once by passing a comma-separated
+ *    list of cryptocurrency or fiat currency symbols. Each additional convert option beyond the first
+ *    requires an additional call credit. A list of supported fiat options can be found here. Each
+ *    conversion is returned in its own "quote" object.
+ * convert_id: Optionally calculate market quotes by CoinMarketCap ID instead of symbol. This option is
+ *    identical to convert outside of ID format. Ex: convert_id=1,2781 would replace convert=BTC,USD in
+ *    your query. This parameter cannot be used when convert is used.
+ * Note: find id on a coin's webpage with "coinId":1027", "200x200/1027.png"
+ * Note: find slug in a coin's URL like https://coinmarketcap.com/currencies/bitcoin/
+ */
 const url_base = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest';
 
+function getCmcCoinIds() {
+
+  config.crypto_cmc_full = [];
+  if (!config.isCmc) return;
+
+  const url = url_base + '?' +
+    'symbol=' + config.crypto_cmc.join();
+  const httpOptions = {
+    url,
+    method: 'get',
+    timeout: 10000,
+    headers: {
+      'X-CMC_PRO_API_KEY': config.cmcApiKey,
+    },
+  };
+
+  axios(httpOptions)
+      .then(function(response) {
+        try {
+          const data = response.data.data;
+          config.crypto_cmc.forEach((ticker) => {
+            const currency = _.findWhere(data, {
+              symbol: ticker.toUpperCase(),
+            });
+            if (currency) {
+              cmc_crypto = {
+                symbol: ticker,
+                cmc_id: currency.id,
+              };
+              config.crypto_cmc_full.push(cmc_crypto);
+            }
+          });
+          Object.keys(config.crypto_cmc_coinids).forEach((ticker) => {
+            cmc_crypto = {
+              symbol: ticker.toUpperCase(),
+              cmc_id: config.crypto_cmc_coinids[ticker],
+            };
+            config.crypto_cmc_full.push(cmc_crypto);
+          });
+          config.crypto_all = config.crypto_all.concat(config.crypto_cmc_full.map((crypto) => crypto.symbol));
+          config.crypto_all = [...new Set(config.crypto_all)]; // Remove duplicates
+          config.isCmcFull = true;
+          log.log(`Coinmarketcap coin ids fetched successfully`);
+        } catch (e) {
+          notify(`Unable to process data ${JSON.stringify(response.data)} from request to ${url}. Unable to get Coinmarketcap coin ids. Try to restart InfoService or there will be no rates from Coinmarketcap. Error: ${e}`, 'error');
+        }
+      })
+      .catch(function(error) {
+        notify(`Request to ${url} failed with ${error.response?.status} status code, ${error.toString()}. Unable to get Coinmarketcap coin ids. Try to restart InfoService or there will be no rates from Coinmarketcap.`, 'error');
+      });
+}
+
+getCmcCoinIds();
+
 module.exports = (base, cb) => {
 
-  if (!config.isCmc) {
+  if (!config.isCmcFull) {
     cb({});
     return;
   }
 
-  const url = url_base + '?' + 'symbol=' + config.crypto_cmc.join() + '&convert=' + base;
+  const url = url_base + '?' +
+    'id=' + config.crypto_cmc_full.map((currency) => currency.cmc_id).join(',') +
+    '&convert=' + base;
   const httpOptions = {
     url,
     method: 'get',
@@ -28,13 +100,26 @@ module.exports = (base, cb) => {
         try {
           const data = response.data.data;
           const rates = {};
-          config.crypto_cmc.forEach((t) => {
+          const unavailableList = [];
+          config.crypto_cmc.forEach((ticker) => {
             const currency = _.findWhere(data, {
-              symbol: t,
+              symbol: ticker,
             });
-            rates[t + '/' + base] = +currency.quote[base].price.toFixed(8);
+            if (currency?.quote?.[base]?.price) {
+              rates[ticker + '/' + base] = +currency.quote[base].price.toFixed(8);
+            } else {
+              unavailableList.push(ticker);
+            }
           });
-          log.log(`Coinmarketcap rates updated against ${base} successfully`);
+          if (unavailableList.length) {
+            if (unavailableList.length === config.crypto_cmc.length) {
+              notify(`Unable to get all of ${config.crypto_cmc.length} coin rates from request to ${url}. Check Coinmarketcap service and config file.`, 'error');
+            } else {
+              log.warn(`Coinmarketcap rates updated against ${base} successfully, except ${unavailableList.join(', ')}`);
+            }
+          } else {
+            log.log(`Coinmarketcap rates updated against ${base} successfully`);
+          }
           cb(rates);
         } catch (e) {
           notify(`Unable to process data ${JSON.stringify(response.data)} from request to ${url}. Wrong Coinmarketcap API key? Error: ${e}`, 'error');

helpers/getCurrencyApi.js → helpers/getCurrencyApi1.js

@@ -22,7 +22,7 @@ module.exports = (cb) => {
               rates[currency.toUpperCase() + '/USD'] = +(1 / +rate).toFixed(8);
             }
           });
-          log.log(`Cryptocurrency-Api rates updated successfully`);
+          log.log(`Currency-Api1 rates updated successfully`);
           cb(rates);
         } catch (e) {
           notify(`Unable to process data ${JSON.stringify(response.data)} from request to ${url}. Error: ${e}`, 'error');

helpers/getCurrencyApi2.js

@@ -0,0 +1,38 @@
+const axios = require('axios');
+const config = require('./configReader');
+const log = require('./log');
+const notify = require('./notify');
+
+// https://github.com/Formicka/exchangerate.host
+const url = 'https://api.exchangerate.host/latest?base=USD';
+// This service doesn't include crypto except Bitcoin
+// Also, it sometimes provides chicken digits, like 47k usd for Bitcoin instead of 16k, and 73 rub/usd instead of 60
+// Good we have built-in check system
+const skipCoins = ['USD', 'ETH'];
+
+module.exports = (cb) => {
+
+  axios.get(url)
+      .then(function(response) {
+        try {
+          const rates = {};
+          const data = response.data.rates;
+          config.baseCoins.forEach((currency) => {
+            const rate = data[currency.toUpperCase()];
+            if (!skipCoins.includes(currency) && rate) {
+              rates['USD/' + currency.toUpperCase()] = +rate.toFixed(8);
+              rates[currency.toUpperCase() + '/USD'] = +(1 / +rate).toFixed(8);
+            }
+          });
+          log.log(`Currency-Api2 rates updated successfully`);
+          cb(rates);
+        } catch (e) {
+          notify(`Unable to process data ${JSON.stringify(response.data)} from request to ${url}. Error: ${e}`, 'error');
+          cb(false);
+        }
+      })
+      .catch(function(error) {
+        notify(`Request to ${url} failed with ${error.response?.status} status code, ${error.toString()}${error.response?.data ? '. Message: ' + JSON.stringify(error.response.data) : ''}.`, 'error');
+        cb(false);
+      });
+};

helpers/getMoex.js

@@ -4,7 +4,8 @@ const log = require('./log');
 const notify = require('./notify');
 const _ = require('underscore');
 
-const url = 'https://iss.moex.com/iss/engines/currency/markets/selt/securities.jsonp';
+// const url = 'https://iss.moex.com/iss/engines/currency/markets/selt/securities.jsonp';
+const url = 'https://rusdoor.adamant.im/securities.jsonp';
 
 module.exports = (cb) => {
 

modules/router.js

@@ -19,16 +19,16 @@ app.get('/get', (req, res) => {
     res.json(respSuccess(tickers));
   } else {
     coins = coins.toUpperCase();
-    const filterredTickers = {};
+    const filteredTickers = {};
     let arrCoins = [coins];
     if (~coins.indexOf(',')) {
       arrCoins = coins.split(',');
     }
     arrCoins.forEach((coin) => {
       const filteredMarkets = Object.keys(tickers).filter((c) => ~c.indexOf(coin.trim()));
-      filteredMarkets.forEach((c) => filterredTickers[c] = tickers[c]);
+      filteredMarkets.forEach((c) => filteredTickers[c] = tickers[c]);
     });
-    res.json(respSuccess(filterredTickers));
+    res.json(respSuccess(filteredTickers));
   }
 });