From 519494668ba1d677d1970fcb6f84da66cb585933 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 11 Jul 2022 17:52:38 +0200 Subject: [PATCH 1/2] Use maxmind to store node locations --- backend/mempool-config.sample.json | 5 ++ backend/package-lock.json | 50 +++++++++++++++ backend/package.json | 1 + backend/src/api/database-migration.ts | 23 ++++++- backend/src/api/explorer/nodes.api.ts | 11 ++++ backend/src/config.ts | 14 ++++- .../src/tasks/lightning/node-sync.service.ts | 5 ++ .../tasks/lightning/stats-updater.service.ts | 2 +- .../lightning/sync-tasks/node-locations.ts | 63 +++++++++++++++++++ 9 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 backend/src/tasks/lightning/sync-tasks/node-locations.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index b71f3586df..214121ed42 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -63,6 +63,11 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, + "MAXMIND": { + "ENABLED": false, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb", + "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb" + }, "BISQ": { "ENABLED": false, "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db" diff --git a/backend/package-lock.json b/backend/package-lock.json index 7ea9cf43ba..e724ac35bc 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,6 +17,7 @@ "crypto-js": "^4.0.0", "express": "^4.18.0", "lightning": "^5.16.3", + "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", "socks-proxy-agent": "~7.0.0", @@ -2222,6 +2223,19 @@ "node": ">=10" } }, + "node_modules/maxmind": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz", + "integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==", + "dependencies": { + "mmdb-lib": "2.0.2", + "tiny-lru": "8.0.2" + }, + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -2317,6 +2331,15 @@ "node": "*" } }, + "node_modules/mmdb-lib": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz", + "integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA==", + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -3027,6 +3050,14 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "node_modules/tiny-lru": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", + "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==", + "engines": { + "node": ">=6" + } + }, "node_modules/tiny-secp256k1": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", @@ -4971,6 +5002,15 @@ "yallist": "^4.0.0" } }, + "maxmind": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz", + "integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==", + "requires": { + "mmdb-lib": "2.0.2", + "tiny-lru": "8.0.2" + } + }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -5039,6 +5079,11 @@ "brace-expansion": "^1.1.7" } }, + "mmdb-lib": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz", + "integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA==" + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -5549,6 +5594,11 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "tiny-lru": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", + "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" + }, "tiny-secp256k1": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", diff --git a/backend/package.json b/backend/package.json index 5023d6029c..b8930d6e56 100644 --- a/backend/package.json +++ b/backend/package.json @@ -38,6 +38,7 @@ "crypto-js": "^4.0.0", "express": "^4.18.0", "lightning": "^5.16.3", + "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", "socks-proxy-agent": "~7.0.0", diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 4870c5d033..dbc7ce9256 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 28; + private static currentVersion = 29; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -280,6 +280,17 @@ class DatabaseMigration { await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`); } + if (databaseSchemaVersion < 29 && isBitcoin === true) { + await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names')); + await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL'); + } + } catch (e) { throw e; } @@ -693,6 +704,16 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateGeoNamesTableQuery(): string { + return `CREATE TABLE geo_names ( + id int(11) unsigned NOT NULL, + type enum('city','country','division','continent') NOT NULL, + names text DEFAULT NULL, + UNIQUE KEY id (id,type), + KEY id_2 (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 1bf9ce12d6..0c63d7e84b 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -13,6 +13,17 @@ class NodesApi { } } + public async $getAllNodes(): Promise { + try { + const query = `SELECT * FROM nodes`; + const [rows]: any = await DB.query(query); + return rows; + } catch (e) { + logger.err('$getAllNodes error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getNodeStats(public_key: string): Promise { try { const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`; diff --git a/backend/src/config.ts b/backend/src/config.ts index 49892f0649..bfd89d6a7b 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -98,6 +98,11 @@ interface IConfig { BISQ_URL: string; BISQ_ONION: string; }; + MAXMIND: { + ENABLED: boolean; + GEOLITE2_CITY: string; + GEOLITE2_ASN: string; + }, } const defaults: IConfig = { @@ -197,7 +202,12 @@ const defaults: IConfig = { 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1', 'BISQ_URL': 'https://bisq.markets/api', 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' - } + }, + "MAXMIND": { + 'ENABLED': false, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb", + "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb" + }, }; class Config implements IConfig { @@ -215,6 +225,7 @@ class Config implements IConfig { SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; + MAXMIND: IConfig['MAXMIND']; constructor() { const configs = this.merge(configFile, defaults); @@ -232,6 +243,7 @@ class Config implements IConfig { this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; + this.MAXMIND = configs.MAXMIND; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index b7e23a7fcc..3b2eb18e22 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -8,6 +8,7 @@ import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import lightningApi from '../../api/lightning/lightning-api-factory'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; +import { $lookupNodeLocation } from './sync-tasks/node-locations'; class NodeSyncService { constructor() {} @@ -33,6 +34,10 @@ class NodeSyncService { } logger.info(`Nodes updated.`); + if (config.MAXMIND.ENABLED) { + await $lookupNodeLocation(); + } + await this.$setChannelsInactive(); for (const channel of networkGraph.channels) { diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 1e718188ab..b44f4820c7 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -206,7 +206,7 @@ class LightningStatsUpdater { torNodes++; isUnnanounced = false; } - const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0])); + const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':')))); if (hasClearnet) { clearnetNodes++; isUnnanounced = false; diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts new file mode 100644 index 0000000000..e32dd8bad1 --- /dev/null +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -0,0 +1,63 @@ +import * as net from 'net'; +import maxmind, { CityResponse, AsnResponse } from 'maxmind'; +import nodesApi from '../../../api/explorer/nodes.api'; +import config from '../../../config'; +import DB from '../../../database'; +import logger from '../../../logger'; + +export async function $lookupNodeLocation(): Promise { + logger.info(`Running node location updater using Maxmind...`); + try { + const nodes = await nodesApi.$getAllNodes(); + const lookupCity = await maxmind.open(config.MAXMIND.GEOLITE2_CITY); + const lookupAsn = await maxmind.open(config.MAXMIND.GEOLITE2_ASN); + + for (const node of nodes) { + const sockets: string[] = node.sockets.split(','); + for (const socket of sockets) { + const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', ''); + const hasClearnet = [4, 6].includes(net.isIP(ip)); + if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') { + const city = lookupCity.get(ip); + const asn = lookupAsn.get(ip); + if (city && asn) { + const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`; + const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key]; + await DB.query(query, params); + + // Store Continent + if (city.continent?.geoname_id) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`, + [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]); + } + + // Store Country + if (city.country?.geoname_id) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`, + [city.country?.geoname_id, JSON.stringify(city.country?.names)]); + } + + // Store Division + if (city.subdivisions && city.subdivisions[0]) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`, + [city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]); + } + + // Store City + if (city.city?.geoname_id) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`, + [city.city?.geoname_id, JSON.stringify(city.city?.names)]); + } + } + } + } + } + logger.info(`Node location data updated.`); + } catch (e) { + logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e)); + } +} \ No newline at end of file From ca86364c35b092db67505b3a39eaf82c982451b2 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 11 Jul 2022 18:02:54 +0200 Subject: [PATCH 2/2] Add lightning to logger --- backend/src/logger.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/logger.ts b/backend/src/logger.ts index 43373e0432..ea7e8cd3d6 100644 --- a/backend/src/logger.ts +++ b/backend/src/logger.ts @@ -73,6 +73,9 @@ class Logger { } private getNetwork(): string { + if (config.LIGHTNING.ENABLED) { + return 'lightning'; + } if (config.BISQ.ENABLED) { return 'bisq'; }