From 672bda2a14090889e4a957babb5e33eeae4a3b00 Mon Sep 17 00:00:00 2001 From: Matthew Fettig Date: Thu, 11 Apr 2019 14:39:01 -0500 Subject: [PATCH 1/3] use setInterval for crawls --- index.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/index.js b/index.js index 6a95cdd..c59a157 100755 --- a/index.js +++ b/index.js @@ -1,29 +1,26 @@ +require('colors'); const crawlNodes = require('./src/crawlNodes'); const geolocateNodes = require('./src/geolocateNodes'); const save = require('./src/saveCrawl'); const config = require('./src/config'); const log = require('./src/logger'); -const TIMEOUT = 120 * 1000; -const INTERVAL = 6 * 60 * 60 * 1000; + +const CRAWL_INTERVAL = 120 * 1000; +const INTERVAL = 24 * 60 * 60 * 1000; const geolocate = () => { geolocateNodes().catch(log.error); } -const recursiveCrawl = () => { +const doCrawl = () => { crawlNodes(config.get('entry')) .then(save) - .then(() => { - setTimeout(recursiveCrawl, TIMEOUT); - }) .catch(error => { log.error(error); log.info('failed to record crawl'); - setTimeout(recursiveCrawl, TIMEOUT); }); } -recursiveCrawl(); -geolocate(); +doCrawl(); setInterval(geolocate, INTERVAL); - +setInterval(doCrawl, CRAWL_INTERVAL); From 0a6593fd2fb3e34a4f7331ef3d9cd6977b5f1ea8 Mon Sep 17 00:00:00 2001 From: Elliot Lee Date: Tue, 1 Sep 2020 13:06:45 -0700 Subject: [PATCH 2/3] README - add 'moved' notice --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d4845a5..745f1ec 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +# [Moved to xpring-eng](https://github.com/xpring-eng/validations-etl) + +If you are developing the instance of this project that is used with the [XRP Ledger Explorer](https://livenet.xrpl.org/), please use https://github.com/xpring-eng/rippled-network-crawler + # XRPL Network Crawler The network crawler looks for all nodes on the XRPL network. The crawler starts with a single rippled IP address, and queries its `/crawl` endpoint to get other peers connected to it. All of these nodes are added to the list, and any that also have an IP listed via the endpoint is then queried to find more nodes. The process is repeated until no new nodes with IP addresses are found. The interval between network crawls is 2 minutes. The full results of each crawl are added to the `prod_network_crawls` table, and the data for each node found is used to update the `prod_node_state` table. From 9cc3aa6c3e28defcfad9d624b3505c343b6e7755 Mon Sep 17 00:00:00 2001 From: Elliot Lee Date: Tue, 1 Sep 2020 13:07:01 -0700 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 745f1ec..f1fc5b3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# [Moved to xpring-eng](https://github.com/xpring-eng/validations-etl) +# [Moved to xpring-eng](https://github.com/xpring-eng/rippled-network-crawler) If you are developing the instance of this project that is used with the [XRP Ledger Explorer](https://livenet.xrpl.org/), please use https://github.com/xpring-eng/rippled-network-crawler