Add a CLI reporting tool (#9972)

* Add a reporting tool that produces a CSV file with the list of transactions and running balance for the given accounts * Add a note on how to smoke test the GraphQL module * Limit how much data Tomcat will read from a POST request * Remove an unused variable in the monitor after a recent refactor --------- Signed-off-by: Steven Sheehy <[email protected]>
hashgraph · Jan 3, 2025 · f87bed8 · f87bed8
1 parent 219bc9a
commit f87bed8
Show file tree

Hide file tree

Showing 19 changed files with 4,671 additions and 10 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -41,3 +41,9 @@ updates:
     package-ecosystem: "npm"
     schedule:
       interval: "weekly"
+
+  - directory: "/tools/mirror-report"
+    open-pull-requests-limit: 10
+    package-ecosystem: "npm"
+    schedule:
+      interval: "weekly"
diff --git a/.gitignore b/.gitignore
@@ -161,6 +161,6 @@ snyk
 
 hedera-mirror-rest/report.*
 hedera-mirror-rest/monitoring/config/serverlist.json
+report-*.csv
 
-
-hedera-mirror-web3/src/testHistorical/solidity/openzeppelin
+hedera-mirror-web3/src/testHistorical/solidity/openzeppelin
diff --git a/docs/graphql/README.md b/docs/graphql/README.md
@@ -22,3 +22,15 @@ value, it is recommended to only populate overridden properties in the custom `a
 | `hedera.mirror.graphql.db.sslMode`          | DISABLE                                          | The ssl level of protection against eavesdropping, man-in-the-middle (MITM) and impersonation on the db connection. Accepts either DISABLE, ALLOW, PREFER, REQUIRE, VERIFY_CA or VERIFY_FULL. |
 | `hedera.mirror.graphql.db.statementTimeout` | 10000                                            | The maximum amount of time in seconds to wait for a query to finish                                                                                                                           |
 | `hedera.mirror.graphql.db.username`         | mirror_graphql                                   | The username used to connect to the database.                                                                                                                                                 |
+
+## Smoke Testing
+
+The GraphiQL browser based GraphQL interface can be accessed at http://localhost:8083/graphiql when running the graphql
+module. This tool can be used to explore the API and make adhoc requests.
+
+If a command line tool is needed, curl can be used to make basic requests:
+
+```bash
+curl -X POST http://localhost:8083/graphql/alpha -H 'Content-Type: application/json' \
+  -d '{"query": "{account(input: {entityId: {shard: 0, realm: 0, num: 2}}) { balance }}"}'
+```
diff --git a/hedera-mirror-monitor/src/main/java/com/hedera/mirror/monitor/publish/PublishMetrics.java b/hedera-mirror-monitor/src/main/java/com/hedera/mirror/monitor/publish/PublishMetrics.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2024 Hedera Hashgraph, LLC
+ * Copyright (C) 2020-2025 Hedera Hashgraph, LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -46,7 +46,6 @@ public class PublishMetrics {
     private final Map<Tags, Timer> handleTimers = new ConcurrentHashMap<>();
     private final Map<Tags, Timer> submitTimers = new ConcurrentHashMap<>();
     private final MeterRegistry meterRegistry;
-    private final NodeSupplier nodeSupplier;
     private final PublishProperties publishProperties;
 
     public void onSuccess(PublishResponse response) {

diff --git a/...ra-mirror-monitor/src/test/java/com/hedera/mirror/monitor/publish/PublishMetricsTest.java b/...ra-mirror-monitor/src/test/java/com/hedera/mirror/monitor/publish/PublishMetricsTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021-2024 Hedera Hashgraph, LLC
+ * Copyright (C) 2021-2025 Hedera Hashgraph, LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -45,7 +45,6 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.boot.test.system.CapturedOutput;
 import org.springframework.boot.test.system.OutputCaptureExtension;
@@ -55,9 +54,6 @@ class PublishMetricsTest {
 
     private static final String SCENARIO_NAME = "test";
 
-    @Mock
-    private NodeSupplier nodeSupplier;
-
     private MeterRegistry meterRegistry;
     private PublishMetrics publishMetrics;
     private PublishProperties publishProperties;
@@ -68,7 +64,7 @@ class PublishMetricsTest {
     void setup() {
         meterRegistry = new SimpleMeterRegistry();
         publishProperties = new PublishProperties();
-        publishMetrics = new PublishMetrics(meterRegistry, nodeSupplier, publishProperties);
+        publishMetrics = new PublishMetrics(meterRegistry, publishProperties);
 
         PublishScenarioProperties publishScenarioProperties = new PublishScenarioProperties();
         publishScenarioProperties.setName(SCENARIO_NAME);

diff --git a/hedera-mirror-rest-java/src/main/resources/application.yml b/hedera-mirror-rest-java/src/main/resources/application.yml
@@ -58,6 +58,7 @@ server:
   shutdown: graceful
   tomcat:
     connection-timeout: 3s
+    max-http-form-post-size: 1KB
 spring:
   application:
     name: hedera-mirror-rest-java

diff --git a/hedera-mirror-web3/src/main/resources/application.yml b/hedera-mirror-web3/src/main/resources/application.yml
@@ -50,6 +50,7 @@ server:
   shutdown: graceful
   tomcat:
     connection-timeout: 3s
+    max-http-form-post-size: 1MB
 spring:
   application:
     name: hedera-mirror-web3

diff --git a/tools/mirror-report/README.md b/tools/mirror-report/README.md
@@ -0,0 +1,98 @@
+# Mirror Node Report Tool
+
+This tool is a CLI tool that queries the REST API for an account and produces a report showing any transfers to or from
+the account and its resulting balance.
+
+## Install
+
+First ensure Node and NPM are installed. Following the instructions specific to your operating system.
+
+### MacOS
+
+Open the Terminal app and run the following commands:
+
+```shell
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+nvm install 22
+npm install -g @hashgraph/mirror-report
+```
+
+### Windows
+
+Download the Node.js 22 prebuilt [installer](https://nodejs.org/dist/v22.12.0/node-v22.12.0-x64.msi) for Windows x64.
+Follow the prompts to complete the installation with the default options. Open the Command Prompt application and run
+the following:
+
+```shell
+npm install -g @hashgraph/mirror-report
+```
+
+## Upgrading
+
+To upgrade the version of the report tool, re-run the NPM install command to grab the latest version:
+
+```shell
+npm install -g @hashgraph/mirror-report
+```
+
+## Running
+
+To generate the report, use the `mirror` command line tool from the installation section.
+
+```shell
+Usage: mirror [options] [command]
+
+Options:
+  -h, --help        display help for command
+
+Commands:
+  report [options]  Generate a report for specific accounts
+  help [command]    display help for command
+```
+
+Run the `report` sub-command to generate the report data. Pass at least one account you want to include in the report.
+By default, it generates a report for the current date. Pass the `--from-date YYYY-MM-DD` and `--to-date YYYY-MM-DD`
+arguments to customize the time range the report should cover.
+
+```shell
+Usage: mirror report [options]
+
+Generate a report for the given accounts.
+
+Options:
+  -a, --account <accountId...>  The accounts to include in the report
+  -c, --combined                Whether a single combined report should be generated for all accounts. By default it produces separate reports
+  -f, --from-date <YYYY-MM-DD>  The day the report should start (inclusive) (default: today)
+  -n, --network <network>       The Hedera network to connect to (choices: "mainnet", "testnet", "previewnet", default: "mainnet")
+  -t, --to-date <YYYY-MM-DD>    The day the report should end (exclusive) (default: tomorrow)
+  -h, --help                    display help for command
+```
+
+Example execution:
+
+```shell
+mirror report --combined -a 0.0.1000-0.0.1001 0.0.1003 -f 2024-11-29 -n testnet
+```
+
+```shell
+2025-01-02T19:06:08.399Z Running report with options: {"fromDate":"2024-11-29","network":"testnet","toDate":"2025-01-03","combined":true,"account":["0.0.1000","0.0.1001","0.0.1003"]}
+2025-01-02T19:06:08.399Z Invoking https://testnet.mirrornode.hedera.com/api/v1/accounts/0.0.1000?timestamp=1732838400
+2025-01-02T19:06:08.515Z Starting balance of 1300000002 for account 0.0.1000 at 1732837599.823821194
+2025-01-02T19:06:08.515Z Invoking https://testnet.mirrornode.hedera.com/api/v1/transactions?account.id=0.0.1000&limit=100&order=asc&timestamp=gt:1732837599.823821194&timestamp=lt:1735862400
+2025-01-02T19:06:08.591Z Invoking https://testnet.mirrornode.hedera.com/api/v1/accounts/0.0.1001?timestamp=1732838400
+2025-01-02T19:06:08.639Z Starting balance of 197103815708295 for account 0.0.1001 at 1732837599.823821194
+2025-01-02T19:06:08.639Z Invoking https://testnet.mirrornode.hedera.com/api/v1/transactions?account.id=0.0.1001&limit=100&order=asc&timestamp=gt:1732837599.823821194&timestamp=lt:1735862400
+2025-01-02T19:06:08.687Z Invoking https://testnet.mirrornode.hedera.com/api/v1/accounts/0.0.1003?timestamp=1732838400
+2025-01-02T19:06:08.737Z Starting balance of 1001658807600 for account 0.0.1003 at 1732837599.823821194
+2025-01-02T19:06:08.737Z Invoking https://testnet.mirrornode.hedera.com/api/v1/transactions?account.id=0.0.1003&limit=100&order=asc&timestamp=gt:1732837599.823821194&timestamp=lt:1735862400
+2025-01-02T19:06:08.781Z Generated report successfully at report-2024-11-29.csv with 1 entries
+```
+
+```shell
+cat report-2024-11-29.csv
+```
+
+```shell
+timestamp,sender,receiver,fees,amount,balance
+1732901875.430169000,0.0.5190744,0.0.1000,0.00000000,1.00000000,1.400000002
+```
diff --git a/tools/mirror-report/index.js b/tools/mirror-report/index.js
@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+
+/*
+ * Copyright (C) 2024-2025 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Option, program} from 'commander';
+import {report} from './src/report.js';
+import {parseAccount, validateDate} from './src/utils.js';
+
+const now = new Date();
+const today = now.toISOString().slice(0, 10);
+now.setDate(now.getDate() + 1);
+const tomorrow = now.toISOString().slice(0, 10);
+
+program.command('report')
+.description('Generate a report for the given accounts.')
+.requiredOption('-a, --account <accountId...>',
+  'The accounts to include in the report. Can be single account or range (e.g. 0.0.3-0.0.9)', parseAccount)
+.option('-c, --combined',
+  'Whether a single combined report should be generated for all accounts. By default it produces separate reports')
+.requiredOption('-f, --from-date <YYYY-MM-DD>', 'The day the report should start (inclusive)', validateDate, today)
+.addOption(new Option('-n, --network <network>', 'The Hedera network to connect to')
+.choices(['mainnet', 'testnet', 'previewnet'])
+.default('mainnet')
+.makeOptionMandatory())
+.requiredOption('-t, --to-date <YYYY-MM-DD>', 'The day the report should end (exclusive)', validateDate, tomorrow)
+.showHelpAfterError()
+.action(report);
+
+program.parse();