Problem: testground stateless test validator don't quit automatically (…

…#1533)

* Problem: testground stateless test validator don't quit automatically

Solution:
- validator quit when detected the chain is in idle

* improve log

* more stable benchmark

go.mod

@@ -276,7 +276,7 @@ replace (
 	github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2
 	github.com/ethereum/go-ethereum => github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7
 	// block-stm branch
-	github.com/evmos/ethermint => github.com/crypto-org-chain/ethermint v0.6.1-0.20240724133625-f3e62cb25070
+	github.com/evmos/ethermint => github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14
 	// Fix upstream GHSA-h395-qcrw-5vmq and GHSA-3vp4-m3rf-835h vulnerabilities.
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.0

go.sum

@@ -430,8 +430,8 @@ github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1
 github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg=
 github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1 h1:r0ALP31Wnw19FqEmqzsK2SFNqdMetHshnM/X/FeJRIo=
 github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY=
-github.com/crypto-org-chain/ethermint v0.6.1-0.20240724133625-f3e62cb25070 h1:BNjkJlyOAvr7Crd2kb2M/j3syKi4AjdiWN93M6iQaCA=
-github.com/crypto-org-chain/ethermint v0.6.1-0.20240724133625-f3e62cb25070/go.mod h1:Xap56y3WoiP7DyNammmB6N7P+Y83QQd6363RHGuaYV0=
+github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14 h1:ZVIlVCtx5X6sckKyMhc8nPBjXvr+/CzmXYqh7e/287I=
+github.com/crypto-org-chain/ethermint v0.6.1-0.20240729070824-350955227f14/go.mod h1:Xap56y3WoiP7DyNammmB6N7P+Y83QQd6363RHGuaYV0=
 github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde h1:sQIHTJfVt5VTrF7po9eZiFkZiPjlHbFvnXtGCOoBjNM=
 github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 h1:V43F3JFcqG4MUThf9W/DytnPblpR6CcaLBw2Wx6zTgE=

gomod2nix.toml

@@ -262,8 +262,8 @@ schema = 3
     hash = "sha256-lE4G5FaRb3MVi9FFVn+WlwsSTOB4SbjmVboKyQ5yB0A="
     replaced = "github.com/crypto-org-chain/go-ethereum"
   [mod."github.com/evmos/ethermint"]
-    version = "v0.6.1-0.20240724133625-f3e62cb25070"
-    hash = "sha256-XL7h6tMMIP3dWKO33UydnMlHZRotfbj1mQGonIEzHe0="
+    version = "v0.6.1-0.20240729070824-350955227f14"
+    hash = "sha256-aZ9Kq7PujK4alMXp1kk7LF7PTap8UdJbt+gK2ljvwyw="
     replaced = "github.com/crypto-org-chain/ethermint"
   [mod."github.com/fatih/color"]
     version = "v1.16.0"

testground/benchmark/benchmark/sendtx.py

@@ -28,11 +28,16 @@ def fund_test_accounts(w3, from_account, num_accounts) -> [Account]:
 
 
 def sendtx(w3: web3.Web3, acct: Account, tx_amount: int):
+    initial_nonce = w3.eth.get_transaction_count(acct.address)
     print(
-        "test begin, address", acct.address, "balance", w3.eth.get_balance(acct.address)
+        "test begin, address:",
+        acct.address,
+        "balance:",
+        w3.eth.get_balance(acct.address),
+        "nonce:",
+        initial_nonce,
     )
 
-    initial_nonce = w3.eth.get_transaction_count(acct.address)
     nonce = initial_nonce
     while nonce < initial_nonce + tx_amount:
         tx = {
@@ -59,7 +64,12 @@ def sendtx(w3: web3.Web3, acct: Account, tx_amount: int):
             print(f"{acct.address} sent {nonce} transactions")
 
     print(
-        "test end, address", acct.address, "balance", w3.eth.get_balance(acct.address)
+        "test end, address:",
+        acct.address,
+        "balance:",
+        w3.eth.get_balance(acct.address),
+        "nonce:",
+        w3.eth.get_transaction_count(acct.address),
     )
 
 

testground/benchmark/benchmark/stateless.py

@@ -5,10 +5,12 @@
 import subprocess
 import tarfile
 import tempfile
+import time
 from pathlib import Path
 from typing import List
 
 import fire
+import requests
 
 from .cli import ChainCommand
 from .peer import (
@@ -22,14 +24,15 @@
 from .sendtx import generate_load
 from .topology import connect_all
 from .types import PeerPacket
-from .utils import wait_for_block, wait_for_port
+from .utils import wait_for_block, wait_for_port, wait_for_w3
 
 # use cronosd on host machine
 LOCAL_CRONOSD_PATH = "cronosd"
 DEFAULT_CHAIN_ID = "cronos_777-1"
 DEFAULT_DENOM = "basecro"
 # the container must be deployed with the prefixed name
 HOSTNAME_TEMPLATE = "testplan-{index}"
+LOCAL_RPC = "http://localhost:26657"
 
 
 class CLI:
@@ -130,24 +133,56 @@ def run(
         cli = ChainCommand(cronosd)
         wait_for_port(26657)
         wait_for_port(8545)
-        wait_for_port(9090)
-        wait_for_block(cli, 1)
+        wait_for_block(cli, 3)
 
         if group == VALIDATOR_GROUP:
-            # validators don't quit
-            proc.wait()
+            # validators quit when the chain is idle for a while
+            detect_idle(20, 20)
         else:
+            wait_for_w3()
             generate_load(cli, cfg["num_accounts"], cfg["num_txs"], home=home)
-            proc.kill()
-            proc.wait()
-
-            # collect outputs
-            outdir = Path(outdir)
-            if outdir.exists():
-                with tarfile.open(
-                    outdir / f"{group}_{group_seq}.tar.bz2", "x:bz2"
-                ) as tar:
-                    tar.add(home, arcname="data")
+
+        proc.kill()
+        proc.wait()
+
+        # collect outputs
+        outdir = Path(outdir)
+        if outdir.exists():
+            filename = outdir / f"{group}_{group_seq}.tar.bz2"
+            filename.unlink(missing_ok=True)
+            with tarfile.open(filename, "x:bz2") as tar:
+                tar.add(home, arcname="data")
+
+
+def detect_idle(idle_blocks: int, interval: int):
+    """
+    returns if the chain is empty for consecutive idle_blocks
+    """
+    while True:
+        latest = block_height()
+        for i in range(idle_blocks):
+            height = latest - i
+            if height <= 0:
+                break
+            if len(block_txs(height)) > 0:
+                break
+        else:
+            # normal quit means idle
+            return
+
+        # break early means not idle
+        time.sleep(interval)
+        continue
+
+
+def block_height():
+    rsp = requests.get(f"{LOCAL_RPC}/status").json()
+    return int(rsp["result"]["sync_info"]["latest_block_height"])
+
+
+def block_txs(height):
+    rsp = requests.get(f"{LOCAL_RPC}/block?height={height}").json()
+    return rsp["result"]["block"]["data"]["txs"]
 
 
 def init_node_local(

testground/benchmark/benchmark/utils.py

@@ -5,6 +5,7 @@
 
 import bech32
 import tomlkit
+import web3
 from eth_account import Account
 from hexbytes import HexBytes
 from web3._utils.transactions import fill_nonce, fill_transaction_defaults
@@ -68,6 +69,20 @@ def wait_for_block(cli, target: int, timeout=40):
     return height
 
 
+def wait_for_w3(timeout=40):
+    for i in range(timeout):
+        try:
+            w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
+            w3.eth.get_balance("0x0000000000000000000000000000000000000001")
+        except:  # noqa
+            time.sleep(1)
+            continue
+
+        break
+    else:
+        raise TimeoutError("Waited too long for web3 json-rpc to be ready.")
+
+
 def decode_bech32(addr):
     _, bz = bech32.bech32_decode(addr)
     return HexBytes(bytes(bech32.convertbits(bz, 5, 8)))

testground/benchmark/pyproject.toml

@@ -15,6 +15,7 @@ web3 = "^6"
 hexbytes = "^0"
 bech32 = "^1"
 fire = "^0"
+requests = "^2.32"
 
 [tool.poetry.dev-dependencies]
 pytest = "^8.2"