Problem: test_tx_inclusion is flaky when nodes halt due to different …

…gas_wanted (#1657)

* Problem: test_tx_inclusion is flaky when nodes halt due to different gas_wanted

wait longer to avoid nodes start with different max_gas_wanted

* add max timeout

* cleanup

integration_tests/test_basic.py

@@ -866,15 +866,19 @@ def fn(cmd):
         cronos.base_dir / "tasks.ini",
         lambda cmd: fn(cmd),
     )
+    cli = cronos.cosmos_cli()
+    # update right after a new block start
+    wait_for_new_blocks(cli, 1, sleep=0.1)
     cronos.supervisorctl("update")
+    # ensure nodes stop and start at the same time
+    time.sleep(2)
     wait_for_port(ports.evmrpc_port(cronos.base_port(0)))
 
     # reset to origin_cmd only
     if max_gas_wanted is None:
         return
 
     w3 = cronos.w3
-    cli = cronos.cosmos_cli()
     block_gas_limit = 81500000
     tx_gas_limit = 80000000
     max_tx_in_block = block_gas_limit // min(max_gas_wanted, tx_gas_limit)

integration_tests/utils.py

@@ -122,11 +122,14 @@ def wait_for_block(cli, height, timeout=240):
         raise TimeoutError(f"wait for block {height} timeout")
 
 
-def wait_for_new_blocks(cli, n, sleep=0.5):
+def wait_for_new_blocks(cli, n, sleep=0.5, timeout=240):
     cur_height = begin_height = int(get_sync_info(cli.status())["latest_block_height"])
+    start_time = time.time()
     while cur_height - begin_height < n:
         time.sleep(sleep)
         cur_height = int(get_sync_info(cli.status())["latest_block_height"])
+        if time.time() - start_time > timeout:
+            raise TimeoutError(f"wait for block {begin_height + n} timeout")
     return cur_height