tools: Reorganize block generator scripts. (#5582)

tools/block-generator/Makefile

@@ -12,7 +12,7 @@ clean-generator:
 	rm -f block-generator
 
 debug-blockgen:
-	python run_runner.py \
+	python scripts/run_runner.py \
 	--conduit-binary ./conduit \
 	--scenario $(SCENARIO) \
 	--report-directory $(REPORTS) \
@@ -38,7 +38,7 @@ run-runner: block-generator
 	--report-directory $(REPORTS)
 
 clean-reports:
-	rm -rf $(REPORTS) 
+	rm -rf $(REPORTS)
 
 pre-git-push:
 	mv _go.mod go.mod

tools/block-generator/README.md

@@ -170,7 +170,7 @@ Flags:
   -v, --verbose                             If set the runner will print debugging information from the generator and ledger.
  ```
 
-## Example Run using Conduit and Postgres in **bash** via `run_runner.sh`
+## Example Run using Conduit and Postgres
 
 A typical **runner** scenario involves:
 
@@ -179,30 +179,30 @@ A typical **runner** scenario involves:
 * a datastore -such as a postgres database- to collect `conduit`'s output
 * a `conduit` config file to define its import/export behavior
 
-`run_runner.sh` makes the following choices for the previous bullet points:
-
-* it can accept any scenario as its second argument, but defaults to [test_config.yml](./test_config.yml) when this isn't provided (this is a scenario with a lifetime of ~30 seconds)
-* knows how to import through a mock Algod running on port 11112 (which is the port the runner avails)
-* sets up a dockerized postgres database to receive conduit's output
-* configures `conduit` for these specs using [this config template](./runner/template/conduit.yml.tmpl)
+The `block-generator runner` subcommand has a number of options to configure behavion.
 
 ### Sample Run
 
 First you'll need to get a `conduit` binary. For example you can follow the [developer portal's instructions](https://developer.algorand.org/docs/get-details/conduit/GettingStarted/#installation) or run `go build .` inside of the directory `cmd/conduit` after downloading the `conduit` repo.
 
-Assume you've navigated to the `tools/block-generator` directory of
-the `go-algorand` repo, and:
+Run `make install` from the `go-algorand` root, this should add `block-generator` to your path.
 
-* saved the conduit binary to `tools/block-generator/conduit`
-* created a block generator scenario config at `tools/block-generator/scenario.yml`
+Start a postgres container using `scripts/run_postgres.sh`. This starts a container on port 15432 a database named generator_db and a user with credentials algorand/algorand.
 
-Then you can execute the following command to run the scenario:
+Now run `block-generator runner` to run the test:
 
 ```sh
-./run_runner.sh ./conduit scenario.yml
+block-generator runner \
+  --conduit-binary "$CONDUIT_BINARY" \
+  --report-directory reports \
+  --test-duration 30s \
+  --conduit-log-level trace \
+  --postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \
+  --scenario generator/test_scenario.yml \
+  --reset-db
 ```
 
 ### Scenario Report
 
-If all goes well, the run will generate a directory `../../tmp/OUTPUT_RUN_RUNNER_TEST`
-and in that directory you can see the statistics of the run in `scenario.report`.
+If all goes well, the run will generate a directory named reports.
+In that directory you can see the statistics of the run in the file ending with `.report`.

tools/block-generator/generator/config_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestInitConfigFile(t *testing.T) {
 	partitiontest.PartitionTest(t)
-	config, err := initializeConfigFile("../test_config.yml")
+	config, err := initializeConfigFile("test_scenario.yml")
 	require.NoError(t, err)
 	require.Equal(t, uint64(10), config.NumGenesisAccounts)
 	require.Equal(t, float32(0.25), config.AssetCloseFraction)

tools/block-generator/run_postgres.sh → tools/block-generator/scripts/run_postgres.sh

@@ -14,8 +14,6 @@ set -e
 POSTGRES_CONTAINER=generator-test-container
 POSTGRES_PORT=15432
 POSTGRES_DATABASE=generator_db
-CONFIG=${1:-"$(dirname $0)/test_config.yml"}
-echo "Using config file: $CONFIG"
 
 function start_postgres() {
   docker rm -f $POSTGRES_CONTAINER > /dev/null 2>&1 || true

tools/block-generator/run_runner.py → tools/block-generator/scripts/run_runner.py

@@ -82,7 +82,7 @@ def parse_args():
     parser.add_argument("--conduit-binary", help="Path to conduit binary")
     parser.add_argument(
         "--scenario",
-        default=(default := CWD / "test_config.yml"),
+        default=(default := CWD.parents[1] / "test_scenario.yml"),
         help=f"Scenario configuration file ({default=!s})",
     )
     parser.add_argument(

tools/block-generator/run_runner.sh → tools/block-generator/scripts/run_runner.sh

@@ -15,7 +15,7 @@ fi
 POSTGRES_CONTAINER=generator-test-container
 POSTGRES_PORT=15432
 POSTGRES_DATABASE=generator_db
-SCENARIO=${2:-"$(dirname $0)/test_config.yml"}
+SCENARIO=${2:-"$(dirname $0)/../test_scenario.yml"}
 echo "Using scenario config file: $SCENARIO"
 
 function start_postgres() {
@@ -51,10 +51,10 @@ echo "Starting postgres container."
 start_postgres
 echo "Starting test runner"
 $(dirname "$0")/block-generator runner \
-	--conduit-binary "$CONDUIT_BINARY" \
-	--report-directory $OUTPUT \
-	--test-duration 30s \
-	--conduit-log-level trace \
-	--postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \
-	--scenario ${SCENARIO} \
+  --conduit-binary "$CONDUIT_BINARY" \
+  --report-directory $OUTPUT \
+  --test-duration 30s \
+  --conduit-log-level trace \
+  --postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \
+  --scenario ${SCENARIO} \
   --reset-db