Revive replayer test:

- update dump
- use dockers

buildkite/scripts/replayer-test.sh

@@ -1,41 +1,67 @@
 #!/bin/bash
 
-TEST_DIR=/workdir/src/app/replayer/test
-PGPASSWORD=arbitraryduck
+TEST_DIR=/workdir/src/app/replayer/
 
-set -eo pipefail
+set -eox pipefail
 
 echo "Updating apt, installing packages"
 apt-get update
+
 # Don't prompt for answers during apt-get install
 export DEBIAN_FRONTEND=noninteractive
 
-# time zone = US Pacific
-/bin/echo -e "12\n10" | apt-get install -y tzdata
-apt-get install -y git postgresql apt-transport-https ca-certificates curl
+apt-get install -y git apt-transport-https ca-certificates curl wget
 
-git config --global --add safe.directory /workdir
+git config --global --add safe.directory $BUILDKITE_BUILD_CHECKOUT_PATH
 
 source buildkite/scripts/export-git-env-vars.sh
 
-echo "deb [trusted=yes] http://packages.o1test.net bullseye ${MINA_DEB_RELEASE}" | tee /etc/apt/sources.list.d/mina.list
-apt-get update
+DB=archive
+DOCKER_IMAGE=12.4-alpine
+CONTAINER_FILE=docker.container
+
+PG_PORT=5433
+PG_PASSWORD=somepassword
+DOCKER_IMAGE=12.4-alpine
+CONTAINER_FILE=docker.container
+
+function cleanup () {
+    CONTAINER=`cat $CONTAINER_FILE`
+
+    if [[ ! -z $CONTAINER ]] ; then
+	echo "Killing, removing docker container"
+	for action in kill rm; do
+	    docker container $action $CONTAINER
+	done
+    fi
+
+    rm -f $CONTAINER_FILE
+}
+
+docker network create replayer || true
 
-echo "Installing archive node package: mina-archive=${MINA_DEB_VERSION}"
-apt-get install --allow-downgrades -y mina-archive=${MINA_DEB_VERSION}
+# -v mounts dir with Unix socket on host
+echo "Starting docker with Postgresql"
+docker run \
+       --network replayer \
+       --volume $BUILDKITE_BUILD_CHECKOUT_PATH:/workdir \
+       --name replayer-postgres -d -p $PG_PORT:5432 \
+       -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=$PG_PASSWORD -e POSTGRES_DB=$DB postgres:$DOCKER_IMAGE > $CONTAINER_FILE
 
-echo "Generating locale for Postgresql"
-locale-gen en_US.UTF-8
+trap "cleanup; exit 1" SIGINT
 
-echo "Starting Postgresql service"
-service postgresql start
+# wait for Postgresql to become available
+sleep 5
 
 echo "Populating archive database"
-cd ~postgres
-su postgres -c psql < $TEST_DIR/archive_db.sql
-echo "ALTER USER postgres PASSWORD '$PGPASSWORD';" | su postgres -c psql
-cd /workdir
-
-echo "Running replayer"
-mina-replayer --archive-uri postgres://postgres:$PGPASSWORD@localhost:5432/archive \
-	      --input-file $TEST_DIR/input.json --output-file /dev/null
+
+NETWORK_GATEWAY=$(docker network inspect -f "{{(index .IPAM.Config 0).Gateway}}" replayer)
+
+PG_CONN="postgres://postgres:$PG_PASSWORD@$NETWORK_GATEWAY:$PG_PORT/$DB"
+
+
+docker exec replayer-postgres psql $PG_CONN -f $TEST_DIR/test/archive_db.sql
+
+docker run --network replayer --volume $BUILDKITE_BUILD_CHECKOUT_PATH:/workdir gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG /workdir/scripts/replayer-test.sh -d $TEST_DIR -a mina-replayer -p $PG_CONN
+
+cleanup

buildkite/src/Command/Base.dhall

@@ -126,7 +126,8 @@ let targetToAgent = \(target : Size) ->
           Large = toMap { size = "generic" },
           Medium = toMap { size = "generic" },
           Small = toMap { size = "generic" },
-          Integration = toMap { size = "integration" }
+          Integration = toMap { size = "integration" },
+          QA = toMap { size = "qa" }
         }
         target
 

buildkite/src/Command/ReplayerTest.dhall

@@ -10,11 +10,7 @@ let Cmd = ../Lib/Cmds.dhall in
     Command.build
       Command.Config::{
         commands = [
-          Cmd.runInDocker
-            Cmd.Docker::{
-              image = (../Constants/ContainerImages.dhall).ubuntu2004
-            }
-            "./buildkite/scripts/replayer-test.sh"
+          Cmd.run "./buildkite/scripts/replayer-test.sh"
         ],
         label = "Replayer test",
         key = "replayer-test",

buildkite/src/Command/Size.dhall

@@ -1 +1 @@
-<XLarge|Large|Medium|Small|Integration>
+<XLarge|Large|Medium|Small|Integration|QA>

buildkite/src/Jobs/Test/ReplayerTest.dhall.SUSPENDED → buildkite/src/Jobs/Test/ReplayerTest.dhall

@@ -1,7 +1,7 @@
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
-
+let PipelineTag = ../../Pipeline/Tag.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let ReplayerTest = ../../Command/ReplayerTest.dhall
@@ -16,9 +16,12 @@ in  Pipeline.build
         , dirtyWhen =
           [ S.strictlyStart (S.contains "src")
           , S.exactly "buildkite/scripts/replayer-test" "sh"
+          , S.exactly "buildkite/src/Jobs/Test/ReplayerTest" "dhall"
+          , S.exactly "buildkite/src/Command/ReplayerTest" "dhall"
           ]
         , path = "Test"
         , name = "ReplayerTest"
+        , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ]
         }
       , steps = [ ReplayerTest.step dependsOn ]
       }

scripts/replayer-test.sh

@@ -3,29 +3,15 @@
 # test replayer on known archive db
 
 REPLAYER_DIR=src/app/replayer
-DB=archive
+REPLAYER_APP=_build/default/src/app/replayer/replayer.exe
+PG_CONN=PG_CONN=postgres://postgres:postgres@localhost:5433/archive
 
-DOCKER_IMAGE=12.4-alpine
-CONTAINER_FILE=docker.container
-
-PG_PORT=5432
-PG_PASSWORD=somepassword
-PG_CONN=postgres://postgres:$PG_PASSWORD@localhost:$PG_PORT/$DB
-
-SOCKET_DIR=/var/run/postgresql
-
-function cleanup () {
-    CONTAINER=`cat $CONTAINER_FILE`
-
-    if [[ ! -z $CONTAINER ]] ; then
-	echo "Killing, removing docker container"
-	for action in kill rm; do
-	    docker container $action $CONTAINER
-	done
-    fi
-
-    rm -f $CONTAINER_FILE
-}
+while [[ "$#" -gt 0 ]]; do case $1 in
+  -d|--dir) REPLAYER_DIR="$2"; shift;;
+  -a|--app) REPLAYER_APP="$2"; shift;;
+  -p| --pg) PG_CONN="$2"; shift;;
+  *) echo "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
 
 function report () {
  if [[ $1 == 0 ]]; then
@@ -35,30 +21,11 @@ function report () {
  fi
 }
 
-# -v mounts dir with Unix socket on host
-echo "Starting docker with Postgresql"
-docker run \
-       --name replayer-postgres -d -v $SOCKET_DIR:$SOCKET_DIR -p $PG_PORT:$PG_PORT \
-       -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=$PG_PASSWORD -e POSTGRES_DB=$DB postgres:$DOCKER_IMAGE > $CONTAINER_FILE
-
-trap "cleanup; exit 1" SIGINT
-
-# wait for Postgresql to become available
-sleep 5
-
-echo "Populating archive database"
-psql -U postgres -d $DB < $REPLAYER_DIR/test/archive_db.sql
-
-echo "Building replayer"
-make replayer
-
 echo "Running replayer"
-./_build/default/src/app/replayer/replayer.exe --archive-uri $PG_CONN --input-file $REPLAYER_DIR/test/input.json
+$REPLAYER_APP --archive-uri $PG_CONN --input-file $REPLAYER_DIR/test/input.json
 
 RESULT=$?
 
 report $RESULT
 
-cleanup
-
 exit $RESULT