Merge branch 'feature/even-more-chunks' into feature/and-yet-more-chunks

MinaProtocol · Oct 10, 2023 · 45fc3d8 · 45fc3d8
2 parents bf2022e + f309ad2
commit 45fc3d8
Show file tree

Hide file tree

Showing 46 changed files with 336 additions and 182 deletions.
diff --git a/buildkite/scripts/build-test-executive.sh b/buildkite/scripts/build-test-executive.sh
diff --git a/buildkite/scripts/run-test-executive.sh b/buildkite/scripts/run-test-executive.sh
@@ -10,9 +10,21 @@ if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]
   exit 0
 fi
 
-./test_executive.exe cloud "$TEST_NAME" \
+# Don't prompt for answers during apt-get install
+export DEBIAN_FRONTEND=noninteractive
+
+echo "deb [trusted=yes] https://apt.releases.hashicorp.com $MINA_DEB_CODENAME main" | tee /etc/apt/sources.list.d/hashicorp.list
+apt-get update
+apt-get install -y "terraform"
+
+echo "Installing mina daemon package: mina-test-executive=${MINA_DEB_VERSION}"
+echo "deb [trusted=yes] http://packages.o1test.net $MINA_DEB_CODENAME $MINA_DEB_RELEASE" | tee /etc/apt/sources.list.d/mina.list
+apt-get update
+apt-get install --allow-downgrades -y "mina-test-executive=$MINA_DEB_VERSION" "mina-logproc=$MINA_DEB_VERSION"
+
+mina-test-executive cloud "$TEST_NAME" \
   --mina-image "$MINA_IMAGE" \
   --archive-image "$ARCHIVE_IMAGE" \
   --mina-automation-location ./automation \
   | tee "$TEST_NAME.test.log" \
-  | ./logproc.exe -i inline -f '!(.level in ["Debug", "Spam"])'
+  | mina-logproc -i inline -f '!(.level in ["Debug", "Spam"])'
diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall
@@ -5,6 +5,9 @@ let S = ../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../Pipeline/Dsl.dhall
+let PipelineTag = ../Pipeline/Tag.dhall
+let PipelineMode = ../Pipeline/Mode.dhall
+
 let JobSpec = ../Pipeline/JobSpec.dhall
 
 let Command = ./Base.dhall
@@ -16,13 +19,16 @@ let DebianVersions = ../Constants/DebianVersions.dhall
 
 in
 
-let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+\(mode: PipelineMode.Type) -> 
     Pipeline.Config::{
       spec =
         JobSpec::{
           dirtyWhen = DebianVersions.dirtyWhen debVersion,
           path = "Release",
-          name = "MinaArtifact${DebianVersions.capitalName debVersion}"
+          name = "MinaArtifact${DebianVersions.capitalName debVersion}",
+          tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ],
+          mode = mode
         },
       steps = [
         Libp2p.step debVersion,
@@ -119,7 +125,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
 
 in
 {
-  bullseye  = pipeline DebianVersions.DebVersion.Bullseye
-  , buster  = pipeline DebianVersions.DebVersion.Buster
-  , focal   = pipeline DebianVersions.DebVersion.Focal
+  bullseye  = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest
+  , buster  = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.PullRequest
+  , focal   = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.PullRequest
 }
diff --git a/buildkite/src/Command/TestExecutive.dhall b/buildkite/src/Command/TestExecutive.dhall
@@ -15,36 +15,11 @@ let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 in
 
 {
-  build = \(duneProfile : Text) ->
-    Command.build
-      Command.Config::{
-        commands =
-            -- Build test executive binary
-            RunInToolchain.runInToolchain [
-              "DUNE_PROFILE=${duneProfile}"
-            ] "./buildkite/scripts/build-test-executive.sh"
-
-            #
-
-            [
-              -- Cache test-executive binary
-              Cmd.run "artifact-cache-helper.sh test_executive.exe --upload",
-              Cmd.run "artifact-cache-helper.sh logproc.exe --upload"
-            ],
-        label = "Build test-executive",
-        key = "build-test-executive",
-        target = Size.XLarge
-      },
-
   execute = \(testName : Text) -> \(dependsOn : List Command.TaggedKey.Type) ->
     Command.build
       Command.Config::{
         commands =
             [
-              -- Download test dependencies
-              Cmd.run "artifact-cache-helper.sh test_executive.exe && chmod +x test_executive.exe",
-              Cmd.run "artifact-cache-helper.sh logproc.exe && chmod +x logproc.exe",
-
               -- Execute test based on BUILD image
               Cmd.run "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive.sh ${testName}"
             ],
@@ -78,9 +53,6 @@ in
       Command.Config::{
         commands =
             [
-              -- Download test dependencies
-              Cmd.run "artifact-cache-helper.sh test_executive.exe && chmod +x test_executive.exe",
-              Cmd.run "artifact-cache-helper.sh logproc.exe && chmod +x logproc.exe",
               Cmd.run "artifact-cache-helper.sh snarkyjs_test.tar.gz && tar -xzf snarkyjs_test.tar.gz",
 
               -- Execute test based on BUILD image

diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall
@@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
@@ -36,6 +37,7 @@ in  Pipeline.build
           ]
         , path = "Lint"
         , name = "Fast"
+        , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
         }
       , steps =
         [ Command.build

diff --git a/buildkite/src/Jobs/Lint/HelmChart.dhall b/buildkite/src/Jobs/Lint/HelmChart.dhall
@@ -4,6 +4,7 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -23,7 +24,8 @@ Pipeline.build
         S.exactly "buildkite/scripts/helm-ci" "sh"
       ],
       path = "Lint",
-      name = "HelmChart"
+      name = "HelmChart",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall
@@ -4,6 +4,7 @@ let B = ../../External/Buildkite.dhall
 let SelectFiles = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Cmd = ../../Lib/Cmds.dhall
@@ -20,7 +21,8 @@ Pipeline.build
     spec = JobSpec::{
       dirtyWhen = [ SelectFiles.everything ],
       path = "Lint",
-      name = "Merge"
+      name = "Merge",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/OCaml.dhall b/buildkite/src/Jobs/Lint/OCaml.dhall
@@ -7,6 +7,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 
 let RunInToolchain = ../../Command/RunInToolchain.dhall
 
@@ -31,6 +32,7 @@ in  Pipeline.build
                 [ dirtyDhallDir, S.strictlyStart (S.contains "src/") ]
               , path = "Lint"
               , name = "OCaml"
+              , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
               }
       , steps =
         [ Command.build

diff --git a/buildkite/src/Jobs/Lint/Rust.dhall b/buildkite/src/Jobs/Lint/Rust.dhall
@@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -19,7 +21,8 @@ Pipeline.build
     spec = JobSpec::{
       dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") ],
       path = "Lint",
-      name = "Rust"
+      name = "Rust",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall
@@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -25,7 +27,8 @@ Pipeline.build
         S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts")
       ],
       path = "Lint",
-      name = "TestnetAlerts"
+      name = "TestnetAlerts",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall
@@ -4,6 +4,8 @@ let List/map = Prelude.List.map
 
 let S = ../../Lib/SelectFiles.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let Cmd = ../../Lib/Cmds.dhall
 let Command = ../../Command/Base.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
@@ -52,7 +54,8 @@ in Pipeline.build Pipeline.Config::{
         S.strictlyStart (S.contains ValidationService.rootPath)
       ],
       path = "Lint",
-      name = "ValidationService"
+      name = "ValidationService",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
   steps = [
     Command.build Command.Config::{

diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall
@@ -4,6 +4,8 @@ let B = ../../External/Buildkite.dhall
 let SelectFiles = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Cmd = ../../Lib/Cmds.dhall
@@ -23,7 +25,8 @@ Pipeline.build
         SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml")
       ],
       path = "Lint",
-      name = "Xrefcheck"
+      name = "Xrefcheck",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Release/HelmRelease.dhall b/buildkite/src/Jobs/Release/HelmRelease.dhall
@@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -21,7 +23,8 @@ Pipeline.build
         S.exactly "buildkite/scripts/helm-ci" "sh"
       ],
       path = "Release",
-      name = "HelmRelease"
+      name = "HelmRelease",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -29,7 +31,8 @@ Pipeline.build
           S.strictlyStart (S.contains "src/app/itn_orchestrator")
         ],
         path = "Release",
-        name = "ItnOrchestratorArtifact"
+        name = "ItnOrchestratorArtifact",
+        tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
       },
     steps = [
       DockerImage.generateStep spec

diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -27,7 +29,8 @@ Pipeline.build
           S.strictlyStart (S.contains "frontend/leaderboard")
         ],
         path = "Release",
-        name = "LeaderboardArtifact"
+        name = "LeaderboardArtifact",
+        tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
       },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall
@@ -0,0 +1,50 @@
+let Prelude = ../../External/Prelude.dhall
+
+let Cmd = ../../Lib/Cmds.dhall
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+let Size = ../../Command/Size.dhall
+let DockerImage = ../../Command/DockerImage.dhall
+let DockerLogin = ../../Command/DockerLogin/Type.dhall
+
+
+in
+
+Pipeline.build
+  Pipeline.Config::{
+    spec =
+      JobSpec::{
+        dirtyWhen = [
+          S.strictlyStart (S.contains "dockerfiles/stages/1-"),
+          S.strictlyStart (S.contains "dockerfiles/stages/2-"),
+          S.strictlyStart (S.contains "dockerfiles/stages/3-"),
+          S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"),
+          S.strictly (S.contains "opam.export"),
+          -- Rust version has changed
+          S.strictlyEnd (S.contains "rust-toolchain.toml")
+        ],
+        path = "Release",
+        name = "MinaToolchainArtifactBullseye",
+        tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
+      },
+    steps = [
+
+      -- mina-toolchain Debian 11 "Bullseye" Toolchain
+      let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{
+        service="mina-toolchain",
+        deb_codename="bullseye",
+        extra_args="--no-cache",
+        step_key="toolchain-bullseye-docker-image"
+      }
+
+      in
+
+      DockerImage.generateStep toolchainBullseyeSpec
+
+    ]
+  }