From b2cc7de959a2b5e5d19049b3dcbddc25ed94aed6 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 13:28:49 +0200 Subject: [PATCH 1/5] WIP --- buildkite/src/Command/MinaArtifact.dhall | 20 ++++-- buildkite/src/Jobs/Lint/Fast.dhall | 2 + buildkite/src/Jobs/Lint/HelmChart.dhall | 4 +- buildkite/src/Jobs/Lint/Merge.dhall | 4 +- buildkite/src/Jobs/Lint/OCaml.dhall | 2 + buildkite/src/Jobs/Lint/Rust.dhall | 5 +- buildkite/src/Jobs/Lint/TestnetAlerts.dhall | 5 +- .../src/Jobs/Lint/ValidationService.dhall | 5 +- buildkite/src/Jobs/Lint/Xrefcheck.dhall | 5 +- buildkite/src/Jobs/Release/BotArtifact.dhall | 15 ++-- buildkite/src/Jobs/Release/HelmRelease.dhall | 5 +- .../Jobs/Release/LeaderboardArtifact.dhall | 5 +- .../MinaToolchainArtifactBullseye.dhall | 50 +++++++++++++ ...hall => MinaToolchainArtifactBuster.dhall} | 18 ++--- .../src/Jobs/Release/TestnetAlerts.dhall | 5 +- buildkite/src/Jobs/Release/TraceTool.dhall | 5 +- .../src/Jobs/Test/ArchiveNodeUnitTest.dhall | 2 + buildkite/src/Jobs/Test/CheckDhall.dhall | 4 +- .../src/Jobs/Test/CheckGraphQLSchema.dhall | 4 +- .../src/Jobs/Test/CompareSignatures.dhall | 2 + .../src/Jobs/Test/ConnectToMainnet.dhall | 4 +- .../src/Jobs/Test/CoverageTearDown.dhall | 4 +- buildkite/src/Jobs/Test/DaemonUnitTest.dhall | 5 +- .../Jobs/Test/DelegationBackendUnitTest.dhall | 5 +- buildkite/src/Jobs/Test/Libp2pUnitTest.dhall | 4 +- .../Jobs/Test/TestnetIntegrationTests.dhall | 6 +- .../src/Jobs/Test/ValidationService.dhall | 4 +- buildkite/src/Monorepo.dhall | 72 +++++++++++-------- buildkite/src/Pipeline/Filter.dhall | 33 +++++++++ buildkite/src/Pipeline/JobSpec.dhall | 6 +- buildkite/src/Pipeline/Stage.dhall | 44 ------------ buildkite/src/Pipeline/Tag.dhall | 63 ++++++++++++++++ buildkite/src/Prepare.dhall | 11 +-- 33 files changed, 295 insertions(+), 133 deletions(-) create mode 100644 buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall rename buildkite/src/Jobs/Release/{MinaToolchainArtifact.dhall => MinaToolchainArtifactBuster.dhall} (75%) create mode 100644 buildkite/src/Pipeline/Filter.dhall delete mode 100644 buildkite/src/Pipeline/Stage.dhall create mode 100644 buildkite/src/Pipeline/Tag.dhall diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 3039f5cd83a..186793ac7e1 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -5,6 +5,9 @@ let S = ../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../Pipeline/Dsl.dhall +let PipelineTag = ../Pipeline/Tag.dhall +let PipelineMode = ../Pipeline/Mode.dhall + let JobSpec = ../Pipeline/JobSpec.dhall let Command = ./Base.dhall @@ -16,13 +19,16 @@ let DebianVersions = ../Constants/DebianVersions.dhall in -let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> +let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> +\(mode: PipelineMode.Type) -> Pipeline.Config::{ spec = JobSpec::{ dirtyWhen = DebianVersions.dirtyWhen debVersion, path = "Release", - name = "MinaArtifact${DebianVersions.capitalName debVersion}" + name = "MinaArtifact${DebianVersions.capitalName debVersion}", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ], + mode = mode }, steps = [ Libp2p.step debVersion, @@ -105,9 +111,9 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion in { - bullseye = pipeline DebianVersions.DebVersion.Bullseye - , buster = pipeline DebianVersions.DebVersion.Buster - , stretch = pipeline DebianVersions.DebVersion.Stretch - , focal = pipeline DebianVersions.DebVersion.Focal - , bionic = pipeline DebianVersions.DebVersion.Bionic + bullseye = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest + , buster = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.PullRequest + , stretch = pipeline DebianVersions.DebVersion.Stretch PipelineMode.Type.PullRequest + , focal = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.PullRequest + , bionic = pipeline DebianVersions.DebVersion.Bionic PipelineMode.Type.PullRequest } diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall index c41140e79cb..aec5fab68d4 100644 --- a/buildkite/src/Jobs/Lint/Fast.dhall +++ b/buildkite/src/Jobs/Lint/Fast.dhall @@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -31,6 +32,7 @@ in Pipeline.build , dirtyWhen = [ S.strictlyStart (S.contains "src/") ] , path = "Lint" , name = "Fast" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/HelmChart.dhall b/buildkite/src/Jobs/Lint/HelmChart.dhall index 2b0e602117c..f0f93b132f1 100644 --- a/buildkite/src/Jobs/Lint/HelmChart.dhall +++ b/buildkite/src/Jobs/Lint/HelmChart.dhall @@ -4,6 +4,7 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Lint", - name = "HelmChart" + name = "HelmChart", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index 83249b2c16e..58ef1e8b28a 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -3,6 +3,7 @@ let Prelude = ../../External/Prelude.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -17,7 +18,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ SelectFiles.everything ], path = "Lint", - name = "Merge" + name = "Merge", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/OCaml.dhall b/buildkite/src/Jobs/Lint/OCaml.dhall index a79f1e00026..3dec98290af 100644 --- a/buildkite/src/Jobs/Lint/OCaml.dhall +++ b/buildkite/src/Jobs/Lint/OCaml.dhall @@ -7,6 +7,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -31,6 +32,7 @@ in Pipeline.build [ dirtyDhallDir, S.strictlyStart (S.contains "src/") ] , path = "Lint" , name = "OCaml" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Rust.dhall b/buildkite/src/Jobs/Lint/Rust.dhall index a551272308d..c099d225646 100644 --- a/buildkite/src/Jobs/Lint/Rust.dhall +++ b/buildkite/src/Jobs/Lint/Rust.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -19,7 +21,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") ], path = "Lint", - name = "Rust" + name = "Rust", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall index 90b69b3f91e..c60772c198b 100644 --- a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -25,7 +27,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Lint", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall index 96fe9d19bf8..a2469a29ef0 100644 --- a/buildkite/src/Jobs/Lint/ValidationService.dhall +++ b/buildkite/src/Jobs/Lint/ValidationService.dhall @@ -4,6 +4,8 @@ let List/map = Prelude.List.map let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let Cmd = ../../Lib/Cmds.dhall let Command = ../../Command/Base.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -52,7 +54,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Lint", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall index 139687b4a5b..28a9268420c 100644 --- a/buildkite/src/Jobs/Lint/Xrefcheck.dhall +++ b/buildkite/src/Jobs/Lint/Xrefcheck.dhall @@ -4,6 +4,8 @@ let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -23,7 +25,8 @@ Pipeline.build SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml") ], path = "Lint", - name = "Xrefcheck" + name = "Xrefcheck", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/BotArtifact.dhall b/buildkite/src/Jobs/Release/BotArtifact.dhall index cbdebb47519..30225567de2 100644 --- a/buildkite/src/Jobs/Release/BotArtifact.dhall +++ b/buildkite/src/Jobs/Release/BotArtifact.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -27,19 +29,10 @@ Pipeline.build S.strictlyStart (S.contains "frontend/bot") ], path = "Release", - name = "BotArtifact" + name = "BotArtifact", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] }, steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.run "echo export MINA_VERSION=$(cat frontend/bot/package.json | jq '.version') > BOT_DEPLOY_ENV" - ], - label = "Setup o1bot docker image deploy environment", - key = "setup-deploy-env", - target = Size.Small, - artifact_paths = [ S.contains "frontend/bot/*.json" ] - }, DockerImage.generateStep spec ] } diff --git a/buildkite/src/Jobs/Release/HelmRelease.dhall b/buildkite/src/Jobs/Release/HelmRelease.dhall index 7fa74079dda..8e843171ab7 100644 --- a/buildkite/src/Jobs/Release/HelmRelease.dhall +++ b/buildkite/src/Jobs/Release/HelmRelease.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -21,7 +23,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Release", - name = "HelmRelease" + name = "HelmRelease", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall index f8b657359fd..252cd012e22 100644 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -27,7 +29,8 @@ Pipeline.build S.strictlyStart (S.contains "frontend/leaderboard") ], path = "Release", - name = "LeaderboardArtifact" + name = "LeaderboardArtifact", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall new file mode 100644 index 00000000000..0e2b4f198ea --- /dev/null +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -0,0 +1,50 @@ +let Prelude = ../../External/Prelude.dhall + +let Cmd = ../../Lib/Cmds.dhall +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall +let Size = ../../Command/Size.dhall +let DockerImage = ../../Command/DockerImage.dhall +let DockerLogin = ../../Command/DockerLogin/Type.dhall + + +in + +Pipeline.build + Pipeline.Config::{ + spec = + JobSpec::{ + dirtyWhen = [ + S.strictlyStart (S.contains "dockerfiles/stages/1-"), + S.strictlyStart (S.contains "dockerfiles/stages/2-"), + S.strictlyStart (S.contains "dockerfiles/stages/3-"), + S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"), + S.strictly (S.contains "opam.export"), + -- Rust version has changed + S.strictlyEnd (S.contains "rust-toolchain.toml") + ], + path = "Release", + name = "MinaToolchainArtifactBullseye", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] + }, + steps = [ + + -- mina-toolchain Debian 11 "Bullseye" Toolchain + let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ + service="mina-toolchain", + deb_codename="bullseye", + extra_args="--no-cache", + step_key="toolchain-bullseye-docker-image" + } + + in + + DockerImage.generateStep toolchainBullseyeSpec + + ] + } \ No newline at end of file diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall similarity index 75% rename from buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall rename to buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall index 3dc7b11e73c..04fc6787889 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall @@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall +let PipelineMode = ../../Pipeline/Mode.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -26,22 +28,12 @@ Pipeline.build S.strictly (S.contains "opam.export") ], path = "Release", - name = "MinaToolchainArtifact" + name = "MinaToolchainArtifactBuster", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ], + mode = PipelineMode.Type.Stable }, steps = [ - -- mina-toolchain Debian 11 "Bullseye" Toolchain - let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - service="mina-toolchain", - deb_codename="bullseye", - extra_args="--no-cache", - step_key="toolchain-bullseye-docker-image" - } - - in - - DockerImage.generateStep toolchainBullseyeSpec, - -- mina-toolchain Debian 10 "Buster" Toolchain let toolchainBusterSpec = DockerImage.ReleaseSpec::{ service="mina-toolchain", diff --git a/buildkite/src/Jobs/Release/TestnetAlerts.dhall b/buildkite/src/Jobs/Release/TestnetAlerts.dhall index 06125455a8c..e444c4e7a33 100644 --- a/buildkite/src/Jobs/Release/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Release/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -24,7 +26,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Release", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/TraceTool.dhall b/buildkite/src/Jobs/Release/TraceTool.dhall index d4a0e4997f4..0d6770523e5 100644 --- a/buildkite/src/Jobs/Release/TraceTool.dhall +++ b/buildkite/src/Jobs/Release/TraceTool.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -20,7 +22,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/TraceTool") ], path = "Release", - name = "TraceTool" + name = "TraceTool", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall index 0be4cc06b35..d746d0381f1 100644 --- a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall @@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -26,6 +27,7 @@ Pipeline.build ] , path = "Test" , name = "ArchiveNodeUnitTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = let outerDir : Text = diff --git a/buildkite/src/Jobs/Test/CheckDhall.dhall b/buildkite/src/Jobs/Test/CheckDhall.dhall index 34f8c78476e..18cace107aa 100644 --- a/buildkite/src/Jobs/Test/CheckDhall.dhall +++ b/buildkite/src/Jobs/Test/CheckDhall.dhall @@ -3,6 +3,7 @@ let D = S.PathPattern let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/generate-jobs" "sh" ], path = "Test", - name = "CheckDhall" + name = "CheckDhall", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index be248e22210..4583a5b38bc 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall @@ -18,7 +19,8 @@ in Pipeline.build Pipeline.Config::{ S.strictly (S.contains "Makefile") ], path = "Test", - name = "CheckGraphQLSchema" + name = "CheckGraphQLSchema", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ CheckGraphQLSchema.step dependsOn diff --git a/buildkite/src/Jobs/Test/CompareSignatures.dhall b/buildkite/src/Jobs/Test/CompareSignatures.dhall index 61b01d6599a..86d40795bcc 100644 --- a/buildkite/src/Jobs/Test/CompareSignatures.dhall +++ b/buildkite/src/Jobs/Test/CompareSignatures.dhall @@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall let WithCargo = ../../Command/WithCargo.dhall @@ -20,6 +21,7 @@ Pipeline.build ] , path = "Test" , name = "CompareSignatures" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall index cbb647c5657..ab805f6cc6e 100644 --- a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall @@ -17,7 +18,8 @@ in Pipeline.build Pipeline.Config::{ S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh" ], path = "Test", - name = "ConnectToMainnet" + name = "ConnectToMainnet", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ ConnectToTestnet.step dependsOn diff --git a/buildkite/src/Jobs/Test/CoverageTearDown.dhall b/buildkite/src/Jobs/Test/CoverageTearDown.dhall index bba020d8025..aa10f2a5ff8 100644 --- a/buildkite/src/Jobs/Test/CoverageTearDown.dhall +++ b/buildkite/src/Jobs/Test/CoverageTearDown.dhall @@ -8,7 +8,7 @@ let Size = ../../Command/Size.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall -let PipelineStage = ../../Pipeline/Stage.dhall +let PipelineTag = ../../Pipeline/Tag.dhall in Pipeline.build Pipeline.Config::{ @@ -16,7 +16,7 @@ in Pipeline.build JobSpec::{ dirtyWhen = [ S.everything ], path = "Test", - stage = PipelineStage.Type.TearDown, + tags = [ PipelineTag.Type.TearDown ], name = "CoverageTearDown" } , steps = [ diff --git a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall index 8e234daed36..43d8d2af793 100644 --- a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -42,7 +44,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "DaemonUnitTest" + name = "DaemonUnitTest", + tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/lib" Size.XLarge diff --git a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall index 03e9d4293aa..248df0d4b77 100644 --- a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall @@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -22,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/DelegationBackendUnitTest" "dhall" ], path = "Test", - name = "DelegationBackendUnitTest" + name = "DelegationBackendUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall index e22384dc2ba..76098a7f444 100644 --- a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall +++ b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall @@ -4,6 +4,7 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall" ], path = "Test", - name = "Libp2pUnitTest" + name = "Libp2pUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 1c1a28bc526..3185faca713 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -2,6 +2,8 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineMode = ../../Pipeline/Mode.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let TestExecutive = ../../Command/TestExecutive.dhall @@ -23,7 +25,9 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet") ], path = "Test", - name = "TestnetIntegrationTests" + name = "TestnetIntegrationTests", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ], + mode = PipelineMode.Type.Stable }, steps = [ TestExecutive.build "integration_tests", diff --git a/buildkite/src/Jobs/Test/ValidationService.dhall b/buildkite/src/Jobs/Test/ValidationService.dhall index 1c2ee6fa571..cce41331433 100644 --- a/buildkite/src/Jobs/Test/ValidationService.dhall +++ b/buildkite/src/Jobs/Test/ValidationService.dhall @@ -1,6 +1,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -15,7 +16,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Test", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index 64783a0582f..3281333b8e6 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -11,12 +11,14 @@ let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall let PipelineMode = ./Pipeline/Mode.dhall -let PipelineStage = ./Pipeline/Stage.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let include_tags = env:BUILDKITE_PIPELINE_TAGS_INCL as Text ? "Fast" +let exclude_tags = env:BUILDKITE_PIPELINE_TAGS_EXCL as Text ? "" let jobs : List JobSpec.Type = List/map @@ -33,34 +35,36 @@ let prefixCommands = [ -- Run a job if we touched a dirty path -let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(targetMode: Text) -> +let commands: PipelineFilter.Type -> PipelineMode.Type -> List Cmd.Type = \(filter: PipelineFilter.Type) -> \(mode: PipelineMode.Type) -> Prelude.List.map JobSpec.Type Cmd.Type (\(job: JobSpec.Type) -> let jobMode = PipelineMode.capitalName job.mode - let jobStage = PipelineStage.capitalName job.stage - + let targetMode = PipelineMode.capitalName mode + let targetTags = PipelineFilter.tags filter + let filter = PipelineFilter.show filter + let isIncluded = Prelude.Bool.show (PipelineTag.contains job.tags targetTags) let dirtyWhen = SelectFiles.compile job.dirtyWhen let trigger = triggerCommand "src/Jobs/${job.path}/${job.name}.dhall" let pipelineHandlers = { PullRequest = '' if [ "${targetMode}" == "PullRequest" ]; then - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then if (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then echo "Triggering ${job.name} for reason:" cat _computed_diff.txt | egrep '${dirtyWhen}' ${Cmd.format trigger} fi else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi else - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then echo "Triggering ${job.name} because this is a stable buildkite run" ${Cmd.format trigger} else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi fi '', @@ -68,11 +72,11 @@ let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(target if [ "${targetMode}" == "PullRequest" ]; then echo "Skipping ${job.name} because this is a PR buildkite run" else - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then echo "Triggering ${job.name} because this is a stable buildkite run" ${Cmd.format trigger} else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi fi '' @@ -81,23 +85,29 @@ let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(target ) jobs -in Pipeline.build Pipeline.Config::{ - spec = JobSpec::{ - name = "monorepo-triage", - -- TODO: Clean up this code so we don't need an unused dirtyWhen here - dirtyWhen = [ SelectFiles.everything ] - }, - steps = [ - Command.build - Command.Config::{ - commands = prefixCommands # (commands stage mode), - label = "Monorepo triage ${stage}", - key = "cmds-${stage}", - target = Size.Small, - docker = Some Docker::{ - image = (./Constants/ContainerImages.dhall).toolchainBase, - environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] - } - } - ] -} +in + +(\(args : { filter : PipelineFilter.Type, mode: PipelineMode.Type }) -> + let pipelineType = Pipeline.build Pipeline.Config::{ + spec = JobSpec::{ + name = "monorepo-triage-${PipelineFilter.show args.filter}", + -- TODO: Clean up this code so we don't need an unused dirtyWhen here + dirtyWhen = [ SelectFiles.everything ] + }, + steps = [ + Command.build + Command.Config::{ + commands = prefixCommands # (commands args.filter args.mode), + label = "Monorepo triage ${PipelineFilter.show args.filter}", + key = "cmds-${PipelineFilter.show args.filter}", + target = Size.Small, + docker = Some Docker::{ + image = (./Constants/ContainerImages.dhall).toolchainBase, + environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] + } + } + ] + } + in pipelineType.pipeline +) + diff --git a/buildkite/src/Pipeline/Filter.dhall b/buildkite/src/Pipeline/Filter.dhall new file mode 100644 index 00000000000..fc9cd7dd5ac --- /dev/null +++ b/buildkite/src/Pipeline/Filter.dhall @@ -0,0 +1,33 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let Tag = ./Tag.dhall + +let Filter : Type = < FastOnly | Long | LongAndVeryLong | TearDownOnly | AllTests > + +let tags: Filter -> List Tag.Type = \(filter: Filter) -> + merge { + FastOnly = [ Tag.Type.Fast ] + , LongAndVeryLong = [ Tag.Type.Long, Tag.Type.VeryLong ] + , Long = [ Tag.Type.Long ] + , TearDownOnly = [ Tag.Type.TearDown ] + , AllTests = [ Tag.Type.Lint, Tag.Type.Release, Tag.Type.Test ] + } filter + +let show: Filter -> Text = \(filter: Filter) -> + merge { + FastOnly = "FastOnly" + , LongAndVeryLong = "LongAndVeryLong" + , Long = "Long" + , TearDownOnly = "TearDownOnly" + , AllTests = "AllTests" + } filter + +in +{ + Type = Filter, + tags = tags, + show = show +} \ No newline at end of file diff --git a/buildkite/src/Pipeline/JobSpec.dhall b/buildkite/src/Pipeline/JobSpec.dhall index 187835c7d90..ea0373a0c2d 100644 --- a/buildkite/src/Pipeline/JobSpec.dhall +++ b/buildkite/src/Pipeline/JobSpec.dhall @@ -1,6 +1,6 @@ let SelectFiles = ../Lib/SelectFiles.dhall let PipelineMode = ./Mode.dhall -let PipelineStage = ./Stage.dhall +let PipelineTag = ./Tag.dhall in @@ -11,12 +11,12 @@ in path: Text, name: Text, mode: PipelineMode.Type, - stage: PipelineStage.Type, + tags: List PipelineTag.Type, dirtyWhen: List SelectFiles.Type }, default = { path = ".", mode = PipelineMode.Type.PullRequest, - stage = PipelineStage.Type.Test + tags = [ PipelineTag.Type.Fast ] } } diff --git a/buildkite/src/Pipeline/Stage.dhall b/buildkite/src/Pipeline/Stage.dhall deleted file mode 100644 index fffad54c7a2..00000000000 --- a/buildkite/src/Pipeline/Stage.dhall +++ /dev/null @@ -1,44 +0,0 @@ --- Mode defines pipeline stages --- --- A pipeline in order to be faster and more cost efficient can have up to 3 stages --- Between each stages there is a '- wait' step defined which cause buildkite to wait --- for ALL jobs to complete before running any job from next stage. --- Current design defines three stages: --- - Stage 1 -> contains fastest and most independent jobs which are supposed to provide quickest feedback possible --- - Stage 2 -> contains heavy jobs that should be run only on clean code (no merges issues or lints problems) --- - Tear down -> should contains all clean up or reporting jobs. For example test coverage gathering - -let Prelude = ../External/Prelude.dhall - -let Stage : Type = < Test | TearDown > - -let toNatural: Stage -> Natural = \(stage: Stage) -> - merge { - Test = 1 - , TearDown = 2 - } stage - -let equal: Stage -> Stage -> Bool = \(left: Stage) -> \(right: Stage) -> - Prelude.Natural.equal (toNatural left) (toNatural right) - -let capitalName = \(stage : Stage) -> - merge { - Test = "Test" - , TearDown = "TearDown" - } stage - -let lowerName = \(stage : Stage) -> - merge { - Test = "test" - , TearDown = "tearDown" - } stage - - -in -{ - Type = Stage, - capitalName = capitalName, - lowerName = lowerName, - toNatural = toNatural, - equal = equal -} \ No newline at end of file diff --git a/buildkite/src/Pipeline/Tag.dhall b/buildkite/src/Pipeline/Tag.dhall new file mode 100644 index 00000000000..b6018062e67 --- /dev/null +++ b/buildkite/src/Pipeline/Tag.dhall @@ -0,0 +1,63 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let List/any = Prelude.List.any + +let Tag : Type = < Fast | Long | VeryLong | TearDown | Lint | Release | Test > + +let toNatural: Tag -> Natural = \(tag: Tag) -> + merge { + Fast = 1 + , Long = 2 + , VeryLong = 3 + , TearDown = 4 + , Lint = 5 + , Release = 6 + , Test = 7 + } tag + +let equal: Tag -> Tag -> Bool = \(left: Tag) -> \(right: Tag) -> + Prelude.Natural.equal (toNatural left) (toNatural right) + + +let hasAny: Tag -> List Tag -> Bool = \(input: Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> equal x input ) tags + +let contains: List Tag -> List Tag -> Bool = \(input: List Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> hasAny x tags ) input + +let capitalName = \(tag : Tag) -> + merge { + Fast = "Fast" + , Long = "Long" + , VeryLong = "VeryLong" + , TearDown = "TearDown" + , Lint = "Lint" + , Release = "Release" + , Test = "Test" + } tag + +let lowerName = \(tag : Tag) -> + merge { + Fast = "fast" + , Long = "long" + , VeryLong = "veryLong" + , TearDown = "tearDown" + , Lint = "lint" + , Release = "release" + , Test = "test" + } tag + + +in +{ + Type = Tag, + capitalName = capitalName, + lowerName = lowerName, + toNatural = toNatural, + equal = equal, + hasAny = hasAny, + contains = contains +} \ No newline at end of file diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index 101e22be770..5b78ff24369 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -8,11 +8,14 @@ let Command = ./Command/Base.dhall let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall +let PipelineMode = ./Pipeline/Mode.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let filter = env:BUILDKITE_PIPELINE_FILTER as Text ? "FastOnly" let config : Pipeline.Config.Type = Pipeline.Config::{ spec = JobSpec::{ @@ -24,12 +27,12 @@ let config : Pipeline.Config.Type = Pipeline.Config::{ Command.build Command.Config::{ commands = [ Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}", - Cmd.run "export BUILDKITE_PIPELINE_STAGE=${stage}", + Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}", Cmd.run "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall", - triggerCommand "src/Monorepo.dhall" + Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" ], label = "Prepare monorepo triage", - key = "monorepo-${stage}", + key = "monorepo-${mode}-${filter}", target = Size.Small, docker = Some Docker::{ image = (./Constants/ContainerImages.dhall).toolchainBase, From b76c4343d8b28d4114d5f014c1795eb55650daa1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 13:54:09 +0200 Subject: [PATCH 2/5] Revert "Auxiliary commit to revert individual files from b2cc7de959a2b5e5d19049b3dcbddc25ed94aed6" This reverts commit 7ef71cad14c3d955fbbbce202e791ac1264be165. --- buildkite/src/Jobs/Release/BotArtifact.dhall | 15 +++++++++++---- buildkite/src/Jobs/Test/ConnectToMainnet.dhall | 4 +--- .../src/Jobs/Test/TestnetIntegrationTests.dhall | 6 +----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/buildkite/src/Jobs/Release/BotArtifact.dhall b/buildkite/src/Jobs/Release/BotArtifact.dhall index 30225567de2..cbdebb47519 100644 --- a/buildkite/src/Jobs/Release/BotArtifact.dhall +++ b/buildkite/src/Jobs/Release/BotArtifact.dhall @@ -5,8 +5,6 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall - let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -29,10 +27,19 @@ Pipeline.build S.strictlyStart (S.contains "frontend/bot") ], path = "Release", - name = "BotArtifact", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] + name = "BotArtifact" }, steps = [ + Command.build + Command.Config::{ + commands = [ + Cmd.run "echo export MINA_VERSION=$(cat frontend/bot/package.json | jq '.version') > BOT_DEPLOY_ENV" + ], + label = "Setup o1bot docker image deploy environment", + key = "setup-deploy-env", + target = Size.Small, + artifact_paths = [ S.contains "frontend/bot/*.json" ] + }, DockerImage.generateStep spec ] } diff --git a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall index ab805f6cc6e..cbb647c5657 100644 --- a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall @@ -2,7 +2,6 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall @@ -18,8 +17,7 @@ in Pipeline.build Pipeline.Config::{ S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh" ], path = "Test", - name = "ConnectToMainnet", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + name = "ConnectToMainnet" }, steps = [ ConnectToTestnet.step dependsOn diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 3185faca713..1c1a28bc526 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -2,8 +2,6 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall -let PipelineTag = ../../Pipeline/Tag.dhall let TestExecutive = ../../Command/TestExecutive.dhall @@ -25,9 +23,7 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet") ], path = "Test", - name = "TestnetIntegrationTests", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ], - mode = PipelineMode.Type.Stable + name = "TestnetIntegrationTests" }, steps = [ TestExecutive.build "integration_tests", From 4df9df26f8dd042e8503b02b40fe43caff5ec61c Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 13:55:17 +0200 Subject: [PATCH 3/5] Revert "Auxiliary commit to revert individual files from b2cc7de959a2b5e5d19049b3dcbddc25ed94aed6" This reverts commit edf6773ba52b81c9856e41b67e8f2b42bc43a5fd. --- ...ster.dhall => MinaToolchainArtifact.dhall} | 18 +++++-- .../MinaToolchainArtifactBullseye.dhall | 50 ------------------- .../src/Jobs/Test/CompareSignatures.dhall | 2 - 3 files changed, 13 insertions(+), 57 deletions(-) rename buildkite/src/Jobs/Release/{MinaToolchainArtifactBuster.dhall => MinaToolchainArtifact.dhall} (75%) delete mode 100644 buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall similarity index 75% rename from buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall rename to buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall index 04fc6787889..3dc7b11e73c 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall @@ -4,8 +4,6 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall -let PipelineMode = ../../Pipeline/Mode.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -28,12 +26,22 @@ Pipeline.build S.strictly (S.contains "opam.export") ], path = "Release", - name = "MinaToolchainArtifactBuster", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ], - mode = PipelineMode.Type.Stable + name = "MinaToolchainArtifact" }, steps = [ + -- mina-toolchain Debian 11 "Bullseye" Toolchain + let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ + service="mina-toolchain", + deb_codename="bullseye", + extra_args="--no-cache", + step_key="toolchain-bullseye-docker-image" + } + + in + + DockerImage.generateStep toolchainBullseyeSpec, + -- mina-toolchain Debian 10 "Buster" Toolchain let toolchainBusterSpec = DockerImage.ReleaseSpec::{ service="mina-toolchain", diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall deleted file mode 100644 index 0e2b4f198ea..00000000000 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ /dev/null @@ -1,50 +0,0 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Command = ../../Command/Base.dhall -let Size = ../../Command/Size.dhall -let DockerImage = ../../Command/DockerImage.dhall -let DockerLogin = ../../Command/DockerLogin/Type.dhall - - -in - -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "dockerfiles/stages/1-"), - S.strictlyStart (S.contains "dockerfiles/stages/2-"), - S.strictlyStart (S.contains "dockerfiles/stages/3-"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"), - S.strictly (S.contains "opam.export"), - -- Rust version has changed - S.strictlyEnd (S.contains "rust-toolchain.toml") - ], - path = "Release", - name = "MinaToolchainArtifactBullseye", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] - }, - steps = [ - - -- mina-toolchain Debian 11 "Bullseye" Toolchain - let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - service="mina-toolchain", - deb_codename="bullseye", - extra_args="--no-cache", - step_key="toolchain-bullseye-docker-image" - } - - in - - DockerImage.generateStep toolchainBullseyeSpec - - ] - } \ No newline at end of file diff --git a/buildkite/src/Jobs/Test/CompareSignatures.dhall b/buildkite/src/Jobs/Test/CompareSignatures.dhall index 86d40795bcc..61b01d6599a 100644 --- a/buildkite/src/Jobs/Test/CompareSignatures.dhall +++ b/buildkite/src/Jobs/Test/CompareSignatures.dhall @@ -2,7 +2,6 @@ let Prelude = ../../External/Prelude.dhall let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall let WithCargo = ../../Command/WithCargo.dhall @@ -21,7 +20,6 @@ Pipeline.build ] , path = "Test" , name = "CompareSignatures" - , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = [ Command.build From a5807e94f1637ace79d0b0edda62663771bbfd01 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 13:56:31 +0200 Subject: [PATCH 4/5] Revert "Auxiliary commit to revert individual files from b2cc7de959a2b5e5d19049b3dcbddc25ed94aed6" This reverts commit 04a153b3f5f0587e531c3c8bb7688fc07d40e27a. --- buildkite/src/Command/MinaArtifact.dhall | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 186793ac7e1..3039f5cd83a 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -5,9 +5,6 @@ let S = ../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../Pipeline/Dsl.dhall -let PipelineTag = ../Pipeline/Tag.dhall -let PipelineMode = ../Pipeline/Mode.dhall - let JobSpec = ../Pipeline/JobSpec.dhall let Command = ./Base.dhall @@ -19,16 +16,13 @@ let DebianVersions = ../Constants/DebianVersions.dhall in -let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> -\(mode: PipelineMode.Type) -> +let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> Pipeline.Config::{ spec = JobSpec::{ dirtyWhen = DebianVersions.dirtyWhen debVersion, path = "Release", - name = "MinaArtifact${DebianVersions.capitalName debVersion}", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ], - mode = mode + name = "MinaArtifact${DebianVersions.capitalName debVersion}" }, steps = [ Libp2p.step debVersion, @@ -111,9 +105,9 @@ let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config in { - bullseye = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest - , buster = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.PullRequest - , stretch = pipeline DebianVersions.DebVersion.Stretch PipelineMode.Type.PullRequest - , focal = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.PullRequest - , bionic = pipeline DebianVersions.DebVersion.Bionic PipelineMode.Type.PullRequest + bullseye = pipeline DebianVersions.DebVersion.Bullseye + , buster = pipeline DebianVersions.DebVersion.Buster + , stretch = pipeline DebianVersions.DebVersion.Stretch + , focal = pipeline DebianVersions.DebVersion.Focal + , bionic = pipeline DebianVersions.DebVersion.Bionic } From 732a2f0cf0f98860a07c2b2e53c59cf58124afc9 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 16:56:08 +0200 Subject: [PATCH 5/5] Revert "Auxiliary commit to revert individual files from b2cc7de959a2b5e5d19049b3dcbddc25ed94aed6" This reverts commit 623d89691d49da86a7014637a08c3b29ad89e1d1. --- buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index 4583a5b38bc..be248e22210 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -2,7 +2,6 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall @@ -19,8 +18,7 @@ in Pipeline.build Pipeline.Config::{ S.strictly (S.contains "Makefile") ], path = "Test", - name = "CheckGraphQLSchema", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + name = "CheckGraphQLSchema" }, steps = [ CheckGraphQLSchema.step dependsOn