From 139763c9e0fd434c8689e5f0d568b589de6adaf1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Sep 2023 22:51:46 +0200 Subject: [PATCH] once again --- buildkite/src/Command/MinaArtifact.dhall | 16 ++-- buildkite/src/Jobs/Lint/Fast.dhall | 5 ++ buildkite/src/Jobs/Lint/HelmChart.dhall | 4 +- buildkite/src/Jobs/Lint/Merge.dhall | 4 +- buildkite/src/Jobs/Lint/OCaml.dhall | 2 + buildkite/src/Jobs/Lint/Rust.dhall | 5 +- buildkite/src/Jobs/Lint/TestnetAlerts.dhall | 5 +- .../src/Jobs/Lint/ValidationService.dhall | 5 +- buildkite/src/Jobs/Lint/Xrefcheck.dhall | 5 +- buildkite/src/Jobs/Release/HelmRelease.dhall | 5 +- .../Release/ItnOrchestratorArtifact.dhall | 5 +- .../Jobs/Release/LeaderboardArtifact.dhall | 5 +- .../MinaToolchainArtifactBullseye.dhall | 50 +++++++++++ ...hall => MinaToolchainArtifactBuster.dhall} | 18 ++-- .../src/Jobs/Release/TestnetAlerts.dhall | 5 +- buildkite/src/Jobs/Release/TraceTool.dhall | 5 +- .../src/Jobs/Test/ArchiveNodeUnitTest.dhall | 2 + buildkite/src/Jobs/Test/CheckDhall.dhall | 4 +- .../src/Jobs/Test/CheckGraphQLSchema.dhall | 4 +- .../src/Jobs/Test/ConnectToBerkeley.dhall | 4 +- .../src/Jobs/Test/CoverageTearDown.dhall | 4 +- buildkite/src/Jobs/Test/DaemonUnitTest.dhall | 5 +- .../Jobs/Test/DelegationBackendUnitTest.dhall | 5 +- buildkite/src/Jobs/Test/FuzzyZkappTest.dhall | 3 + buildkite/src/Jobs/Test/Libp2pUnitTest.dhall | 4 +- .../Jobs/Test/RosettaIntegrationTests.dhall | 4 +- buildkite/src/Jobs/Test/RosettaUnitTest.dhall | 4 +- .../src/Jobs/Test/RunSnarkProfiler.dhall | 4 +- buildkite/src/Jobs/Test/SingleNodeTest.dhall | 5 +- buildkite/src/Jobs/Test/SnarkyJSTest.dhall | 4 +- .../Jobs/Test/TestnetIntegrationTests.dhall | 4 +- .../Test/TestnetIntegrationTestsLong.dhall | 4 +- .../src/Jobs/Test/ValidationService.dhall | 4 +- buildkite/src/Jobs/Test/ZkappMetrics.dhall | 2 + .../src/Jobs/Test/ZkappTestToolUnitTest.dhall | 4 +- .../src/Jobs/Test/ZkappsExamplesTest.dhall | 4 +- buildkite/src/Monorepo.dhall | 89 ++++++++----------- buildkite/src/Pipeline/Filter.dhall | 31 +++++++ buildkite/src/Pipeline/JobSpec.dhall | 6 +- buildkite/src/Pipeline/Stage.dhall | 44 --------- buildkite/src/Pipeline/Tag.dhall | 63 +++++++++++++ buildkite/src/Prepare.dhall | 12 ++- 42 files changed, 316 insertions(+), 151 deletions(-) create mode 100644 buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall rename buildkite/src/Jobs/Release/{MinaToolchainArtifact.dhall => MinaToolchainArtifactBuster.dhall} (77%) create mode 100644 buildkite/src/Pipeline/Filter.dhall delete mode 100644 buildkite/src/Pipeline/Stage.dhall create mode 100644 buildkite/src/Pipeline/Tag.dhall diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 89ed25e662b2..9be8fa92b7c7 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -5,6 +5,9 @@ let S = ../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../Pipeline/Dsl.dhall +let PipelineTag = ../Pipeline/Tag.dhall +let PipelineMode = ../Pipeline/Mode.dhall + let JobSpec = ../Pipeline/JobSpec.dhall let Command = ./Base.dhall @@ -16,13 +19,16 @@ let DebianVersions = ../Constants/DebianVersions.dhall in -let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> +let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) -> +\(mode: PipelineMode.Type) -> Pipeline.Config::{ spec = JobSpec::{ dirtyWhen = DebianVersions.dirtyWhen debVersion, path = "Release", - name = "MinaArtifact${DebianVersions.capitalName debVersion}" + name = "MinaArtifact${DebianVersions.capitalName debVersion}", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ], + mode = mode }, steps = [ Libp2p.step debVersion, @@ -119,7 +125,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion in { - bullseye = pipeline DebianVersions.DebVersion.Bullseye - , buster = pipeline DebianVersions.DebVersion.Buster - , focal = pipeline DebianVersions.DebVersion.Focal + bullseye = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest + , buster = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.PullRequest + , focal = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.PullRequest } diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall index 7c4649f975b1..36043a472dc7 100644 --- a/buildkite/src/Jobs/Lint/Fast.dhall +++ b/buildkite/src/Jobs/Lint/Fast.dhall @@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -18,6 +19,8 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Size = ../../Command/Size.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + let commands = [ Cmd.run "./scripts/lint_codeowners.sh" , Cmd.run "./scripts/lint_rfcs.sh" @@ -38,6 +41,7 @@ in Pipeline.build ] , path = "Lint" , name = "Fast" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build @@ -47,6 +51,7 @@ in Pipeline.build "Fast lint steps; CODEOWNERs, RFCs, Check Snarky & Proof-Systems submodules, Preprocessor Deps" , key = "lint" , target = Size.Small + , soft_fail = Some (B/SoftFail.Boolean True) , docker = Some Docker::{ , image = (../../Constants/ContainerImages.dhall).toolchainBase } diff --git a/buildkite/src/Jobs/Lint/HelmChart.dhall b/buildkite/src/Jobs/Lint/HelmChart.dhall index 2b0e602117cc..f0f93b132f12 100644 --- a/buildkite/src/Jobs/Lint/HelmChart.dhall +++ b/buildkite/src/Jobs/Lint/HelmChart.dhall @@ -4,6 +4,7 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Lint", - name = "HelmChart" + name = "HelmChart", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index f6d06707e46d..eae39a7b21a3 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -4,6 +4,7 @@ let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -20,7 +21,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ SelectFiles.everything ], path = "Lint", - name = "Merge" + name = "Merge", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/OCaml.dhall b/buildkite/src/Jobs/Lint/OCaml.dhall index a79f1e00026e..3dec98290af5 100644 --- a/buildkite/src/Jobs/Lint/OCaml.dhall +++ b/buildkite/src/Jobs/Lint/OCaml.dhall @@ -7,6 +7,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -31,6 +32,7 @@ in Pipeline.build [ dirtyDhallDir, S.strictlyStart (S.contains "src/") ] , path = "Lint" , name = "OCaml" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Rust.dhall b/buildkite/src/Jobs/Lint/Rust.dhall index a551272308d7..c099d2256462 100644 --- a/buildkite/src/Jobs/Lint/Rust.dhall +++ b/buildkite/src/Jobs/Lint/Rust.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -19,7 +21,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") ], path = "Lint", - name = "Rust" + name = "Rust", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall index 90b69b3f91ea..c60772c198b6 100644 --- a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -25,7 +27,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Lint", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall index 96fe9d19bf85..a2469a29ef03 100644 --- a/buildkite/src/Jobs/Lint/ValidationService.dhall +++ b/buildkite/src/Jobs/Lint/ValidationService.dhall @@ -4,6 +4,8 @@ let List/map = Prelude.List.map let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let Cmd = ../../Lib/Cmds.dhall let Command = ../../Command/Base.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -52,7 +54,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Lint", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall index 139687b4a5b5..28a9268420cf 100644 --- a/buildkite/src/Jobs/Lint/Xrefcheck.dhall +++ b/buildkite/src/Jobs/Lint/Xrefcheck.dhall @@ -4,6 +4,8 @@ let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -23,7 +25,8 @@ Pipeline.build SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml") ], path = "Lint", - name = "Xrefcheck" + name = "Xrefcheck", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/HelmRelease.dhall b/buildkite/src/Jobs/Release/HelmRelease.dhall index 7fa74079dda1..8e843171ab7d 100644 --- a/buildkite/src/Jobs/Release/HelmRelease.dhall +++ b/buildkite/src/Jobs/Release/HelmRelease.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -21,7 +23,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Release", - name = "HelmRelease" + name = "HelmRelease", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall index e1232b6278e2..45d262463100 100644 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -29,7 +31,8 @@ Pipeline.build S.strictlyStart (S.contains "src/app/itn_orchestrator") ], path = "Release", - name = "ItnOrchestratorArtifact" + name = "ItnOrchestratorArtifact", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ DockerImage.generateStep spec diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall index f8b657359fd5..0c407d11fdf0 100644 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -27,7 +29,8 @@ Pipeline.build S.strictlyStart (S.contains "frontend/leaderboard") ], path = "Release", - name = "LeaderboardArtifact" + name = "LeaderboardArtifact", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall new file mode 100644 index 000000000000..ee885c40e08a --- /dev/null +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -0,0 +1,50 @@ +let Prelude = ../../External/Prelude.dhall + +let Cmd = ../../Lib/Cmds.dhall +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall +let Size = ../../Command/Size.dhall +let DockerImage = ../../Command/DockerImage.dhall +let DockerLogin = ../../Command/DockerLogin/Type.dhall + + +in + +Pipeline.build + Pipeline.Config::{ + spec = + JobSpec::{ + dirtyWhen = [ + S.strictlyStart (S.contains "dockerfiles/stages/1-"), + S.strictlyStart (S.contains "dockerfiles/stages/2-"), + S.strictlyStart (S.contains "dockerfiles/stages/3-"), + S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"), + S.strictly (S.contains "opam.export"), + -- Rust version has changed + S.strictlyEnd (S.contains "rust-toolchain.toml") + ], + path = "Release", + name = "MinaToolchainArtifactBullseye", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] + }, + steps = [ + + -- mina-toolchain Debian 11 "Bullseye" Toolchain + let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ + service="mina-toolchain", + deb_codename="bullseye", + extra_args="--no-cache", + step_key="toolchain-bullseye-docker-image" + } + + in + + DockerImage.generateStep toolchainBullseyeSpec + + ] + } \ No newline at end of file diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall similarity index 77% rename from buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall rename to buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall index 3780943bb12e..4f911e86d7ea 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall @@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall +let PipelineMode = ../../Pipeline/Mode.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -28,22 +30,12 @@ Pipeline.build S.strictlyEnd (S.contains "rust-toolchain.toml") ], path = "Release", - name = "MinaToolchainArtifact" + name = "MinaToolchainArtifactBuster", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ], + mode = PipelineMode.Type.Stable }, steps = [ - -- mina-toolchain Debian 11 "Bullseye" Toolchain - let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - service="mina-toolchain", - deb_codename="bullseye", - extra_args="--no-cache", - step_key="toolchain-bullseye-docker-image" - } - - in - - DockerImage.generateStep toolchainBullseyeSpec, - -- mina-toolchain Debian 10 "Buster" Toolchain let toolchainBusterSpec = DockerImage.ReleaseSpec::{ service="mina-toolchain", diff --git a/buildkite/src/Jobs/Release/TestnetAlerts.dhall b/buildkite/src/Jobs/Release/TestnetAlerts.dhall index 06125455a8ca..e444c4e7a33c 100644 --- a/buildkite/src/Jobs/Release/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Release/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -24,7 +26,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Release", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/TraceTool.dhall b/buildkite/src/Jobs/Release/TraceTool.dhall index d4a0e4997f45..0d6770523e56 100644 --- a/buildkite/src/Jobs/Release/TraceTool.dhall +++ b/buildkite/src/Jobs/Release/TraceTool.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -20,7 +22,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/TraceTool") ], path = "Release", - name = "TraceTool" + name = "TraceTool", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall index e92cf6be77b3..16fa19b1cc18 100644 --- a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall @@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -27,6 +28,7 @@ Pipeline.build ] , path = "Test" , name = "ArchiveNodeUnitTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = let outerDir : Text = diff --git a/buildkite/src/Jobs/Test/CheckDhall.dhall b/buildkite/src/Jobs/Test/CheckDhall.dhall index 34f8c78476e8..18cace107aaa 100644 --- a/buildkite/src/Jobs/Test/CheckDhall.dhall +++ b/buildkite/src/Jobs/Test/CheckDhall.dhall @@ -3,6 +3,7 @@ let D = S.PathPattern let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/generate-jobs" "sh" ], path = "Test", - name = "CheckDhall" + name = "CheckDhall", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index be248e222109..9826930106bf 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall @@ -18,7 +19,8 @@ in Pipeline.build Pipeline.Config::{ S.strictly (S.contains "Makefile") ], path = "Test", - name = "CheckGraphQLSchema" + name = "CheckGraphQLSchema", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ CheckGraphQLSchema.step dependsOn diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall index d6f404bfcf5a..4537de014e9b 100644 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall @@ -19,7 +20,8 @@ in Pipeline.build Pipeline.Config::{ S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" ], path = "Test", - name = "ConnectToBerkeley" + name = "ConnectToBerkeley", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ ConnectToTestnet.step dependsOn diff --git a/buildkite/src/Jobs/Test/CoverageTearDown.dhall b/buildkite/src/Jobs/Test/CoverageTearDown.dhall index bba020d80253..aa10f2a5ff8f 100644 --- a/buildkite/src/Jobs/Test/CoverageTearDown.dhall +++ b/buildkite/src/Jobs/Test/CoverageTearDown.dhall @@ -8,7 +8,7 @@ let Size = ../../Command/Size.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall -let PipelineStage = ../../Pipeline/Stage.dhall +let PipelineTag = ../../Pipeline/Tag.dhall in Pipeline.build Pipeline.Config::{ @@ -16,7 +16,7 @@ in Pipeline.build JobSpec::{ dirtyWhen = [ S.everything ], path = "Test", - stage = PipelineStage.Type.TearDown, + tags = [ PipelineTag.Type.TearDown ], name = "CoverageTearDown" } , steps = [ diff --git a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall index 7b769901aa7c..a21bc935750c 100644 --- a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -44,7 +46,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "DaemonUnitTest" + name = "DaemonUnitTest", + tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/lib" Size.XLarge diff --git a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall index f0b8e42f78e8..a41a66fce877 100644 --- a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall @@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -22,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/DelegationBackendUnitTest" "dhall" ], path = "Test", - name = "DelegationBackendUnitTest" + name = "DelegationBackendUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall b/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall index 62ea69f27241..cce7b364e497 100644 --- a/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall +++ b/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall @@ -6,6 +6,8 @@ let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -46,6 +48,7 @@ Pipeline.build dirtyWhen = unitDirtyWhen, path = "Test", name = "FuzzyZkappTest", + tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ], mode = PipelineMode.Type.Stable }, steps = [ diff --git a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall index e22384dc2ba6..76098a7f444c 100644 --- a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall +++ b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall @@ -4,6 +4,7 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall" ], path = "Test", - name = "Libp2pUnitTest" + name = "Libp2pUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 4b8d0c733027..d4be1d71bad1 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -5,6 +5,7 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -32,7 +33,8 @@ Pipeline.build JobSpec::{ dirtyWhen = dirtyWhen, path = "Test", - name = "RosettaIntegrationTests" + name = "RosettaIntegrationTests", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/RosettaUnitTest.dhall b/buildkite/src/Jobs/Test/RosettaUnitTest.dhall index 7e802917effc..6375974d2069 100644 --- a/buildkite/src/Jobs/Test/RosettaUnitTest.dhall +++ b/buildkite/src/Jobs/Test/RosettaUnitTest.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -41,7 +42,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "RosettaUnitTest" + name = "RosettaUnitTest", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/app/rosetta" Size.Small diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index dee246854bec..7a8b0251f5a0 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -54,7 +55,8 @@ Pipeline.build JobSpec::{ dirtyWhen = lintDirtyWhen, path = "Test", - name = "RunSnarkProfiler" + name = "RunSnarkProfiler", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ buildTestCmd Size.Small dependsOn diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 06ed548a2f30..dd7d5e6f0d85 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -56,7 +58,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "SingleNodeTest" + name = "SingleNodeTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/test/command_line_tests/command_line_tests.exe" Size.XLarge diff --git a/buildkite/src/Jobs/Test/SnarkyJSTest.dhall b/buildkite/src/Jobs/Test/SnarkyJSTest.dhall index d5054981a12a..6cc3f2790057 100644 --- a/buildkite/src/Jobs/Test/SnarkyJSTest.dhall +++ b/buildkite/src/Jobs/Test/SnarkyJSTest.dhall @@ -2,6 +2,7 @@ let S = ../../Lib/SelectFiles.dhall let B = ../../External/Buildkite.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -24,7 +25,8 @@ Pipeline.build S.strictlyStart (S.contains "src/lib") ], path = "Test", - name = "SnarkyJSTest" + name = "SnarkyJSTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index be2e42fee414..c46178d4572b 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -3,6 +3,8 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let TestExecutive = ../../Command/TestExecutive.dhall let dependsOn = [ @@ -30,7 +32,7 @@ in Pipeline.build Pipeline.Config::{ ], path = "Test", name = "TestnetIntegrationTests", - mode = PipelineMode.Type.Stable + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ TestExecutive.build "integration_tests", diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index feef489902b5..faf2dbdab554 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -3,6 +3,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let TestExecutive = ../../Command/TestExecutive.dhall let dependsOn = [ @@ -24,7 +25,8 @@ in Pipeline.build Pipeline.Config::{ ], path = "Test", name = "TestnetIntegrationTestsLong", - mode = PipelineMode.Type.Stable + mode = PipelineMode.Type.Stable, + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] }, steps = [ TestExecutive.execute "hard-fork" dependsOn diff --git a/buildkite/src/Jobs/Test/ValidationService.dhall b/buildkite/src/Jobs/Test/ValidationService.dhall index 1c2ee6fa5712..cce41331433a 100644 --- a/buildkite/src/Jobs/Test/ValidationService.dhall +++ b/buildkite/src/Jobs/Test/ValidationService.dhall @@ -1,6 +1,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -15,7 +16,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Test", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Jobs/Test/ZkappMetrics.dhall b/buildkite/src/Jobs/Test/ZkappMetrics.dhall index 4bf912a70c48..31ec5072d7b3 100644 --- a/buildkite/src/Jobs/Test/ZkappMetrics.dhall +++ b/buildkite/src/Jobs/Test/ZkappMetrics.dhall @@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall let WithCargo = ../../Command/WithCargo.dhall @@ -20,6 +21,7 @@ Pipeline.build ] , path = "Test" , name = "ZkappMetrics" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall b/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall index 45b00d8c2b52..7d0996abe307 100644 --- a/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -41,7 +42,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "ZkappTestToolUnitTest" + name = "ZkappTestToolUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/app/zkapp_test_transaction" Size.Small diff --git a/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall b/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall index 0fcdc592d981..9a6e4be6c952 100644 --- a/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall +++ b/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall @@ -5,6 +5,7 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -43,7 +44,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "ZkappsExamplesTest" + name = "ZkappsExamplesTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" Size.XLarge diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index 64783a0582fe..18a6ff6daa3d 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -11,12 +11,14 @@ let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall let PipelineMode = ./Pipeline/Mode.dhall -let PipelineStage = ./Pipeline/Stage.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let include_tags = env:BUILDKITE_PIPELINE_TAGS_INCL as Text ? "Fast" +let exclude_tags = env:BUILDKITE_PIPELINE_TAGS_EXCL as Text ? "" let jobs : List JobSpec.Type = List/map @@ -33,71 +35,52 @@ let prefixCommands = [ -- Run a job if we touched a dirty path -let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(targetMode: Text) -> +let commands: PipelineFilter.Type -> PipelineMode.Type -> List Cmd.Type = \(filter: PipelineFilter.Type) -> \(mode: PipelineMode.Type) -> Prelude.List.map JobSpec.Type Cmd.Type (\(job: JobSpec.Type) -> let jobMode = PipelineMode.capitalName job.mode - let jobStage = PipelineStage.capitalName job.stage - + let targetMode = PipelineMode.capitalName mode + let targetTags = PipelineFilter.tags filter + let filter = PipelineFilter.show filter + let isIncluded = Prelude.Bool.show (PipelineTag.contains job.tags targetTags) let dirtyWhen = SelectFiles.compile job.dirtyWhen let trigger = triggerCommand "src/Jobs/${job.path}/${job.name}.dhall" let pipelineHandlers = { PullRequest = '' - if [ "${targetMode}" == "PullRequest" ]; then - if [ "${jobStage}" == "${targetStage}" ]; then - if (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then - echo "Triggering ${job.name} for reason:" - cat _computed_diff.txt | egrep '${dirtyWhen}' - ${Cmd.format trigger} - fi - else - echo "Skipping ${job.name} because this is a ${targetStage} stage" - fi - else - if [ "${jobStage}" == "${targetStage}" ]; then - echo "Triggering ${job.name} because this is a stable buildkite run" - ${Cmd.format trigger} - else - echo "Skipping ${job.name} because this is a ${targetStage} stage" - fi - fi + ${Cmd.format trigger} '', Stable = '' - if [ "${targetMode}" == "PullRequest" ]; then - echo "Skipping ${job.name} because this is a PR buildkite run" - else - if [ "${jobStage}" == "${targetStage}" ]; then - echo "Triggering ${job.name} because this is a stable buildkite run" - ${Cmd.format trigger} - else - echo "Skipping ${job.name} because this is a ${targetStage} stage" - fi - fi + ${Cmd.format trigger} '' } in Cmd.quietly (merge pipelineHandlers job.mode) ) jobs -in Pipeline.build Pipeline.Config::{ - spec = JobSpec::{ - name = "monorepo-triage", - -- TODO: Clean up this code so we don't need an unused dirtyWhen here - dirtyWhen = [ SelectFiles.everything ] - }, - steps = [ - Command.build - Command.Config::{ - commands = prefixCommands # (commands stage mode), - label = "Monorepo triage ${stage}", - key = "cmds-${stage}", - target = Size.Small, - docker = Some Docker::{ - image = (./Constants/ContainerImages.dhall).toolchainBase, - environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] - } - } - ] -} +in +-- (\(args : { filter : PipelineFilter.Type, mode: PipelineMode.Type }) -> + Pipeline.build Pipeline.Config::{ + spec = JobSpec::{ + name = "monorepo-triage-${PipelineFilter.show args.filter}", + -- TODO: Clean up this code so we don't need an unused dirtyWhen here + dirtyWhen = [ SelectFiles.everything ] + }, + steps = [ + Command.build + Command.Config::{ + commands = prefixCommands, + label = "Monorepo triage ${PipelineFilter.show args.filter}", + key = "cmds-${PipelineFilter.show args.filter}", + target = Size.Small, + docker = Some Docker::{ + image = (./Constants/ContainerImages.dhall).toolchainBase, + environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] + } + } + ] + } +--) +---- # (commands args.filter args.mode), + diff --git a/buildkite/src/Pipeline/Filter.dhall b/buildkite/src/Pipeline/Filter.dhall new file mode 100644 index 000000000000..2cfb76979897 --- /dev/null +++ b/buildkite/src/Pipeline/Filter.dhall @@ -0,0 +1,31 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let Tag = ./Tag.dhall + +let Filter : Type = < FastOnly | Long | LongAndVeryLong | TearDownOnly > + +let tags: Filter -> List Tag.Type = \(filter: Filter) -> + merge { + FastOnly = [ Tag.Type.Fast ] + , LongAndVeryLong = [ Tag.Type.Long, Tag.Type.VeryLong ] + , Long = [ Tag.Type.Long ] + , TearDownOnly = [ Tag.Type.TearDown ] + } filter + +let show: Filter -> Text = \(filter: Filter) -> + merge { + FastOnly = "FastOnly" + , LongAndVeryLong = "LongAndVeryLong" + , Long = "Long" + , TearDownOnly = "TearDownOnly" + } filter + +in +{ + Type = Filter, + tags = tags, + show = show +} \ No newline at end of file diff --git a/buildkite/src/Pipeline/JobSpec.dhall b/buildkite/src/Pipeline/JobSpec.dhall index 187835c7d900..ea0373a0c2db 100644 --- a/buildkite/src/Pipeline/JobSpec.dhall +++ b/buildkite/src/Pipeline/JobSpec.dhall @@ -1,6 +1,6 @@ let SelectFiles = ../Lib/SelectFiles.dhall let PipelineMode = ./Mode.dhall -let PipelineStage = ./Stage.dhall +let PipelineTag = ./Tag.dhall in @@ -11,12 +11,12 @@ in path: Text, name: Text, mode: PipelineMode.Type, - stage: PipelineStage.Type, + tags: List PipelineTag.Type, dirtyWhen: List SelectFiles.Type }, default = { path = ".", mode = PipelineMode.Type.PullRequest, - stage = PipelineStage.Type.Test + tags = [ PipelineTag.Type.Fast ] } } diff --git a/buildkite/src/Pipeline/Stage.dhall b/buildkite/src/Pipeline/Stage.dhall deleted file mode 100644 index fffad54c7a2e..000000000000 --- a/buildkite/src/Pipeline/Stage.dhall +++ /dev/null @@ -1,44 +0,0 @@ --- Mode defines pipeline stages --- --- A pipeline in order to be faster and more cost efficient can have up to 3 stages --- Between each stages there is a '- wait' step defined which cause buildkite to wait --- for ALL jobs to complete before running any job from next stage. --- Current design defines three stages: --- - Stage 1 -> contains fastest and most independent jobs which are supposed to provide quickest feedback possible --- - Stage 2 -> contains heavy jobs that should be run only on clean code (no merges issues or lints problems) --- - Tear down -> should contains all clean up or reporting jobs. For example test coverage gathering - -let Prelude = ../External/Prelude.dhall - -let Stage : Type = < Test | TearDown > - -let toNatural: Stage -> Natural = \(stage: Stage) -> - merge { - Test = 1 - , TearDown = 2 - } stage - -let equal: Stage -> Stage -> Bool = \(left: Stage) -> \(right: Stage) -> - Prelude.Natural.equal (toNatural left) (toNatural right) - -let capitalName = \(stage : Stage) -> - merge { - Test = "Test" - , TearDown = "TearDown" - } stage - -let lowerName = \(stage : Stage) -> - merge { - Test = "test" - , TearDown = "tearDown" - } stage - - -in -{ - Type = Stage, - capitalName = capitalName, - lowerName = lowerName, - toNatural = toNatural, - equal = equal -} \ No newline at end of file diff --git a/buildkite/src/Pipeline/Tag.dhall b/buildkite/src/Pipeline/Tag.dhall new file mode 100644 index 000000000000..b6018062e675 --- /dev/null +++ b/buildkite/src/Pipeline/Tag.dhall @@ -0,0 +1,63 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let List/any = Prelude.List.any + +let Tag : Type = < Fast | Long | VeryLong | TearDown | Lint | Release | Test > + +let toNatural: Tag -> Natural = \(tag: Tag) -> + merge { + Fast = 1 + , Long = 2 + , VeryLong = 3 + , TearDown = 4 + , Lint = 5 + , Release = 6 + , Test = 7 + } tag + +let equal: Tag -> Tag -> Bool = \(left: Tag) -> \(right: Tag) -> + Prelude.Natural.equal (toNatural left) (toNatural right) + + +let hasAny: Tag -> List Tag -> Bool = \(input: Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> equal x input ) tags + +let contains: List Tag -> List Tag -> Bool = \(input: List Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> hasAny x tags ) input + +let capitalName = \(tag : Tag) -> + merge { + Fast = "Fast" + , Long = "Long" + , VeryLong = "VeryLong" + , TearDown = "TearDown" + , Lint = "Lint" + , Release = "Release" + , Test = "Test" + } tag + +let lowerName = \(tag : Tag) -> + merge { + Fast = "fast" + , Long = "long" + , VeryLong = "veryLong" + , TearDown = "tearDown" + , Lint = "lint" + , Release = "release" + , Test = "test" + } tag + + +in +{ + Type = Tag, + capitalName = capitalName, + lowerName = lowerName, + toNatural = toNatural, + equal = equal, + hasAny = hasAny, + contains = contains +} \ No newline at end of file diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index 101e22be7704..f35e486ec844 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -8,11 +8,14 @@ let Command = ./Command/Base.dhall let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall +let PipelineMode = ./Pipeline/Mode.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let filter = env:BUILDKITE_PIPELINE_FILTER as Text ? "FastOnly" let config : Pipeline.Config.Type = Pipeline.Config::{ spec = JobSpec::{ @@ -24,12 +27,13 @@ let config : Pipeline.Config.Type = Pipeline.Config::{ Command.build Command.Config::{ commands = [ Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}", - Cmd.run "export BUILDKITE_PIPELINE_STAGE=${stage}", + Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}", Cmd.run "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall", - triggerCommand "src/Monorepo.dhall" + Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall)' | buildkite-agent pipeline upload" + -- Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" ], label = "Prepare monorepo triage", - key = "monorepo-${stage}", + key = "monorepo-${mode}-${filter}", target = Size.Small, docker = Some Docker::{ image = (./Constants/ContainerImages.dhall).toolchainBase,