diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall index c41140e79cb..aec5fab68d4 100644 --- a/buildkite/src/Jobs/Lint/Fast.dhall +++ b/buildkite/src/Jobs/Lint/Fast.dhall @@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -31,6 +32,7 @@ in Pipeline.build , dirtyWhen = [ S.strictlyStart (S.contains "src/") ] , path = "Lint" , name = "Fast" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/HelmChart.dhall b/buildkite/src/Jobs/Lint/HelmChart.dhall index 2b0e602117c..f0f93b132f1 100644 --- a/buildkite/src/Jobs/Lint/HelmChart.dhall +++ b/buildkite/src/Jobs/Lint/HelmChart.dhall @@ -4,6 +4,7 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Lint", - name = "HelmChart" + name = "HelmChart", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index 83249b2c16e..58ef1e8b28a 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -3,6 +3,7 @@ let Prelude = ../../External/Prelude.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -17,7 +18,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ SelectFiles.everything ], path = "Lint", - name = "Merge" + name = "Merge", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/OCaml.dhall b/buildkite/src/Jobs/Lint/OCaml.dhall index a79f1e00026..3dec98290af 100644 --- a/buildkite/src/Jobs/Lint/OCaml.dhall +++ b/buildkite/src/Jobs/Lint/OCaml.dhall @@ -7,6 +7,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -31,6 +32,7 @@ in Pipeline.build [ dirtyDhallDir, S.strictlyStart (S.contains "src/") ] , path = "Lint" , name = "OCaml" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } , steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/Rust.dhall b/buildkite/src/Jobs/Lint/Rust.dhall index a551272308d..c099d225646 100644 --- a/buildkite/src/Jobs/Lint/Rust.dhall +++ b/buildkite/src/Jobs/Lint/Rust.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -19,7 +21,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") ], path = "Lint", - name = "Rust" + name = "Rust", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall index 90b69b3f91e..c60772c198b 100644 --- a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -25,7 +27,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Lint", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall index 96fe9d19bf8..a2469a29ef0 100644 --- a/buildkite/src/Jobs/Lint/ValidationService.dhall +++ b/buildkite/src/Jobs/Lint/ValidationService.dhall @@ -4,6 +4,8 @@ let List/map = Prelude.List.map let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let Cmd = ../../Lib/Cmds.dhall let Command = ../../Command/Base.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -52,7 +54,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Lint", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall index 139687b4a5b..28a9268420c 100644 --- a/buildkite/src/Jobs/Lint/Xrefcheck.dhall +++ b/buildkite/src/Jobs/Lint/Xrefcheck.dhall @@ -4,6 +4,8 @@ let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall @@ -23,7 +25,8 @@ Pipeline.build SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml") ], path = "Lint", - name = "Xrefcheck" + name = "Xrefcheck", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/HelmRelease.dhall b/buildkite/src/Jobs/Release/HelmRelease.dhall index 7fa74079dda..8e843171ab7 100644 --- a/buildkite/src/Jobs/Release/HelmRelease.dhall +++ b/buildkite/src/Jobs/Release/HelmRelease.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -21,7 +23,8 @@ Pipeline.build S.exactly "buildkite/scripts/helm-ci" "sh" ], path = "Release", - name = "HelmRelease" + name = "HelmRelease", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall index f8b657359fd..252cd012e22 100644 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -27,7 +29,8 @@ Pipeline.build S.strictlyStart (S.contains "frontend/leaderboard") ], path = "Release", - name = "LeaderboardArtifact" + name = "LeaderboardArtifact", + tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/TestnetAlerts.dhall b/buildkite/src/Jobs/Release/TestnetAlerts.dhall index 06125455a8c..e444c4e7a33 100644 --- a/buildkite/src/Jobs/Release/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Release/TestnetAlerts.dhall @@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -24,7 +26,8 @@ Pipeline.build S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ], path = "Release", - name = "TestnetAlerts" + name = "TestnetAlerts", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Release/TraceTool.dhall b/buildkite/src/Jobs/Release/TraceTool.dhall index d4a0e4997f4..0d6770523e5 100644 --- a/buildkite/src/Jobs/Release/TraceTool.dhall +++ b/buildkite/src/Jobs/Release/TraceTool.dhall @@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -20,7 +22,8 @@ Pipeline.build spec = JobSpec::{ dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/TraceTool") ], path = "Release", - name = "TraceTool" + name = "TraceTool", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall index 0be4cc06b35..d746d0381f1 100644 --- a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall @@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -26,6 +27,7 @@ Pipeline.build ] , path = "Test" , name = "ArchiveNodeUnitTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } , steps = let outerDir : Text = diff --git a/buildkite/src/Jobs/Test/CheckDhall.dhall b/buildkite/src/Jobs/Test/CheckDhall.dhall index 34f8c78476e..18cace107aa 100644 --- a/buildkite/src/Jobs/Test/CheckDhall.dhall +++ b/buildkite/src/Jobs/Test/CheckDhall.dhall @@ -3,6 +3,7 @@ let D = S.PathPattern let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/scripts/generate-jobs" "sh" ], path = "Test", - name = "CheckDhall" + name = "CheckDhall", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/CoverageTearDown.dhall b/buildkite/src/Jobs/Test/CoverageTearDown.dhall index bba020d8025..aa10f2a5ff8 100644 --- a/buildkite/src/Jobs/Test/CoverageTearDown.dhall +++ b/buildkite/src/Jobs/Test/CoverageTearDown.dhall @@ -8,7 +8,7 @@ let Size = ../../Command/Size.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall -let PipelineStage = ../../Pipeline/Stage.dhall +let PipelineTag = ../../Pipeline/Tag.dhall in Pipeline.build Pipeline.Config::{ @@ -16,7 +16,7 @@ in Pipeline.build JobSpec::{ dirtyWhen = [ S.everything ], path = "Test", - stage = PipelineStage.Type.TearDown, + tags = [ PipelineTag.Type.TearDown ], name = "CoverageTearDown" } , steps = [ diff --git a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall index 8e234daed36..43d8d2af793 100644 --- a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall @@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -42,7 +44,8 @@ Pipeline.build JobSpec::{ dirtyWhen = unitDirtyWhen, path = "Test", - name = "DaemonUnitTest" + name = "DaemonUnitTest", + tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ] }, steps = [ buildTestCmd "dev" "src/lib" Size.XLarge diff --git a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall index 03e9d4293aa..248df0d4b77 100644 --- a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall @@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -22,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/DelegationBackendUnitTest" "dhall" ], path = "Test", - name = "DelegationBackendUnitTest" + name = "DelegationBackendUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall index e22384dc2ba..76098a7f444 100644 --- a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall +++ b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall @@ -4,6 +4,7 @@ let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall @@ -23,7 +24,8 @@ Pipeline.build S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall" ], path = "Test", - name = "Libp2pUnitTest" + name = "Libp2pUnitTest", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build diff --git a/buildkite/src/Jobs/Test/ValidationService.dhall b/buildkite/src/Jobs/Test/ValidationService.dhall index 1c2ee6fa571..cce41331433 100644 --- a/buildkite/src/Jobs/Test/ValidationService.dhall +++ b/buildkite/src/Jobs/Test/ValidationService.dhall @@ -1,6 +1,7 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -15,7 +16,8 @@ in Pipeline.build Pipeline.Config::{ S.strictlyStart (S.contains ValidationService.rootPath) ], path = "Test", - name = "ValidationService" + name = "ValidationService", + tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] }, steps = [ Command.build Command.Config::{ diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index 64783a0582f..3281333b8e6 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -11,12 +11,14 @@ let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall let PipelineMode = ./Pipeline/Mode.dhall -let PipelineStage = ./Pipeline/Stage.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let include_tags = env:BUILDKITE_PIPELINE_TAGS_INCL as Text ? "Fast" +let exclude_tags = env:BUILDKITE_PIPELINE_TAGS_EXCL as Text ? "" let jobs : List JobSpec.Type = List/map @@ -33,34 +35,36 @@ let prefixCommands = [ -- Run a job if we touched a dirty path -let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(targetMode: Text) -> +let commands: PipelineFilter.Type -> PipelineMode.Type -> List Cmd.Type = \(filter: PipelineFilter.Type) -> \(mode: PipelineMode.Type) -> Prelude.List.map JobSpec.Type Cmd.Type (\(job: JobSpec.Type) -> let jobMode = PipelineMode.capitalName job.mode - let jobStage = PipelineStage.capitalName job.stage - + let targetMode = PipelineMode.capitalName mode + let targetTags = PipelineFilter.tags filter + let filter = PipelineFilter.show filter + let isIncluded = Prelude.Bool.show (PipelineTag.contains job.tags targetTags) let dirtyWhen = SelectFiles.compile job.dirtyWhen let trigger = triggerCommand "src/Jobs/${job.path}/${job.name}.dhall" let pipelineHandlers = { PullRequest = '' if [ "${targetMode}" == "PullRequest" ]; then - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then if (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then echo "Triggering ${job.name} for reason:" cat _computed_diff.txt | egrep '${dirtyWhen}' ${Cmd.format trigger} fi else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi else - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then echo "Triggering ${job.name} because this is a stable buildkite run" ${Cmd.format trigger} else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi fi '', @@ -68,11 +72,11 @@ let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(target if [ "${targetMode}" == "PullRequest" ]; then echo "Skipping ${job.name} because this is a PR buildkite run" else - if [ "${jobStage}" == "${targetStage}" ]; then + if [ "${isIncluded}" == "True" ]; then echo "Triggering ${job.name} because this is a stable buildkite run" ${Cmd.format trigger} else - echo "Skipping ${job.name} because this is a ${targetStage} stage" + echo "Skipping ${job.name} because this is a ${filter} stage" fi fi '' @@ -81,23 +85,29 @@ let commands: Text -> Text -> List Cmd.Type = \(targetStage: Text) -> \(target ) jobs -in Pipeline.build Pipeline.Config::{ - spec = JobSpec::{ - name = "monorepo-triage", - -- TODO: Clean up this code so we don't need an unused dirtyWhen here - dirtyWhen = [ SelectFiles.everything ] - }, - steps = [ - Command.build - Command.Config::{ - commands = prefixCommands # (commands stage mode), - label = "Monorepo triage ${stage}", - key = "cmds-${stage}", - target = Size.Small, - docker = Some Docker::{ - image = (./Constants/ContainerImages.dhall).toolchainBase, - environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] - } - } - ] -} +in + +(\(args : { filter : PipelineFilter.Type, mode: PipelineMode.Type }) -> + let pipelineType = Pipeline.build Pipeline.Config::{ + spec = JobSpec::{ + name = "monorepo-triage-${PipelineFilter.show args.filter}", + -- TODO: Clean up this code so we don't need an unused dirtyWhen here + dirtyWhen = [ SelectFiles.everything ] + }, + steps = [ + Command.build + Command.Config::{ + commands = prefixCommands # (commands args.filter args.mode), + label = "Monorepo triage ${PipelineFilter.show args.filter}", + key = "cmds-${PipelineFilter.show args.filter}", + target = Size.Small, + docker = Some Docker::{ + image = (./Constants/ContainerImages.dhall).toolchainBase, + environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] + } + } + ] + } + in pipelineType.pipeline +) + diff --git a/buildkite/src/Pipeline/Filter.dhall b/buildkite/src/Pipeline/Filter.dhall new file mode 100644 index 00000000000..fc9cd7dd5ac --- /dev/null +++ b/buildkite/src/Pipeline/Filter.dhall @@ -0,0 +1,33 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let Tag = ./Tag.dhall + +let Filter : Type = < FastOnly | Long | LongAndVeryLong | TearDownOnly | AllTests > + +let tags: Filter -> List Tag.Type = \(filter: Filter) -> + merge { + FastOnly = [ Tag.Type.Fast ] + , LongAndVeryLong = [ Tag.Type.Long, Tag.Type.VeryLong ] + , Long = [ Tag.Type.Long ] + , TearDownOnly = [ Tag.Type.TearDown ] + , AllTests = [ Tag.Type.Lint, Tag.Type.Release, Tag.Type.Test ] + } filter + +let show: Filter -> Text = \(filter: Filter) -> + merge { + FastOnly = "FastOnly" + , LongAndVeryLong = "LongAndVeryLong" + , Long = "Long" + , TearDownOnly = "TearDownOnly" + , AllTests = "AllTests" + } filter + +in +{ + Type = Filter, + tags = tags, + show = show +} \ No newline at end of file diff --git a/buildkite/src/Pipeline/JobSpec.dhall b/buildkite/src/Pipeline/JobSpec.dhall index 187835c7d90..ea0373a0c2d 100644 --- a/buildkite/src/Pipeline/JobSpec.dhall +++ b/buildkite/src/Pipeline/JobSpec.dhall @@ -1,6 +1,6 @@ let SelectFiles = ../Lib/SelectFiles.dhall let PipelineMode = ./Mode.dhall -let PipelineStage = ./Stage.dhall +let PipelineTag = ./Tag.dhall in @@ -11,12 +11,12 @@ in path: Text, name: Text, mode: PipelineMode.Type, - stage: PipelineStage.Type, + tags: List PipelineTag.Type, dirtyWhen: List SelectFiles.Type }, default = { path = ".", mode = PipelineMode.Type.PullRequest, - stage = PipelineStage.Type.Test + tags = [ PipelineTag.Type.Fast ] } } diff --git a/buildkite/src/Pipeline/Stage.dhall b/buildkite/src/Pipeline/Stage.dhall deleted file mode 100644 index fffad54c7a2..00000000000 --- a/buildkite/src/Pipeline/Stage.dhall +++ /dev/null @@ -1,44 +0,0 @@ --- Mode defines pipeline stages --- --- A pipeline in order to be faster and more cost efficient can have up to 3 stages --- Between each stages there is a '- wait' step defined which cause buildkite to wait --- for ALL jobs to complete before running any job from next stage. --- Current design defines three stages: --- - Stage 1 -> contains fastest and most independent jobs which are supposed to provide quickest feedback possible --- - Stage 2 -> contains heavy jobs that should be run only on clean code (no merges issues or lints problems) --- - Tear down -> should contains all clean up or reporting jobs. For example test coverage gathering - -let Prelude = ../External/Prelude.dhall - -let Stage : Type = < Test | TearDown > - -let toNatural: Stage -> Natural = \(stage: Stage) -> - merge { - Test = 1 - , TearDown = 2 - } stage - -let equal: Stage -> Stage -> Bool = \(left: Stage) -> \(right: Stage) -> - Prelude.Natural.equal (toNatural left) (toNatural right) - -let capitalName = \(stage : Stage) -> - merge { - Test = "Test" - , TearDown = "TearDown" - } stage - -let lowerName = \(stage : Stage) -> - merge { - Test = "test" - , TearDown = "tearDown" - } stage - - -in -{ - Type = Stage, - capitalName = capitalName, - lowerName = lowerName, - toNatural = toNatural, - equal = equal -} \ No newline at end of file diff --git a/buildkite/src/Pipeline/Tag.dhall b/buildkite/src/Pipeline/Tag.dhall new file mode 100644 index 00000000000..b6018062e67 --- /dev/null +++ b/buildkite/src/Pipeline/Tag.dhall @@ -0,0 +1,63 @@ +-- Tag defines pipeline +-- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline + +let Prelude = ../External/Prelude.dhall +let List/any = Prelude.List.any + +let Tag : Type = < Fast | Long | VeryLong | TearDown | Lint | Release | Test > + +let toNatural: Tag -> Natural = \(tag: Tag) -> + merge { + Fast = 1 + , Long = 2 + , VeryLong = 3 + , TearDown = 4 + , Lint = 5 + , Release = 6 + , Test = 7 + } tag + +let equal: Tag -> Tag -> Bool = \(left: Tag) -> \(right: Tag) -> + Prelude.Natural.equal (toNatural left) (toNatural right) + + +let hasAny: Tag -> List Tag -> Bool = \(input: Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> equal x input ) tags + +let contains: List Tag -> List Tag -> Bool = \(input: List Tag) -> \(tags: List Tag) -> + List/any Tag (\(x: Tag) -> hasAny x tags ) input + +let capitalName = \(tag : Tag) -> + merge { + Fast = "Fast" + , Long = "Long" + , VeryLong = "VeryLong" + , TearDown = "TearDown" + , Lint = "Lint" + , Release = "Release" + , Test = "Test" + } tag + +let lowerName = \(tag : Tag) -> + merge { + Fast = "fast" + , Long = "long" + , VeryLong = "veryLong" + , TearDown = "tearDown" + , Lint = "lint" + , Release = "release" + , Test = "test" + } tag + + +in +{ + Type = Tag, + capitalName = capitalName, + lowerName = lowerName, + toNatural = toNatural, + equal = equal, + hasAny = hasAny, + contains = contains +} \ No newline at end of file diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index 101e22be770..5b78ff24369 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -8,11 +8,14 @@ let Command = ./Command/Base.dhall let Docker = ./Command/Docker/Type.dhall let JobSpec = ./Pipeline/JobSpec.dhall let Pipeline = ./Pipeline/Dsl.dhall +let PipelineMode = ./Pipeline/Mode.dhall +let PipelineFilter = ./Pipeline/Filter.dhall +let PipelineTag = ./Pipeline/Tag.dhall let Size = ./Command/Size.dhall let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let stage = env:BUILDKITE_PIPELINE_STAGE as Text ? "Test" +let filter = env:BUILDKITE_PIPELINE_FILTER as Text ? "FastOnly" let config : Pipeline.Config.Type = Pipeline.Config::{ spec = JobSpec::{ @@ -24,12 +27,12 @@ let config : Pipeline.Config.Type = Pipeline.Config::{ Command.build Command.Config::{ commands = [ Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}", - Cmd.run "export BUILDKITE_PIPELINE_STAGE=${stage}", + Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}", Cmd.run "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall", - triggerCommand "src/Monorepo.dhall" + Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" ], label = "Prepare monorepo triage", - key = "monorepo-${stage}", + key = "monorepo-${mode}-${filter}", target = Size.Small, docker = Some Docker::{ image = (./Constants/ContainerImages.dhall).toolchainBase,