From 850cbac5b81e6b50c6f444eddda957d4f2d97eac Mon Sep 17 00:00:00 2001 From: Michael Foley Date: Fri, 11 Oct 2024 12:08:38 +1100 Subject: [PATCH] fix(BREAKING): only pass the output result locally --- README.md | 17 +++++++++--- hooks/post-command | 12 ++++++--- plugin.yml | 5 ++++ tests/.gitignore | 3 ++- tests/post-command.bats | 58 +++++++++++++++++------------------------ 5 files changed, 52 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index a45a327..e8e4661 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Add the following to your `pipeline.yml`: steps: - command: ~ plugins: - - fatzebra/terragrunt-workspace#v1.4.7: + - fatzebra/terragrunt-workspace#v2.0.0: module_dir: "test/test/" ``` @@ -45,20 +45,22 @@ Then the block will ask you to deploy the db and web modules Modules are discovered using the terragrunt command `terragrunt output-module-groups` during a post-command hook, so if you don't have terragrunt installed on your agent and instead use the docker or docker-compose plugins this will fail to run. We recommend the [devopsinfra/docker-terragrunt](https://hub.docker.com/r/devopsinfra/docker-terragrunt) docker image with the docker plugin, the docker image has all the tools requried and the docker plugin saves having to have a docker-compose file. -After adding the docker plugin you also need to run the module discovery as a command, this is then used by the plugin to run the deployment. +When you using the docker plugin add the following the command to provide the output to the plugin ```yml steps: - - command: buildkite-agent meta-data set terragrunt-workspace-module-groups "$(terragrunt output-module-groups --terragrunt-working-dir /test/test)" + - command: terragrunt output-module-groups > .terragrunt_module_groups_output.json plugins: - - fatzebra/terragrunt-workspace#v1.4.7: + - fatzebra/terragrunt-workspace#v2.0.0: module_dir: "test/test/" - docker#v5.11.0: image: "devopsinfra/docker-terragrunt:aws-tf-1.9.7-tg-0.67.16" mount-buildkite-agent: true ``` +This will output the JSON data to a location that the plugin tries to look for the output from. To change this value see the `output_module_groups_path` property + The plugin also copies the env, plugin and agent configuration to any steps it generates so this will be applied for subsequent steps. ## Configuration @@ -67,6 +69,12 @@ The plugin also copies the env, plugin and agent configuration to any steps it g The relative path to the directory where the terragrunt modules you want to run are in. +### `output_module_groups_path` (Optional, string) + +The output from the command `terragrunt output-module-groups` piped to this path when run in the `module_dir` this is a JSON object which outlines the available modules and the order they should be executed in. This can be used when you don't terragrunt on your workspace but use docker to run terragrunt. If the file can't be found or doesn't have content then the pipeline will try and run the command itself. + +Default: `.terragrunt_module_groups_output.json` + ### `allowed_modules` (Optional, array) A list of directory/module names that can be used as part of this plugin @@ -95,6 +103,7 @@ Writes the pipeline to the nominated output path If no modules are found for deployment should the pipeline fail +Default: true ## Developing diff --git a/hooks/post-command b/hooks/post-command index b988571..9b65974 100755 --- a/hooks/post-command +++ b/hooks/post-command @@ -40,13 +40,17 @@ else fi # look for the available terragrunt modules -if buildkite-agent meta-data exists "terragrunt-workspace-module-groups" ; then - module_groups="$(buildkite-agent meta-data get "terragrunt-workspace-module-groups")" +MODULE_GROUP_OUTPUT_PATH="${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH-".terragrunt_module_groups_output.json"}" + +# check to see if the output path exists and has content +if [[ -s "${MODULE_GROUP_OUTPUT_PATH}" ]] ; then + MODULE_GROUPS="$(cat "${MODULE_GROUP_OUTPUT_PATH}")" else # shellcheck disable=SC2068 - module_groups="$(terragrunt output-module-groups --terragrunt-working-dir "${MODULE_DIR}" ${terragrunt_args[@]})" + MODULE_GROUPS="$(terragrunt output-module-groups --terragrunt-working-dir "${MODULE_DIR}" ${terragrunt_args[@]})" fi -discovered_modules_list="$(echo "${module_groups}" | jq -r '[keys[] as $k | .[$k] ]| flatten | .[]')" + +discovered_modules_list="$(echo "${MODULE_GROUPS}" | jq -r '[keys[] as $k | .[$k] ]| flatten | .[]')" discovered_modules=() diff --git a/plugin.yml b/plugin.yml index c4087eb..2a2ced0 100644 --- a/plugin.yml +++ b/plugin.yml @@ -12,6 +12,11 @@ configuration: description: The path to the modules that you want to apply type: string + output_module_groups_path: + description: The path to a file where the output from `terragrunt output-module-groups` has been saved. Used in a command when your agent doesn't have access to terragrunt + type: string + default: ".terragrunt_module_groups_output.json" + allowed_modules: description: A list of modules that can be used by this plugin type: array diff --git a/tests/.gitignore b/tests/.gitignore index d828726..8617f27 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,2 @@ -.outputs \ No newline at end of file +.outputs +.data \ No newline at end of file diff --git a/tests/post-command.bats b/tests/post-command.bats index 1e21a63..e968958 100755 --- a/tests/post-command.bats +++ b/tests/post-command.bats @@ -6,9 +6,13 @@ load "$BATS_PLUGIN_PATH/load.bash" #export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty setup_file() { - export OUTPUT_PATH="$PWD/tests/.outputs/" + export OUTPUT_PATH="$PWD/tests/.outputs/" mkdir -p "${OUTPUT_PATH}" rm -f ${OUTPUT_PATH}/* + + export DATA_PATH="$PWD/tests/.data/" + mkdir -p "${DATA_PATH}" + rm -f ${DATA_PATH}/* } setup() { @@ -25,15 +29,14 @@ setup() { export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT="${OUTPUT_PATH}/${BATS_TEST_NAME// /"_"}.yml" + export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH="${DATA_PATH}/${BATS_TEST_NAME// /"_"}_output_module_groups.json" + echo "{\"Group1\": [\"$PWD/test/test/$MODULE\"]}" > "${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH}" + stub buildkite-agent \ - 'meta-data exists "terragrunt-workspace-module-groups" : false' \ "step get --format json : echo '${STEP_OUTPUT}'" \ 'annotate \* \* \* \* \* \* : echo Noted' \ 'pipeline upload : echo Uploading pipeline' - stub terragrunt \ - "output-module-groups --terragrunt-working-dir ${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR} : echo '{\"Group1\": [\"$PWD/test/test/$MODULE\"]}'" - run "$PWD/hooks/post-command" assert_success @@ -41,7 +44,6 @@ setup() { assert_output --partial "Modules for deployment - $MODULE" unstub buildkite-agent - unstub terragrunt # Is pipeline valid yaml run yq '.' $BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT @@ -72,11 +74,10 @@ setup() { export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT="${OUTPUT_PATH}/${BATS_TEST_NAME// /"_"}.yml" export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_0="passwords" - stub terragrunt \ - "output-module-groups --terragrunt-working-dir ${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR} : echo '{\"Group1\": [\"$PWD/test/test/$MODULE\", \"$PWD/test/test/${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_0}\"]}'" + export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH="${DATA_PATH}/${BATS_TEST_NAME// /"_"}_output_module_groups.json" + echo "{\"Group1\": [\"$PWD/test/test/$MODULE\", \"$PWD/test/test/${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_0}\"]}" > "${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH}" stub buildkite-agent \ - 'meta-data exists "terragrunt-workspace-module-groups" : false' \ "step get --format json : echo '${STEP_OUTPUT}'" \ 'annotate \* \* \* \* \* \* : echo Noted' \ 'pipeline upload : echo "Uploading pipeline"' @@ -88,7 +89,6 @@ setup() { assert_line "Modules for deployment - $MODULE " assert_line "Uploading pipeline" - unstub terragrunt unstub buildkite-agent # The refresh module should never be planned @@ -109,15 +109,14 @@ setup() { export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT="${OUTPUT_PATH}/${BATS_TEST_NAME// /"_"}.yml" export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_ALLOWED_MODULES_0="${MODULE}" + export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH="${DATA_PATH}/${BATS_TEST_NAME// /"_"}_output_module_groups.json" + echo "{\"Group1\":[\"$PWD/test/test/$MODULE\",\"$PWD/test/test/$DANGER_MODULE\"]}" > "${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH}" + stub buildkite-agent \ - 'meta-data exists "terragrunt-workspace-module-groups" : false' \ "step get --format json : echo '${STEP_OUTPUT}'" \ 'annotate \* \* \* \* \* \* : echo Noted' \ 'pipeline upload : echo Uploading pipeline' - stub terragrunt \ - "output-module-groups --terragrunt-working-dir ${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR} : echo '{\"Group1\":[\"$PWD/test/test/$MODULE\",\"$PWD/test/test/$DANGER_MODULE\"]}'" - run "$PWD/hooks/post-command" assert_success @@ -126,7 +125,6 @@ setup() { assert_line "Modules after filtering - $MODULE " unstub buildkite-agent - unstub terragrunt # Is pipeline valid yaml run yq '.' $BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT @@ -144,21 +142,19 @@ setup() { export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_ALLOWED_MODULES_0="${MODULE}" export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_PLAN_ENCRYPTION_KMS_KEY_ARN="arn:aws:kms:ap-southeast-2:123456789012:key/5a14434f-9d4e-55dh-27df-f32711fe0492" + export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH="${DATA_PATH}/${BATS_TEST_NAME// /"_"}_output_module_groups.json" + echo "{\"Group1\":[\"$PWD/test/test/$MODULE\",\"$PWD/test/test/$DANGER_MODULE\"]}" > "${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH}" + stub buildkite-agent \ - 'meta-data exists "terragrunt-workspace-module-groups" : false' \ "step get --format json : echo '${STEP_OUTPUT}'" \ 'annotate \* \* \* \* \* \* : echo Noted' \ 'pipeline upload : echo Uploading pipeline' - stub terragrunt \ - "output-module-groups --terragrunt-working-dir ${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR} : echo '{\"Group1\":[\"$PWD/test/test/$MODULE\",\"$PWD/test/test/$DANGER_MODULE\"]}'" - run "$PWD/hooks/post-command" assert_success unstub buildkite-agent - unstub terragrunt # Is pipeline valid yaml run yq '.' $BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT @@ -176,11 +172,10 @@ setup() { export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_0="passwords" export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_1="constants" - stub terragrunt \ - "output-module-groups --terragrunt-working-dir ${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR} : echo '{\"Group1\": [\"$PWD/test/test/$MODULE_1\", \"$PWD/test/test/$MODULE_2\", \"$PWD/test/test/${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_0}\", \"$PWD/test/test/${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_1}\"]}'" + export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH="${DATA_PATH}/${BATS_TEST_NAME// /"_"}_output_module_groups.json" + echo "{\"Group1\": [\"$PWD/test/test/$MODULE_1\", \"$PWD/test/test/$MODULE_2\", \"$PWD/test/test/${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_0}\", \"$PWD/test/test/${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES_1}\"]}" > "${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH}" stub buildkite-agent \ - 'meta-data exists "terragrunt-workspace-module-groups" : false' \ "step get --format json : echo '${STEP_OUTPUT}'" \ 'annotate \* \* \* \* \* \* : echo Noted' \ 'pipeline upload : echo Uploading pipeline' @@ -192,7 +187,6 @@ setup() { assert_line "Modules for deployment - $MODULE_1 $MODULE_2 " assert_line "Uploading pipeline" - unstub terragrunt unstub buildkite-agent # The refresh module should never be planned @@ -214,12 +208,11 @@ setup() { export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT="${OUTPUT_PATH}/${BATS_TEST_NAME// /"_"}.yml" export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_ALLOWED_MODULES_0="other" - - stub terragrunt \ - "output-module-groups --terragrunt-working-dir ${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR} : echo '{\"Group1\": [\"$PWD/test/test/$MODULE_1\", \"$PWD/test/test/$MODULE_2\"]}'" + + export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH="${DATA_PATH}/${BATS_TEST_NAME// /"_"}_output_module_groups.json" + echo "{\"Group1\": [\"$PWD/test/test/$MODULE_1\", \"$PWD/test/test/$MODULE_2\"]}" > "${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH}" stub buildkite-agent \ - 'meta-data exists "terragrunt-workspace-module-groups" : false' \ "step get --format json : echo '${STEP_OUTPUT}'" \ 'annotate \* \* \* \* \* \* : echo Noted' @@ -230,7 +223,6 @@ setup() { refute_line "Modules for deployment - $MODULE_1 $MODULE_2 " refute_line "Uploading pipeline" - unstub terragrunt unstub buildkite-agent } @@ -241,12 +233,11 @@ setup() { export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT="${OUTPUT_PATH}/${BATS_TEST_NAME// /"_"}.yml" export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_ALLOWED_MODULES_0="other" export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_FAIL_ON_NO_MODULES="false" - - stub terragrunt \ - "output-module-groups --terragrunt-working-dir ${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR} : echo '{\"Group1\": [\"$PWD/test/test/$MODULE_1\", \"$PWD/test/test/$MODULE_2\"]}'" + + export BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH="${DATA_PATH}/${BATS_TEST_NAME// /"_"}_output_module_groups.json" + echo "{\"Group1\": [\"$PWD/test/test/$MODULE_1\", \"$PWD/test/test/$MODULE_2\"]}" > "${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_OUTPUT_MODULE_GROUPS_PATH}" stub buildkite-agent \ - 'meta-data exists "terragrunt-workspace-module-groups" : false' \ "step get --format json : echo '${STEP_OUTPUT}'" \ 'annotate \* \* \* \* \* \* : echo Noted' @@ -257,6 +248,5 @@ setup() { refute_line "Modules for deployment - $MODULE_1 $MODULE_2 " refute_line "Uploading pipeline" - unstub terragrunt unstub buildkite-agent } \ No newline at end of file