Initial commit

.github/workflows/test.yml

@@ -0,0 +1,15 @@
+name: testing
+on: 
+  - push
+  - pull_request
+
+jobs:
+  lint: 
+    runs-on: "ubuntu-latest"
+    steps:
+      - run: docker compose run --rm lint
+
+  test: 
+    runs-on: "ubuntu-latest"
+    steps:
+      - run: docker compose run --rm test

README.md

@@ -0,0 +1,82 @@
+# Terragrunt Workspace Buildkite Plugin
+
+Provides a standard pipeline for deploying terragrunt workspaces
+
+The plugin generates a dynamic pipeline based on the modules discovered by terragrunt. 
+
+The pipeline consists of:
+
+- A block step which allows you to select the modules to apply. This is ordered based on how it will be applied
+- A command step which will run a plan for each of the selected modules and provide the output
+- A block to confirm you want to run apply
+- A command step to run apply for each of the selected modules
+
+By default all modules found by terragrunt will be included in the block step, however you can filter this by provided a list of allowed modules. 
+
+If you have modules that are just made up of data components that provide information to your other modules you can specify these modules under the data_modules option, before running the plan or apply commands on the selected modules each of the data modules will be refreshed. This builds a local state file since modules without resources don't save their state in terraform.
+
+
+## Example
+
+Add the following to your `pipeline.yml`:
+
+```yml
+steps:
+  - command: ~
+    plugins:
+      - roleyfoley/terragrunt-workspace#v1.0.0:
+          name: "test"
+          module_dir: "test/test/"
+```
+
+If you have the following terragrunt setup
+
+```
+- test
+    - test
+        - db
+            - terragrunt.hcl
+        - web
+            - terragrunt.hcl
+```
+
+Then the block will ask you to deploy the db and web modules
+
+
+## Configuration
+
+### `name` (Required, string)
+
+A name for the set of modules you want to deploy generally this an environment like test, staging, production. Only used for display purposes here
+
+### `module_dir` (Required, string)
+
+The relative path to the directory where the terragrunt modules you want to run are in.
+
+### `allowed_modules` (Optional, array)
+
+A list of directory/module names that can be used as part of this plugin
+
+### `data_modules` (Optional, array)
+
+The directory names of the modules you want to run `terragrunt refresh` each time a plan or apply is run on the `modules` or `always_modules`. Sometimes you might have modules that only have data components that lookup passwords, parameters etc. Since these don't save there state to a backend you need to refresh them each time to get their outputs.
+
+### `debug_pipeline_output` (Optional, string)
+
+Writes the pipeline to the nominated output path
+
+## Developing
+
+To run the tests:
+
+```shell
+docker-compose run --rm tests
+```
+
+## Contributing
+
+1. Fork the repo
+2. Make the changes
+3. Run the tests
+4. Commit and push your changes
+5. Send a pull request

docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  lint:
+    image: buildkite/plugin-linter
+    command: ['--id', 'roleyfoley/terragrunt-workspace']
+    volumes:
+      - ".:/plugin:ro"
+
+  test:
+    build:
+      dockerfile: tests/Dockerfile
+      context: .
+    volumes:
+      - "./:/plugin"
+

hooks/post-command

@@ -0,0 +1,164 @@
+#!/bin/bash
+set -euo pipefail
+
+# Reads a list from plugin config into a global result array
+# Returns success if values were read
+function plugin_read_list_into_result() {
+  result=()
+
+  for prefix in "$@" ; do
+    local i=0
+    local parameter="${prefix}_${i}"
+
+    if [[ -n "${!prefix:-}" ]] ; then
+      echo "🚨 Plugin received a string for $prefix, expected an array" >&2
+      exit 1
+    fi
+
+    while [[ -n "${!parameter:-}" ]]; do
+      result+=("${!parameter}")
+      i=$((i+1))
+      parameter="${prefix}_${i}"
+    done
+  done
+
+  [[ ${#result[@]} -gt 0 ]] || return 1
+}
+
+NAME="${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_NAME}"
+MODULE_DIR="${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_MODULE_DIR}"
+DEBUG_PIPELINE_OUTPUT="${BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DEBUG_PIPELINE_OUTPUT-""}"
+
+# Terragrunt extra args
+if plugin_read_list_into_result BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_TERRAGRUNT_ARGS ; then
+  terragrunt_args=()
+  for arg in "${result[@]}" ; do
+    args+=( "${arg}" )
+  done
+  terragrunt_args="$(printf "%q " "${args[@]}")"
+
+else 
+  terragrunt_args=""
+fi
+
+# look for the available terragrunt modules
+discovered_modules=()
+discovered_modules_list="$(terragrunt output-module-groups --terragrunt-working-dir ${MODULE_DIR} ${terragrunt_args} | jq -r '[keys[] as $k | .[$k] ]| flatten | .[]')"
+
+if [[ -z "${discovered_modules_list}" ]]; then
+  echo "No Modules found"
+  exit 1
+fi
+
+for module in ${discovered_modules_list}; do 
+  discovered_modules+=("${module/"${PWD}/${MODULE_DIR}/"/""}")
+done
+
+echo ":building_construction: Discovered modules - $(printf '%s ' "${discovered_modules[@]}")"
+
+# Filter based on the allowed modules
+if plugin_read_list_into_result BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_ALLOWED_MODULES ; then
+  available_modules=()
+  for discovered_module in "${discovered_modules[@]}"; do
+    for allowed_module in "${result[@]}" ; do 
+      if [[ "${discovered_module}" == "${allowed_module}" ]]; then 
+        available_modules+=("${discovered_module}")
+      fi
+    done
+  done
+
+  echo ":policeman: Modules after filtering - $(printf '%s ' "${available_modules[@]}")"
+else 
+  available_modules=("${discovered_modules[@]}")
+fi
+
+
+# Split the data modules from the deploy modules
+if plugin_read_list_into_result BUILDKITE_PLUGIN_TERRAGRUNT_WORKSPACE_DATA_MODULES ; then
+  data_modules=()
+  deploy_modules=()
+
+  for available_module in "${available_modules[@]}"; do 
+    for data_module in "${result[@]}" ; do 
+      if [[ "${data_module}" == "${available_module}" ]]; then 
+        data_modules+=("${available_module}")
+      else
+        deploy_modules+=("${available_module}")
+      fi
+    done 
+  done
+
+  echo ":chart_with_upwards_trend: Data modules - $(printf '%s ' "${data_modules[@]}")"
+else
+  deploy_modules=("${available_modules[@]}")
+fi
+
+echo ":rocket: Modules for deployment - $(printf '%s ' "${deploy_modules[@]}")"
+
+# Get the plugin settings for the source job so we can apply them - minus this one to the generated steps
+step_plugins="$(echo ${BUILDKITE_PLUGINS} | jq -c '[.[] | select(keys[] | contains("terragrunt-workspace") != true )]')"
+
+# Data Module Commands
+if [[ -n "${data_modules[@]}" ]] ; then
+
+  refresh_commands=()
+  for module in "${data_modules[@]}" ; do
+    refresh_commands+=( "- terragrunt refresh --terragrunt-working-dir ${MODULE_DIR}/${module} ${terragrunt_args}")
+  done
+
+  refresh_commands="$(printf '%s\n' "${refresh_commands[@]}")"
+else
+  refresh_commands=""
+fi
+
+BASE_PIPELINE="steps:"
+PIPELINE="${BASE_PIPELINE}"
+
+PIPELINE+="
+- block: \":terragrunt: [${NAME}] Select Modules\"
+  prompt: Select the modules to deploy
+  fields: 
+    - select: \"Modules\"
+      key: \"modules\"
+      multiple: true
+      options: 
+"
+
+for module in "${deploy_modules[@]}" ; do
+  PIPELINE+="
+        - value: \"${module}\"
+  "
+done
+
+PIPELINE+="
+- label: \":terragrunt: [${NAME}] Plan Modules\"
+  commands:
+    ${refresh_commands}
+    - mkdir -p \"/tmp/\$\${BUILDKITE_JOB_ID}/\"
+    - |-
+      for module in \$\$(buildkite-agent meta-data get modules); do
+        terragrunt plan -out \"/tmp/\$\${BUILDKITE_JOB_ID}/\$\${module}\" --terragrunt-working-dir ${MODULE_DIR}/\$\${module} ${terragrunt_args}
+        buildkite-agent annotate \"**\$\${module}**\n\`\`\`\$\$(terragrunt show -no-color \"/tmp/\$\${BUILDKITE_JOB_ID}/\$\${module}\")\`\`\`\n\" --context \"\$\${module}\"
+      done
+  plugins: ${step_plugins}
+
+- block: \":terragrunt: [${NAME}] Apply Changes?\"
+  prompt: Apply changes?
+
+- label: \":terragrunt: [${NAME}] Apply Modules\"
+  commands:
+    ${refresh_commands}
+    - |-
+      for workspace in \$\$(buildkite-agent meta-data get modules); do
+        terragrunt apply --terragrunt-working-dir ${MODULE_DIR}/\$\${workspace}
+      done
+  plugins: ${step_plugins}
+"
+
+if [[ -n "${DEBUG_PIPELINE_OUTPUT}" ]]; then 
+  echo ":bug: writing pipeline output"
+  echo "${PIPELINE}" > ${DEBUG_PIPELINE_OUTPUT}
+fi 
+
+echo "${PIPELINE}" | buildkite-agent pipeline upload
+

plugin.yml

@@ -0,0 +1,37 @@
+name: Terragrunt Workspace Deploy
+description: Helps with terragrunt workspace deployments
+author: https://github.com/roleyfoley
+requirements: 
+  - terragrunt
+  - terraform
+  - jq
+configuration:
+  properties:
+    name: 
+      description: The name of the module group you are running (test, prod etc.)
+      type: string
+
+    module_dir:
+      description: The path to the modules that you want to apply
+      type: string
+
+    allowed_modules:
+      description: A list of modules that can be used by this plugin
+      type: array
+
+    data_modules:
+      description: Modules that are run with refresh at the start of each command. Used to get the state from modules with only data components
+      type: array
+
+    terragrunt_args:
+      description: Extra arguments to add to terragrunt commands
+      type: array
+
+    debug_pipeline_output:
+      description: A file path to output the pipeline to for testing
+      type: string
+
+  required: 
+    - name
+    - module_dir
+  additionalProperties: false

tests/.gitignore

@@ -0,0 +1 @@
+.outputs

tests/Dockerfile

@@ -0,0 +1,5 @@
+FROM buildkite/plugin-tester
+
+ARG YQ_VERSION=v4.44.3
+ARG YQ_BINARY=yq_linux_amd64
+RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY}.tar.gz -O - | tar xz && mv ${YQ_BINARY} /usr/bin/yq