Merge pull request #2 from opendevstack/task/terraform-support

initial terraform support

.github/workflows/main.yaml

@@ -0,0 +1,48 @@
+name: Task Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  IMAGE_BASE: ${{ github.repository }}
+  IMAGE_NAME: helm
+
+jobs:
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.19'
+      -
+        name: Run tests
+        run: |
+          set -o pipefail
+          make ci | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/''
+      -
+        name: Log into ghcr.io
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Push images to ghcr.io
+        if: ${{ github.event_name != 'pull_request' }}
+        run: |
+          echo "::group::Push ${IMAGE_NAME} to ghcr.io"
+          docker tag localhost:5000/ods-pipeline/${IMAGE_NAME}:latest ghcr.io/${IMAGE_BASE,,}/${IMAGE_NAME}:latest
+          docker push ghcr.io/${IMAGE_BASE,,}/${IMAGE_NAME}:latest
+          echo "::endgroup::"

.github/workflows/release.yaml

@@ -0,0 +1,47 @@
+name: Release images
+on:
+  release:
+    types:
+      - released
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_BASE: ${{ github.repository }}
+  IMAGE_NAME: helm
+
+jobs:
+  build:
+    name: Build and release images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - id: createImageTag
+        name: Create image tag
+        run: |
+          IMAGE_TAG=$(echo ${{ github.event.release.tag_name }} | sed 's/v//')
+          echo "imageTag=$IMAGE_TAG" >> $GITHUB_OUTPUT
+
+      - id: createImageBase
+        run: |
+          echo "imageBase=${IMAGE_BASE,,}" >> $GITHUB_OUTPUT
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and release Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: build/images/Dockerfile.${{env.IMAGE_NAME}}
+          push: true
+          tags: ${{env.REGISTRY}}/${{steps.createImageBase.outputs.imageBase}}/${{env.IMAGE_NAME}}:${{steps.createImageTag.outputs.imageTag}}

.gitignore

@@ -0,0 +1,2 @@
+.ods/
+/.vscode/

CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Note that changes which ONLY affect documentation or the testsuite will not be
+listed in the changelog.
+
+## [Unreleased]
+
+## [0.1.0] - 2023-12-18
+
+Initial version.
+

Makefile

@@ -0,0 +1,68 @@
+SHELL = /bin/bash
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+
+##@ General
+
+# help target is based on https://github.com/operator-framework/operator-sdk/blob/master/release/Makefile.
+.DEFAULT_GOAL := help
+help: ## Show this help screen.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+.PHONY: help
+
+.PHONY: lint-shell
+
+##@ Building
+
+images: image-terraform ## Build images.
+.PHONY: images
+
+image-terraform: ## Build terraform image.
+	docker build \
+        -f build/images/Dockerfile.terraform \
+		-t localhost:5000/ods-pipeline/terraform \
+		.
+.PHONY: image-terraform
+
+tasks: ## Render tasks. Use VERSION=1.0.0 make tasks to render specific version.
+	go run github.com/opendevstack/ods-pipeline/cmd/taskmanifest \
+		-data ImageRepository=ghcr.io/opendevstack/ods-pipeline-terraform \
+		-data Version=$$(cat version) \
+		-template build/tasks/deploy.yaml \
+		-destination tasks/deploy.yaml
+.PHONY: tasks
+
+docs: tasks ## Render documentation for tasks.
+	go run github.com/opendevstack/ods-pipeline/cmd/taskdoc \
+		-task tasks/deploy.yaml \
+		-description build/docs/deploy.adoc \
+		-destination docs/deploy.adoc
+.PHONY: docs
+
+##@ Testing
+
+test: test-cmd test-internal test-e2e ## Run complete testsuite.
+.PHONY: test
+
+test-cmd: ## Run cmd tests.
+	go test -v -count=1 -timeout 10m ./cmd/...
+.PHONY: test-cmd
+
+test-internal: ## Run cmd tests.
+	go test -v -count=1 -timeout 10m ./internal/...
+.PHONY: test-internal
+
+test-e2e: ## Run testsuite of end-to-end task runs.
+	go test -v -count=1 -timeout 10m ./test/e2e/...
+.PHONY: test-e2e
+
+##@ CI
+
+check-docs: docs ## Check docs are up-to-date
+	@printf "Docs / tasks are " && git diff --quiet docs tasks && echo "up-to-date." || (echo "not up-to-date! Run 'make docs' to update."; false)
+.PHONY: check-docs
+
+ci: check-docs test ## Run CI tasks
+.PHONY: ci

README.md

@@ -0,0 +1,27 @@
+# ods-pipeline-terraform
+
+[![Tests](https://github.com/opendevstack/ods-pipeline-terraform/actions/workflows/main.yaml/badge.svg)](https://github.com/opendevstack/ods-pipeline-terraform/actions/workflows/main.yaml)
+
+Tekton task for use with [ODS Pipeline](https://github.com/opendevstack/ods-pipeline) to provision/configure infrastructure using Terraform .
+
+
+## Usage
+
+```yaml
+tasks:
+- name: build
+  taskRef:
+    resolver: git
+    params:
+    - { name: url, value: https://github.com/opendevstack/ods-pipeline-terraform.git }
+    - { name: revision, value: v0.1.0 }
+    - { name: pathInRepo, value: tasks/deploy.yaml }
+    workspaces:
+    - { name: source, workspace: shared-workspace }
+```
+
+See the [documentation](/docs/deploy.adoc) for details and available parameters.
+
+## About this repository
+
+`docs` and `tasks` are generated directories from recipes located in `build`. See the `Makefile` target for how everything fits together.

build/docs/deploy.adoc

@@ -0,0 +1,36 @@
+Deploy Infrastructure with Terraform.
+
+This task supports applying a Terraform configuration.
+
+NOTICE: This is experimental and anything is subject to change or even removal. 
+
+The Terraform configuration is associated with a state file unique to the
+target-environment. 
+
+This task provides a terraform kubernetes backend (see https://developer.hashicorp.com/terraform/language/settings/backends/kubernetes). The `secret_suffix` used is `component`-`target-environment`.
+In the future other backends shall be supported as needed where S3 support is an obvious candidate
+
+This task runs the following terraform commands in sequence:
+
+- `terraform init` with parameters to configure the backend and with env variable TF_PLUGIN_CACHE_DIR set to cache the provider plugins. 
+
+- `terraform plan` if no changes are detected or parameter `plan-only` is true tno actual deployments happen.
+
+- `terraform apply` to apply the changes to the target environment.
+
+It is assumed that secrets needed to connected to the infrastructure managed by terraform are provided with environment variables. The task by default expects a kubernetes secret which is used to derived the needed environment variables from. This can be switched off by setting the `env-from-secret` to "false" in case variables are already provided by other means (such as a podTemplate) or not needed.
+
+This mechanism is also the means to provide terraform input variables.
+
+If the pipeline runs for a repository defining subrepos in its `ods.y(a)ml`
+file, then any terraform configs in those subrepos are processed as well. Note that parameters definitions considered are only the ones defined in the repository for which the pipeline
+runs. Therefore, if you use an umbrella repository to promote an
+application consisting of multiple repositories, the umbrella repository
+needs to define the environment specific values for the subcomponents.
+
+
+The following artifacts are generated by the task and placed into `.ods/artifacts/`
+
+* `deployments/`
+  ** `plan-<env>.txt`
+  ** `<subrepo.name>-plan-<env>.txt` 

build/images/Dockerfile.terraform

@@ -0,0 +1,48 @@
+FROM golang:1.19 as builder
+
+ARG TARGETARCH
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+USER root
+WORKDIR /usr/src/app
+
+ENV TERRAFORM_VERSION=1.6.5 \
+    GOBIN=/usr/local/bin
+
+RUN apt-get update && apt-get install -y --no-install-recommends unzip && rm -rf /var/lib/apt/lists/*
+
+# Install Terraform.
+RUN mkdir -p /tmp/terraform \
+    && cd /tmp \
+    && curl -LO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
+    && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip  -d /tmp/terraform \
+    && mv /tmp/terraform/terraform /usr/local/bin/terraform \
+    && chmod a+x /usr/local/bin/terraform \
+    && terraform version
+
+# Build Go binary.
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+COPY . .
+RUN cd cmd/deploy-terraform && CGO_ENABLED=0 go build -o /usr/local/bin/deploy-terraform
+
+# Final image
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
+
+## 2023-Dec-6: no newer version of git than 2.39 seemed to be available
+ENV GIT_VERSION=2.39
+
+RUN microdnf update && microdnf install --nodocs git-${GIT_VERSION}* && microdnf clean all
+RUN microdnf update && microdnf --help && microdnf repoquery git
+
+COPY --from=builder /usr/local/bin/deploy-terraform /usr/local/bin/deploy-terraform
+COPY --from=builder /usr/local/bin/terraform /usr/local/bin/terraform
+
+RUN terraform version 
+
+VOLUME /workspace/source
+# Ensure that file permissions do not prevent Git checkout into workspace.
+# See https://git-scm.com/docs/git-config/#Documentation/git-config.txt-safedirectory.
+RUN git config --system --add safe.directory '/workspace/source'
+
+USER 1001

build/tasks/deploy.yaml

@@ -0,0 +1,76 @@
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: ods-pipeline-terraform-deploy
+spec:
+  description: |
+    Plan and apply with terraform.
+
+    See https://github.com/opendevstack/ods-pipeline-terraform/blob/v{{.Version}}/docs/terraform.adoc
+
+    And [Terraform kubernetes Backend](https://developer.hashicorp.com/terraform/language/settings/backends/kubernetes)
+
+  params:
+    - name: terraform-dir
+      description: Directory containing terraform files (in the Terraform language). These define the configuration to be applied.
+      type: string
+      default: ./terraform
+    - name: target-environment 
+      description: Terraform state file suffix (tfstate-default-{target-environment})
+      type: string
+      default: 'dev'
+    - name: apply-extra-args
+      description: Extra arguments to pass to terraform apply.
+      type: string
+      default: ''
+    - name: plan-extra-args
+      description: Extra arguments to pass to terraform plan.
+      type: string
+      default: ''
+    - name: plan-only
+      description: |
+        If set to true, the task will do a terraform plan, and then stop.
+      type: string
+      default: 'false'
+    - name: verbose
+      description: More verbose output. DEBUG also implies verbose
+      type: string
+      default: 'false'
+  steps:
+    - name: terraform-from-repo
+      # Image is built from build/package/Dockerfile.terraform.
+      image: '{{.ImageRepository}}/terraform:{{.Version}}'
+      # envFrom:  -- is not working with variable substitutionq
+      #   - secretRef:
+      #       name: terraform-envs-$(params.target-environment}
+      env:
+        - name: DEBUG
+          valueFrom:
+            configMapKeyRef:
+              key: debug
+              name: ods-pipeline
+        - name: HOME
+          value: '/tekton/home'
+      resources: {}
+      script: |
+        # deploy-terraform is built from /cmd/deploy-terraform/main.go.
+        deploy-terraform \
+          -terraform-dir=$(params.terraform-dir) \
+          -target-environment=$(params.target-environment) \
+          -apply-extra-args=$(params.apply-extra-args) \
+          -plan-extra-args=$(params.plan-extra-args) \
+          -plan-only=$(params.plan-only) \
+          -verbose=$(params.verbose)
+      volumeMounts:
+        - mountPath: /etc/ssl/certs/private-cert.pem
+          name: private-cert
+          readOnly: true
+          subPath: tls.crt
+      workingDir: $(workspaces.source.path)
+  volumes:
+    - name: private-cert
+      secret:
+        secretName: ods-private-cert
+        optional: true
+  workspaces:
+    - name: source

cmd/deploy-terraform/backend-kubernetes.tf

@@ -0,0 +1,6 @@
+terraform {
+    backend "kubernetes" {
+        secret_suffix = "{{.SecretSuffix}}"
+        in_cluster_config = true 
+    }
+}