Use comment trigger to run acceptance tests for PRs. (#929)

* Initial workflow testing.

* Allow running tests for multiple packages concurrently.

* Set PR status checks.

* Cancel pending on new run.

* Run schedualed sweeper after acceptance tests complete.

* Add CONTRIBUTING.md

.github/workflows/acceptance-test-pr.yml

@@ -0,0 +1,71 @@
+on:
+  repository_dispatch:
+    types: [testacc-command]
+
+name: Acceptance Test PR
+
+jobs:
+  acceptance:
+    runs-on: ubuntu-latest
+    if:
+      github.event.client_payload.slash_command.sha != '' &&
+      github.event.client_payload.slash_command.pkg != '' &&
+      github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.sha
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.event.client_payload.slash_command.pkg }}-${{ github.event.client_payload.pull_request.number }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Set status pending
+        run: |
+          gh api --method POST -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ github.event.client_payload.pull_request.head.sha }} \
+            -f state='pending' \
+            -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \
+            -f description='Running acceptance tests...' \
+            -f context='acceptance/${{ github.event.client_payload.slash_command.pkg }}'
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.x
+
+      - name: Checkout PR
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.client_payload.slash_command.sha }}
+
+      - name: Run Acceptance Tests
+        id: run_tests
+        run: make PKG_NAME="${{ github.event.client_payload.slash_command.pkg }}" testacc
+        env:
+          ACCTEST_PARALLELISM: 10
+          DIGITALOCEAN_TOKEN: ${{ secrets.ACCEPTANCE_TESTS_TOKEN }}
+          SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
+          SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
+
+      - name: Results
+        if: always()
+        run: |
+          if [[ ${{ steps.run_tests.outcome }} == 'success' ]]; then
+            echo "test_result=success" >> $GITHUB_ENV
+            echo "check_description=Acceptance tests for ${{ github.event.client_payload.slash_command.pkg }} successful" >> $GITHUB_ENV
+          else
+            echo "test_result=failure" >> $GITHUB_ENV
+            echo "check_description=Acceptance tests for ${{ github.event.client_payload.slash_command.pkg }} failed" >> $GITHUB_ENV
+          fi
+
+      - name: Update status on PR
+        if: always()
+        run: |
+          gh api --method POST -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ github.event.client_payload.pull_request.head.sha }} \
+            -f state='${{ env.test_result }}' \
+            -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \
+            -f description='${{ env.check_description }}' \
+            -f context='acceptance/${{ github.event.client_payload.slash_command.pkg }}'
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/acceptance-test-schedule.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18.x
+          go-version: 1.19.x
 
       - name: Checkout
         uses: actions/checkout@v2
@@ -30,13 +30,15 @@ jobs:
           SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
 
   sweep:
+    if: ${{ always() }}
+    needs: acceptance
     runs-on: ubuntu-latest
 
     steps:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18.x
+          go-version: 1.19.x
 
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/acceptance-test-trigger.yml

@@ -0,0 +1,19 @@
+name: Acceptance Test Trigger
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  testacc-trigger:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Slash Command Dispatch
+        uses: peter-evans/slash-command-dispatch@v1
+        with:
+          token: ${{ secrets.TRIGGER_TOKEN }}
+          issue-type: pull-request
+          commands: testacc
+          named-args: true
+          permission: write

.github/workflows/test.yml

@@ -38,57 +38,3 @@ jobs:
 
       - name: make test
         run: make test
-
-  acceptance:
-    # Runs the acceptance test suite using the `Acceptance Tests` environment.
-    # The enviroment should be configured to require reviewers approve the
-    # step. This allows the tests to be added as a check to PRs.
-    #
-    # This job has the `continue-on-error` set to true which prevents blocking
-    # PRs if the tests fail.
-    needs: unit
-    runs-on: ubuntu-latest
-    environment:
-      name: Acceptance Tests
-
-    concurrency: acceptance_tests
-
-    continue-on-error: true
-
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.x
-
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: make testacc
-        run: make testacc
-        env:
-          TESTARGS: -parallel 20
-          DIGITALOCEAN_TOKEN: ${{ secrets.ACCEPTANCE_TESTS_TOKEN }}
-          SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
-          SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
-
-  sweep:
-    runs-on: ubuntu-latest
-    needs: acceptance
-    concurrency: acceptance_tests
-
-    steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.19.x
-
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: make sweep
-        run: make sweep
-        env:
-          DIGITALOCEAN_TOKEN: ${{ secrets.ACCEPTANCE_TESTS_TOKEN }}
-          SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
-          SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}

CONTRIBUTING.md

@@ -0,0 +1,126 @@
+Developing the Provider
+---------------------------
+
+If you wish to work on the provider, you'll first need [Go](http://www.golang.org) installed on your machine (version 1.11+ is *required*). You'll also need to correctly setup a [GOPATH](http://golang.org/doc/code.html#GOPATH), as well as adding `$GOPATH/bin` to your `$PATH`.
+
+To compile the provider, run `make build`. This will build the provider and put the provider binary in the `$GOPATH/bin` directory.
+
+```sh
+$ make build
+...
+$ $GOPATH/bin/terraform-provider-digitalocean
+...
+```
+
+In order to test the provider, you can simply run `make test`.
+
+```sh
+$ make test
+```
+
+In order to run the full suite of acceptance tests, run `make testacc`.
+
+*Note:* Acceptance tests create real resources, and often cost money to run.
+
+```sh
+$ make testacc
+```
+
+In order to run a specific acceptance test, use the `TESTARGS` environment variable. For example, the following command will run `TestAccDigitalOceanDomain_Basic` acceptance test only:
+
+```sh
+$ make testacc TESTARGS='-run=TestAccDigitalOceanDomain_Basic'
+```
+
+All acceptance tests for a specific package can be run by setting the `PKG_NAME` environment variable. For example:
+
+```sh
+$ make testacc PKG_NAME=digitalocean/account
+```
+
+In order to check changes you made locally to the provider, you can use the binary you just compiled by adding the following
+to your `~/.terraformrc` file. This is valid for Terraform 0.14+. Please see
+[Terraform's documentation](https://www.terraform.io/docs/cli/config/config-file.html#development-overrides-for-provider-developers)
+for more details.
+
+```
+provider_installation {
+
+  # Use /home/developer/go/bin as an overridden package directory
+  # for the digitalocean/digitalocean provider. This disables the version and checksum
+  # verifications for this provider and forces Terraform to look for the
+  # digitalocean provider plugin in the given directory.
+  dev_overrides {
+    "digitalocean/digitalocean" = "/home/developer/go/bin"
+  }
+
+  # For all other providers, install them directly from their origin provider
+  # registries as normal. If you omit this, Terraform will _only_ use
+  # the dev_overrides block, and so no other providers will be available.
+  direct {}
+}
+```
+
+For information about writing acceptance tests, see the main Terraform [contributing guide](https://github.com/hashicorp/terraform/blob/master/.github/CONTRIBUTING.md#writing-acceptance-tests).
+
+Releasing the Provider
+----------------------
+
+To release the provider:
+
+1. Use
+   [github-changelog-generator](https://github.com/digitalocean/github-changelog-generator)
+   to list the changes since the last release and decide what kind of release
+   you are doing (bugfix, feature or breaking).
+1. Create a new commit that only contains updates to
+   [CHANGELOG.md](CHANGELOG.md) listing the respective changes for the new
+   version. Godo follows [semver](https://www.semver.org/) versioning
+   semantics.
+1. Once the CHANGELOG.md commit is merged, create a new tag on that commit with
+   the new version that will be released (be sure to pull the latest from
+   git).
+
+   ```bash
+   git tag -m "release $new_version" -a "$new_version"
+   ```
+
+1. Push the tag:
+
+   ```bash
+   git push "$origin" tag "$new_version"
+   ```
+
+This repository contains a GitHub Action configured to automatically build and
+publish assets for release when a tag is pushed that matches the pattern `v*`
+(ie. `v0.1.0`).
+
+A [Goreleaser](https://goreleaser.com/) configuration is provided that produces
+build artifacts matching the [layout required](https://www.terraform.io/docs/registry/providers/publishing.html#manually-preparing-a-release)
+to publish the provider in the Terraform Registry.
+
+Releases will appear as drafts. Once marked as published on the GitHub Releases page,
+they will become available via the Terraform Registry.
+
+Reviewing Pull Requests
+-----------------------
+
+Acceptance tests use the production API and create resources that incur costs.
+Running the full suite of acceptance tests can also take quite a long time to run.
+In order to prevent misuse, the acceptance tests for a pull request must be manually
+triggered by a reviewer with write access to the repository.
+
+To trigger a run of the acceptance tests for a PR, you may use the `/testacc` in a
+comment. The `pkg` and `sha` arguments are required. This allows us to limit the
+packages being tested for quick feedback and protect against timing attacks.
+For example, to run the acceptance tests for the `droplet` package, the command
+may look like:
+
+    /testacc pkg=digitalocean/droplet sha=d358bd2418b4e30d7bdf2b98b4c151e357814c63
+
+To run the entire suite, use `pkg=digitalocean`.
+
+If multiple packages are to be tested, each command must be posted as a separate
+comment. Only the first line of the comment is evaluated.
+
+We leverage the [`peter-evans/slash-command-dispatch`](https://github.com/peter-evans/slash-command-dispatch)
+GitHub Action for the command processing.

GNUmakefile

@@ -1,8 +1,8 @@
 TEST?=$$(go list ./... |grep -v 'vendor')
 GOFMT_FILES?=$$(find . -name '*.go' |grep -v vendor)
-PKG_NAME:=digitalocean
-ACCTEST_TIMEOUT:=120m
-ACCTEST_PARALLELISM:=2
+PKG_NAME?=digitalocean
+ACCTEST_TIMEOUT?=120m
+ACCTEST_PARALLELISM?=2
 
 default: build