chore: Add test to make sure flags are up to date and sorted

runatlantis · Nov 5, 2024 · a520f0b · a520f0b
1 parent 24a5458
commit a520f0b
Show file tree

Hide file tree

Showing 2 changed files with 207 additions and 84 deletions.
diff --git a/cmd/server_test.go b/cmd/server_test.go
@@ -14,10 +14,13 @@
 package cmd
 
 import (
+	"bufio"
+	"cmp"
 	"fmt"
 	"os"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 
@@ -225,21 +228,32 @@ func TestExecute_Flags(t *testing.T) {
 	}
 }
 
-func TestUserConfigAllTested(t *testing.T) {
-	t.Log("All settings in userConfig should be tested.")
-
+func getUserConfigKeysWithFlags() []string {
+	var ret []string
 	u := reflect.TypeOf(server.UserConfig{})
 
 	for i := 0; i < u.NumField(); i++ {
 
 		userConfigKey := u.Field(i).Tag.Get("mapstructure")
+		// By default, we expect all fields in UserConfig to have flags defined in server.go and tested here in server_test.go
+		// Some fields are too complicated to have flags, so are only expressible in the config yaml
+		flagKey := u.Field(i).Tag.Get("flag")
+		if flagKey == "false" {
+			continue
+		}
+		ret = append(ret, userConfigKey)
+
+	}
+	return ret
+
+}
+
+func TestUserConfigAllTested(t *testing.T) {
+	t.Log("All settings in userConfig should be tested.")
+
+	for _, userConfigKey := range getUserConfigKeysWithFlags() {
+
 		t.Run(userConfigKey, func(t *testing.T) {
-			// By default, we expect all fields in UserConfig to have flags defined in server.go and tested here in server_test.go
-			// Some fields are too complicated to have flags, so are only expressible in the config yaml
-			flagKey := u.Field(i).Tag.Get("flag")
-			if flagKey == "false" {
-				return
-			}
 			// If a setting is configured in server.UserConfig, it should be tested here. If there is no corresponding const
 			// for specifying the flag, that probably means one *also* needs to be added to server.go
 			if _, ok := testFlags[userConfigKey]; !ok {
@@ -251,6 +265,94 @@ func TestUserConfigAllTested(t *testing.T) {
 
 }
 
+func getDocumentedFlags(t *testing.T) []string {
+
+	var ret []string
+	docFile := "../runatlantis.io/docs/server-configuration.md"
+
+	file, err := os.Open(docFile)
+	Ok(t, err)
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.HasPrefix(line, "### ") {
+			continue
+		}
+		split := strings.Split(line, "`")
+		if len(split) != 3 {
+			t.Errorf("Unexpected line in %s: %s", docFile, line)
+			continue
+		}
+		flag := split[1]
+		if !strings.HasPrefix(flag, "--") {
+			t.Errorf("Unexpected line in %s: %s", docFile, line)
+			continue
+		}
+		flag = strings.TrimPrefix(flag, "--")
+		ret = append(ret, flag)
+	}
+
+	if err := scanner.Err(); err != nil {
+		Ok(t, err)
+	}
+	return ret
+}
+
+func testIsSorted[S ~[]E, E cmp.Ordered](t *testing.T, x S) {
+	// TODO: This is n^2, probably a better algorithm for this
+	// Also, this works best for lists that are mostly sorted, if the whole thing is wrong, it's just
+	// going to say that every individual element is out of order
+	for i, elem := range x {
+		for j, compareTo := range x {
+			if i == j {
+				continue
+			}
+			if i > j && cmp.Less(elem, compareTo) {
+				t.Errorf("%v is out of order (should be before %v)", elem, compareTo)
+				break
+			}
+			if i < j && cmp.Less(compareTo, elem) {
+				t.Errorf("%v is out of order (should be after %v)", elem, compareTo)
+				break
+			}
+		}
+	}
+}
+
+func TestAllFlagsDocumented(t *testing.T) {
+	// This is not a unit test per se, but is a helpful way of making sure when flags are added/removed
+	// the corresponding documentation is kept up-to-date.
+	t.Log("All flags in userConfig should have documentation.")
+
+	userConfigKeys := getUserConfigKeysWithFlags()
+	documentedFlags := getDocumentedFlags(t)
+
+	testIsSorted(t, documentedFlags)
+	slices.Sort(userConfigKeys)
+	slices.Sort(documentedFlags)
+
+	for _, userConfigKey := range userConfigKeys {
+		_, found := slices.BinarySearch(documentedFlags, userConfigKey)
+		if !found {
+			t.Errorf("Found undocumented config key: %s", userConfigKey)
+		}
+	}
+
+	for _, documentedFlag := range documentedFlags {
+		// --help and --config are documented but don't have a setting on userConfig
+		if documentedFlag == "help" || documentedFlag == "config" {
+			continue
+		}
+		_, found := slices.BinarySearch(userConfigKeys, documentedFlag)
+		if !found {
+			t.Errorf("Found documentation for flag that doesn't exist: %s", documentedFlag)
+		}
+	}
+
+}
+
 func TestExecute_ConfigFile(t *testing.T) {
 	t.Log("Should use all the values from the config file.")
 	// Use yaml package to quote values that need quoting

diff --git a/runatlantis.io/docs/server-configuration.md b/runatlantis.io/docs/server-configuration.md
@@ -438,6 +438,17 @@ and set `--autoplan-modules` to `false`.
 
   If `disable-autoplan` property is `true`, this flag has no effect.
 
+### `--disable-global-apply-lock`
+
+  ```bash
+  atlantis server --disable-global-apply-lock
+  # or
+  ATLANTIS_DISABLE_GLOBAL_APPLY_LOCK=true
+  ```
+
+  If true, removes button in the UI that allows users to globally disable apply commands.
+
+
 ### `--disable-markdown-folding`
 
   ```bash
@@ -468,6 +479,16 @@ and set `--autoplan-modules` to `false`.
 
   Stops atlantis from unlocking a pull request with this label. Defaults to "" (feature disabled).
 
+### `--discard-approval-on-plan`
+
+  ```bash
+  atlantis server --discard-approval-on-plan
+  # or
+  ATLANTIS_DISCARD_APPROVAL_ON_PLAN=true
+  ```
+
+  If set, discard approval if a new plan has been executed. Currently only supported in Github.
+
 ### `--emoji-reaction`
 
   ```bash
@@ -546,66 +567,6 @@ and set `--autoplan-modules` to `false`.
 
   Fail and do not run the requested Atlantis command if any of the pre workflow hooks error.
 
-### `--gitea-base-url`
-
-  ```bash
-  atlantis server --gitea-base-url="http://your-gitea.corp:7990/basepath"
-  # or
-  ATLANTIS_GITEA_BASE_URL="http://your-gitea.corp:7990/basepath"
-  ```
-
-  Base URL of Gitea installation. Must include `http://` or `https://`. Defaults to `https://gitea.com` if left empty/absent.
-
-### `--gitea-token`
-
-  ```bash
-  atlantis server --gitea-token="token"
-  # or (recommended)
-  ATLANTIS_GITEA_TOKEN="token"
-  ```
-
-  Gitea app password of API user.
-
-### `--gitea-user`
-
-  ```bash
-  atlantis server --gitea-user="myuser"
-  # or
-  ATLANTIS_GITEA_USER="myuser"
-  ```
-
-  Gitea username of API user.
-
-### `--gitea-webhook-secret`
-
-  ```bash
-  atlantis server --gitea-webhook-secret="secret"
-  # or (recommended)
-  ATLANTIS_GITEA_WEBHOOK_SECRET="secret"
-  ```
-
-  Secret used to validate Gitea webhooks.
-
-  ::: warning SECURITY WARNING
-  If not specified, Atlantis won't be able to validate that the incoming webhook call came from Gitea.
-  This means that an attacker could spoof calls to Atlantis and cause it to perform malicious actions.
-  :::
-
-### `--gitea-page-size`
-
-  ```bash
-  atlantis server --gitea-page-size=30
-  # or (recommended)
-  ATLANTIS_GITEA_PAGE_SIZE=30
-  ```
-
-  Number of items on a single page in Gitea paged responses.
-
-  ::: warning Configuration dependent
-  The default value conforms to the Gitea server's standard config setting: DEFAULT_PAGING_NUM
- The highest valid value depends on the Gitea server's config setting: MAX_RESPONSE_ITEMS
-  :::
-
 ### `--gh-allow-mergeable-bypass-apply`
 
   ```bash
@@ -642,6 +603,21 @@ and set `--autoplan-modules` to `false`.
   After which Atlantis will display your new app's credentials: your app's ID, its generated `--gh-webhook-secret` and the contents of the file for `--gh-app-key-file`. Update your Atlantis config accordingly, and restart the server.
   :::
 
+### `--gh-app-installation-id`
+
+  ```bash
+  atlantis server --gh-app-installation-id="123"
+  # or
+  ATLANTIS_GH_APP_INSTALLATION_ID="123"
+  ```
+
+The installation ID of a specific instance of a GitHub application. Normally this value is
+derived by querying GitHub for the list of installations of the ID supplied via `--gh-app-id` and selecting
+the first one found and where multiple installations results in an error. Use this flag if you have multiple
+instances of Atlantis but you want to use a single already-installed GitHub app for all of them. You would normally do this if
+you are running a proxy as your single GitHub application that will proxy to an appropriate Atlantis instance
+based on the organization or user that triggered the webhook.
+
 ### `--gh-app-key`
 
   ```bash
@@ -687,21 +663,6 @@ and set `--autoplan-modules` to `false`.
   Hostname of your GitHub Enterprise installation. If using [GitHub.com](https://github.com),
   don't set. Defaults to `github.com`.
 
-### `--gh-app-installation-id`
-
-  ```bash
-  atlantis server --gh-app-installation-id="123"
-  # or
-  ATLANTIS_GH_APP_INSTALLATION_ID="123"
-  ```
-
-The installation ID of a specific instance of a GitHub application. Normally this value is
-derived by querying GitHub for the list of installations of the ID supplied via `--gh-app-id` and selecting
-the first one found and where multiple installations results in an error. Use this flag if you have multiple
-instances of Atlantis but you want to use a single already-installed GitHub app for all of them. You would normally do this if
-you are running a proxy as your single GitHub application that will proxy to an appropriate Atlantis instance
-based on the organization or user that triggered the webhook.
-
 ### `--gh-org`
 
   ```bash
@@ -778,6 +739,66 @@ based on the organization or user that triggered the webhook.
   This means that an attacker could spoof calls to Atlantis and cause it to perform malicious actions.
   :::
 
+### `--gitea-base-url`
+
+  ```bash
+  atlantis server --gitea-base-url="http://your-gitea.corp:7990/basepath"
+  # or
+  ATLANTIS_GITEA_BASE_URL="http://your-gitea.corp:7990/basepath"
+  ```
+
+  Base URL of Gitea installation. Must include `http://` or `https://`. Defaults to `https://gitea.com` if left empty/absent.
+
+### `--gitea-page-size`
+
+  ```bash
+  atlantis server --gitea-page-size=30
+  # or (recommended)
+  ATLANTIS_GITEA_PAGE_SIZE=30
+  ```
+
+  Number of items on a single page in Gitea paged responses.
+
+  ::: warning Configuration dependent
+  The default value conforms to the Gitea server's standard config setting: DEFAULT_PAGING_NUM
+ The highest valid value depends on the Gitea server's config setting: MAX_RESPONSE_ITEMS
+  :::
+
+### `--gitea-token`
+
+  ```bash
+  atlantis server --gitea-token="token"
+  # or (recommended)
+  ATLANTIS_GITEA_TOKEN="token"
+  ```
+
+  Gitea app password of API user.
+
+### `--gitea-user`
+
+  ```bash
+  atlantis server --gitea-user="myuser"
+  # or
+  ATLANTIS_GITEA_USER="myuser"
+  ```
+
+  Gitea username of API user.
+
+### `--gitea-webhook-secret`
+
+  ```bash
+  atlantis server --gitea-webhook-secret="secret"
+  # or (recommended)
+  ATLANTIS_GITEA_WEBHOOK_SECRET="secret"
+  ```
+
+  Secret used to validate Gitea webhooks.
+
+  ::: warning SECURITY WARNING
+  If not specified, Atlantis won't be able to validate that the incoming webhook call came from Gitea.
+  This means that an attacker could spoof calls to Atlantis and cause it to perform malicious actions.
+  :::
+
 ### `--gitlab-hostname`
 
   ```bash