[MM-61113] Allow multiple customizable subnets (#833)

* multiple subnet support

* wait for terraform stdout

* revert elasticsearch zone awareness

* fixed line removed on merge conflict

* AWSAvailabilityZone compatibility

* support a single subnet in db groups

* refactor to use explicit subnets

* elasticsearch multiple subnet support

* docs

* review comments

* use list of subnets for all resources

* tags for db-subnet-group

* updated sample files

* aws_db_subnet_group -> aws_elasticcache_subnet_group

* use reflect.DeepEqual to simplify code

* Update docs/config/deployer.md

Co-authored-by: Alejandro García Montoro <[email protected]>

* Update deployment/config.go

Co-authored-by: Alejandro García Montoro <[email protected]>

* docs link to aws docs

* function to struct method

---------

Co-authored-by: Alejandro García Montoro <[email protected]>

config/deployer.sample.json

@@ -4,7 +4,17 @@
   "AWSAMI": "ami-003d3d03cfe1b0468",
   "ClusterName": "loadtest",
   "ClusterVpcID": "",
-  "ClusterSubnetID": "",
+  "ClusterSubnetIDs": {
+    "Agent": [],
+    "App": [],
+    "Database": [],
+    "ElasticSearch": [],
+    "Job": [],
+    "Keycloak": [],
+    "Metrics": [],
+    "Proxy": [],
+    "Redis": []
+  },
   "AppInstanceCount": 1,
   "AppInstanceType": "c7i.xlarge",
   "AgentInstanceCount": 2,
@@ -18,7 +28,9 @@
     "SnapshotRepository": "",
     "SnapshotName": "",
     "RestoreTimeoutMinutes": 45,
-    "ClusterTimeoutMinutes": 45
+    "ClusterTimeoutMinutes": 45,
+    "ZoneAwarenessEnabled": false,
+    "ZoneAwarenessAZCount": 2
   },
   "RedisSettings": {
     "Enabled": false,

config/deployer.sample.toml

@@ -24,7 +24,6 @@ AppInstanceType = 'c5.xlarge'
 
 # Cluster configuration
 ClusterName = 'loadtest'
-ClusterSubnetID = ''
 ClusterVpcID = ''
 
 # Database configuration
@@ -38,12 +37,25 @@ TerraformStateDir = '/var/lib/mattermost-load-test-ng'
 # Proxy server configuration
 ProxyInstanceType = 'c5.xlarge'
 
+[ClusterSubnetIDs]
+App = []
+Job = []
+Proxy = []
+Agent = []
+ElasticSearch = []
+Metrics = []
+Keycloak = []
+Database = []
+Redis = []
+
 [ElasticSearchSettings]
 CreateRole = false
 InstanceCount = 0
 InstanceType = 'r6g.large.search'
 Version = 'Elasticsearch_7.10'
 VpcID = ''
+ZoneAwarenessEnabled = false
+ZoneAwarenessAZCount = 2
 
 [ExternalBucketSettings]
 AmazonS3AccessKeyId = ''

deployment/config.go

@@ -4,14 +4,17 @@
 package deployment
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"reflect"
 	"regexp"
 	"strings"
 
 	"github.com/mattermost/mattermost-load-test-ng/defaults"
 	"github.com/mattermost/mattermost-load-test-ng/loadtest/report"
 	"github.com/mattermost/mattermost-load-test-ng/logger"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
 )
 
 var esDomainNameRe = regexp.MustCompile(`^[a-z][a-z0-9\-]{2,27}$`)
@@ -29,8 +32,8 @@ type Config struct {
 	ClusterName string `default:"loadtest" validate:"alpha"`
 	// ClusterVpcID is the id of the VPC associated to the resources.
 	ClusterVpcID string
-	// ClusterSubnetID is the id of the subnet associated to the resources.
-	ClusterSubnetID string
+	// ClusterSubnetIDs is the ids of the subnets associated to each resource type.
+	ClusterSubnetIDs ClusterSubnetIDs
 	// Number of application instances.
 	AppInstanceCount int `default:"1" validate:"range:[0,)"`
 	// Type of the EC2 instance for app.
@@ -115,6 +118,33 @@ type Config struct {
 	StorageSizes StorageSizes
 }
 
+// ClusterSubnetIDs contains the subnet ids for the different types of instances.
+type ClusterSubnetIDs struct {
+	App           []string `json:"app"`
+	Job           []string `json:"job"`
+	Proxy         []string `json:"proxy"`
+	Agent         []string `json:"agent"`
+	ElasticSearch []string `json:"elasticsearch"`
+	Metrics       []string `json:"metrics"`
+	Keycloak      []string `json:"keycloak"`
+	Database      []string `json:"database"`
+	Redis         []string `json:"redis"`
+}
+
+// IsAnySet returns true if any of the subnet ids are set.
+func (c *ClusterSubnetIDs) IsAnySet() bool {
+	return !reflect.DeepEqual(c, &ClusterSubnetIDs{})
+}
+
+func (c *ClusterSubnetIDs) String() string {
+	b, err := json.Marshal(c)
+	if err != nil {
+		mlog.Error("Failed to marshal ClusterSubnetIDs", mlog.Err(err))
+		return "{}"
+	}
+	return string(b)
+}
+
 type StorageSizes struct {
 	// Size, in GiB, for the storage of the agents instances
 	Agent int `default:"10"`
@@ -251,8 +281,6 @@ type ElasticSearchSettings struct {
 	InstanceType string
 	// Elasticsearch version to be deployed.
 	Version string `default:"Elasticsearch_7.10"`
-	// Id of the VPC associated with the instance to be created.
-	VpcID string
 	// Set to true if the AWSServiceRoleForAmazonElasticsearchService role should be created.
 	CreateRole bool
 	// SnapshotRepository is the name of the S3 bucket where the snapshot to restore lives.
@@ -263,6 +291,10 @@ type ElasticSearchSettings struct {
 	RestoreTimeoutMinutes int `default:"45" validate:"range:[0,)"`
 	// ClusterTimeoutMinutes is the maximum time, in minutes, that the system will wait for the cluster status to get green.
 	ClusterTimeoutMinutes int `default:"45" validate:"range:[0,)"`
+	// ZoneAwarenessEnabled indicates whether to enable zone awareness or not.
+	ZoneAwarenessEnabled bool `default:"false"`
+	// ZoneAwarenessAZCount indicates the number of availability zones to use for zone awareness.
+	ZoneAwarenessAZCount int `default:"2" validate:"range:[1,3]"`
 }
 
 type RedisSettings struct {
@@ -313,6 +345,10 @@ func (p DBParameters) String() string {
 
 // IsValid reports whether a given deployment config is valid or not.
 func (c *Config) IsValid() error {
+	if c.ClusterSubnetIDs.IsAnySet() && c.ClusterVpcID == "" {
+		return errors.New("vpc_id is required when any subnet is specified")
+	}
+
 	if !checkPrefix(c.MattermostDownloadURL) {
 		return fmt.Errorf("mattermost download url is not in correct format: %q", c.MattermostDownloadURL)
 	}
@@ -352,8 +388,8 @@ func (c *Config) validateElasticSearchConfig() error {
 	}
 
 	if (c.ElasticSearchSettings != ElasticSearchSettings{}) {
-		if c.ElasticSearchSettings.VpcID == "" {
-			return errors.New("VpcID must be set in order to create an Elasticsearch instance")
+		if c.ClusterVpcID == "" {
+			return errors.New("ClusterVpcID must be set in order to create an Elasticsearch instance")
 		}
 
 		domainName := c.ClusterName + "-es"

deployment/config_test.go

@@ -74,13 +74,15 @@ func TestConfigIsValid(t *testing.T) {
 func TestValidateElasticSearchConfig(t *testing.T) {
 	baseValidConfig := func() Config {
 		return Config{
+			ClusterVpcID:          "vpc-01234567890abcdef",
 			ClusterName:           "clustername",
 			MattermostDownloadURL: "https://latest.mattermost.com/mattermost-enterprise-linux",
 			LoadTestDownloadURL:   "https://github.com/mattermost/mattermost-load-test-ng/releases/download/v1.20.0/mattermost-load-test-ng-v1.20.0-linux-amd64.tar.gz",
 			ElasticSearchSettings: ElasticSearchSettings{
-				InstanceCount: 1,
-				Version:       "Elasticsearch_7.10",
-				VpcID:         "vpc-01234567890abcdef",
+				InstanceCount:      1,
+				Version:            "OpenSearch_2.7",
+				SnapshotRepository: "somerepo",
+				SnapshotName:       "somename",
 			},
 		}
 	}
@@ -102,7 +104,7 @@ func TestValidateElasticSearchConfig(t *testing.T) {
 
 	t.Run("invalid VPC ID", func(t *testing.T) {
 		cfg := baseValidConfig()
-		cfg.ElasticSearchSettings.VpcID = ""
+		cfg.ClusterVpcID = ""
 		require.Error(t, cfg.validateElasticSearchConfig())
 	})