Improvements to VPCs and RDS

Makefile

@@ -73,6 +73,8 @@ endif
 	$(eval PACKAGE_NAME=mattermost-load-test-ng-$(DIST_VER)-$(PLATFORM))
 	cp -r $(PLATFORM_DIST_PATH) $(DIST_PATH)/$(PACKAGE_NAME)
 	tar -C $(DIST_PATH) -czf $(DIST_PATH)/$(PACKAGE_NAME).tar.gz $(PACKAGE_NAME)
+	rm -rf $(DIST_ROOT)/latest.tar.gz
+	cp $(DIST_PATH)/$(PACKAGE_NAME).tar.gz $(DIST_ROOT)/latest.tar.gz
 	rm -rf $(DIST_PATH)/$(PACKAGE_NAME)
 
 verify-gomod: ## Run go mod verify.

cmd/ltctl/collect.go

@@ -122,7 +122,7 @@ func createClients(output *terraform.Output) (map[string]*ssh.Client, error) {
 	clients := make(map[string]*ssh.Client)
 	if output.HasProxy() {
 		for i, inst := range output.Proxies {
-			sshc, err := extAgent.NewClient(inst.PublicIP)
+			sshc, err := extAgent.NewClient(inst.PrivateIP)
 			if err != nil {
 				return nil, fmt.Errorf("error in getting ssh connection %w", err)
 			}
@@ -131,15 +131,15 @@ func createClients(output *terraform.Output) (map[string]*ssh.Client, error) {
 	}
 
 	for i, instance := range output.Instances {
-		sshc, err := extAgent.NewClient(instance.PublicIP)
+		sshc, err := extAgent.NewClient(instance.PrivateIP)
 		if err != nil {
 			return nil, fmt.Errorf("error in getting ssh connection %w", err)
 		}
 		clients[fmt.Sprintf("app%d", i)] = sshc
 	}
 
 	for i, agent := range output.Agents {
-		sshc, err := extAgent.NewClient(agent.PublicIP)
+		sshc, err := extAgent.NewClient(agent.PrivateIP)
 		if err != nil {
 			return nil, fmt.Errorf("error in getting ssh connection %w", err)
 		}

cmd/ltctl/loadtest.go

@@ -209,7 +209,7 @@ func RunLoadTestStatusCmdF(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	prometheusURL := fmt.Sprintf("http://%s:9090", tfOutput.MetricsServer.PublicIP)
+	prometheusURL := fmt.Sprintf("http://%s:9090", tfOutput.MetricsServer.PrivateIP)
 	helper, err := prometheus.NewHelper(prometheusURL)
 	if err != nil {
 		return fmt.Errorf("failed to create prometheus.Helper: %w", err)

cmd/ltctl/main.go

@@ -28,7 +28,7 @@ func RunCreateCmdF(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to create terraform engine: %w", err)
 	}
 
-	initData := config.DBDumpURI == ""
+	initData := config.DBDumpURI == "" || config.ExternalDBSettings.DataSource != ""
 	err = t.Create(initData)
 	if err != nil {
 		return fmt.Errorf("failed to create terraform env: %w", err)
@@ -157,6 +157,9 @@ func RunSSHListCmdF(cmd *cobra.Command, args []string) error {
 	if output.HasMetrics() {
 		fmt.Printf(" - %s\n", output.MetricsServer.Tags.Name)
 	}
+	if output.HasKeycloak() {
+		fmt.Printf(" - %s\n", output.KeycloakServer.Tags.Name)
+	}
 	return nil
 }
 

cmd/ltctl/report.go

@@ -90,7 +90,7 @@ func RunGenerateReportCmdF(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("could not parse output: %w", err)
 		}
-		promURL = "http://" + output.MetricsServer.PublicIP + ":9090"
+		promURL = "http://" + output.MetricsServer.PrivateIP + ":9090"
 	}
 
 	helper, err := prometheus.NewHelper(promURL)

cmd/ltctl/reset.go

@@ -46,15 +46,17 @@ func RunResetCmdF(cmd *cobra.Command, args []string) error {
 
 	appClients := make([]*ssh.Client, len(output.Instances))
 	for i, instance := range output.Instances {
-		client, err := extAgent.NewClient(instance.PublicIP)
+		// BRANCH: Using private ip
+		client, err := extAgent.NewClient(instance.PrivateIP)
 		if err != nil {
 			return fmt.Errorf("error in getting ssh connection %w", err)
 		}
 		defer client.Close()
 		appClients[i] = client
 	}
 
-	agentClient, err := extAgent.NewClient(output.Agents[0].PublicIP)
+	// BRANCH: Private IP
+	agentClient, err := extAgent.NewClient(output.Agents[0].PrivateIP)
 	if err != nil {
 		return fmt.Errorf("error in getting ssh connection %w", err)
 	}

cmd/ltkeycloak/from_mattermost.go

@@ -237,10 +237,10 @@ func RunSyncFromMattermostCommandF(cmd *cobra.Command, _ []string) error {
 		if err != nil {
 			return fmt.Errorf("failed to get terraform output: %w", err)
 		}
-		if terraformOutput.KeycloakServer.PublicDNS == "" {
+		if terraformOutput.KeycloakServer.PrivateDNS == "" {
 			return fmt.Errorf("keycloak database cluster not found in terraform output")
 		}
-		keycloakHost = terraformOutput.KeycloakServer.PublicDNS
+		keycloakHost = terraformOutput.KeycloakServer.PrivateDNS
 	}
 
 	keycloakRealm, err := cmd.Flags().GetString("keycloak-realm")

comparison/deployment.go

@@ -62,7 +62,7 @@ func provisionFiles(t *terraform.Terraform, dpConfig *deploymentConfig, baseBuil
 	}
 	clients := make([]*ssh.Client, len(output.Instances))
 	for i, instance := range output.Instances {
-		client, err := extAgent.NewClient(instance.PublicIP)
+		client, err := extAgent.NewClient(instance.PrivateIP)
 		if err != nil {
 			return fmt.Errorf("error in getting ssh connection %w", err)
 		}

comparison/loadtest.go

@@ -178,15 +178,15 @@ func initLoadTest(t *terraform.Terraform, buildCfg BuildConfig, dumpFilename str
 		return err
 	}
 
-	agentClient, err := extAgent.NewClient(tfOutput.Agents[0].PublicIP)
+	agentClient, err := extAgent.NewClient(tfOutput.Agents[0].PrivateIP)
 	if err != nil {
 		return fmt.Errorf("error in getting ssh connection %w", err)
 	}
 	defer agentClient.Close()
 
 	appClients := make([]*ssh.Client, len(tfOutput.Instances))
 	for i, instance := range tfOutput.Instances {
-		client, err := extAgent.NewClient(instance.PublicIP)
+		client, err := extAgent.NewClient(instance.PrivateIP)
 		if err != nil {
 			return fmt.Errorf("error in getting ssh connection %w", err)
 		}

comparison/output.go

@@ -108,7 +108,7 @@ func (c *Comparison) getResults(resultsCh <-chan Result) []Result {
 				return
 			}
 
-			promURL := "http://" + output.MetricsServer.PublicIP + ":9090"
+			promURL := "http://" + output.MetricsServer.PrivateIP + ":9090"
 			helper, err := prometheus.NewHelper(promURL)
 			if err != nil {
 				mlog.Error("Failed to create prometheus.Helper", mlog.String("deployment ID", res.deploymentID), mlog.Err(err))
@@ -158,7 +158,7 @@ func (c *Comparison) getResults(resultsCh <-chan Result) []Result {
 					mlog.Error("Failed to upload dashboard", mlog.String("deployment ID", res.deploymentID), mlog.Err(err))
 					return
 				}
-				res.DashboardURL = fmt.Sprintf("http://%s:3000%s", output.MetricsServer.PublicIP, url)
+				res.DashboardURL = fmt.Sprintf("http://%s:3000%s", output.MetricsServer.PrivateIP, url)
 			}
 
 			resCh <- res

config/deployer.sample.json

@@ -25,7 +25,6 @@
   "ElasticSearchSettings": {
     "InstanceCount": 0,
     "InstanceType": "r6g.large.search",
-    "VpcID": "",
     "Version": "Elasticsearch_7.10",
     "CreateRole": false,
     "SnapshotRepository": "",
@@ -62,7 +61,8 @@
     "Password": "mostest80098bigpass_",
     "EnablePerformanceInsights": true,
     "DBParameters": [],
-    "ClusterIdentifier": ""
+    "ClusterIdentifier": "",
+    "EngineVersion": ""
   },
   "ExternalDBSettings": {
     "DriverName": "cockroach",

config/deployer.sample.toml

@@ -60,7 +60,6 @@ CreateRole = false
 InstanceCount = 0
 InstanceType = 'r6g.large.search'
 Version = 'Elasticsearch_7.10'
-VpcID = ''
 ZoneAwarenessEnabled = false
 ZoneAwarenessAZCount = 2
 

defaults/set.go

@@ -50,7 +50,7 @@ func structDefaults(value interface{}) error {
 				return fmt.Errorf("invalid size definition: %q", tag)
 			}
 			dv := field.Interface()
-			newSlice, err := createSlice(dv, size)
+			newSlice, err := createSlice(field.Type(), dv, size)
 			if err != nil {
 				return err
 			}
@@ -129,7 +129,7 @@ func setValue(t reflect.Type, data string) (reflect.Value, error) {
 }
 
 // this function creates a slice for the given slice type
-func createSlice(defaultValue interface{}, size int) (reflect.Value, error) {
+func createSlice(fieldType reflect.Type, defaultValue interface{}, size int) (reflect.Value, error) {
 	t := reflect.ValueOf(defaultValue).Type().Elem()
 	if t.Kind() == reflect.Struct {
 		values := reflect.Zero(reflect.SliceOf(t))
@@ -143,7 +143,7 @@ func createSlice(defaultValue interface{}, size int) (reflect.Value, error) {
 		}
 		return values, nil
 	}
-	return reflect.MakeSlice(t, size, size), nil
+	return reflect.MakeSlice(fieldType, size, size), nil
 }
 
 // this function creates a map for the given map type

defaults/set_test.go

@@ -91,4 +91,25 @@ func TestSet(t *testing.T) {
 		assert.NotNil(t, cfg.Chan)
 		assert.Equal(t, 3, cap(cfg.Chan))
 	})
+
+	t.Run("should be able to set nil slices", func(t *testing.T) {
+		cfg := struct {
+			Strings []string
+		}{}
+
+		err := Set(&cfg)
+		require.NoError(t, err)
+		assert.Nil(t, cfg.Strings)
+	})
+
+	t.Run("should be able to set empty slices", func(t *testing.T) {
+		cfg := struct {
+			Strings []string `default_size:"0"`
+		}{}
+
+		err := Set(&cfg)
+		require.NoError(t, err)
+		assert.NotNil(t, cfg.Strings)
+		assert.Len(t, cfg.Strings, 0)
+	})
 }

deployment/config.go

@@ -35,7 +35,7 @@ type Config struct {
 	ClusterName string `default:"loadtest" validate:"alpha"`
 	// ClusterVpcID is the id of the VPC associated to the resources.
 	ClusterVpcID string
-	// ClusterSubnetIDs is the ids of the subnets associated to each resource type.
+	// ClusterSubnetIDs is the ids of the subnets associated to each resource type.
 	ClusterSubnetIDs ClusterSubnetIDs
 	// Number of application instances.
 	AppInstanceCount int `default:"1" validate:"range:[0,)"`
@@ -144,23 +144,40 @@ func (t TerraformMap) String() string {
 
 // ClusterSubnetIDs contains the subnet ids for the different types of instances.
 type ClusterSubnetIDs struct {
-	App           []string `json:"app"`
-	Job           []string `json:"job"`
-	Proxy         []string `json:"proxy"`
-	Agent         []string `json:"agent"`
-	ElasticSearch []string `json:"elasticsearch"`
-	Metrics       []string `json:"metrics"`
-	Keycloak      []string `json:"keycloak"`
-	Database      []string `json:"database"`
-	Redis         []string `json:"redis"`
+	App           []string `default_size:"0" json:"app"`
+	Job           []string `default_size:"0" json:"job"`
+	Proxy         []string `default_size:"0" json:"proxy"`
+	Agent         []string `default_size:"0" json:"agent"`
+	ElasticSearch []string `default_size:"0" json:"elasticsearch"`
+	Metrics       []string `default_size:"0" json:"metrics"`
+	Keycloak      []string `default_size:"0" json:"keycloak"`
+	Database      []string `default_size:"0" json:"database"`
+	Redis         []string `default_size:"0" json:"redis"`
 }
 
 // IsAnySet returns true if any of the subnet ids are set.
 func (c *ClusterSubnetIDs) IsAnySet() bool {
-	return !reflect.DeepEqual(c, &ClusterSubnetIDs{})
+	value := reflect.ValueOf(*c)
+
+	for i := 0; i < value.NumField(); i++ {
+		field := value.Field(i)
+		// Skip fields that are not slices
+		if field.Kind() != reflect.Slice {
+			continue
+		}
+
+		if field.IsNil() || value.Field(i).Len() == 0 {
+			continue
+		}
+
+		return true
+	}
+
+	return false
+
 }
 
-func (c *ClusterSubnetIDs) String() string {
+func (c ClusterSubnetIDs) String() string {
 	b, err := json.Marshal(c)
 	if err != nil {
 		mlog.Error("Failed to marshal ClusterSubnetIDs", mlog.Err(err))
@@ -222,6 +239,8 @@ type TerraformDBSettings struct {
 	// If ClusterIdentifier is not empty, DBName should be set to the name of the database in such cluster.
 	// If ClusterIdentifier is empty, the database created will use DBName as its name.
 	DBName string `default:""`
+	// EngineVersion specifies the version of the database engine.
+	EngineVersion string `default:""`
 }
 
 // ExternalDBSettings contains the necessary data
@@ -437,9 +456,9 @@ func (c *Config) validateElasticSearchConfig() error {
 
 	}
 
-	if !strings.HasPrefix(c.ElasticSearchSettings.Version, "OpenSearch") {
-		return fmt.Errorf("Incorrect engine version: %s. Must start with %q", c.ElasticSearchSettings.Version, "OpenSearch")
-	}
+	// if !strings.HasPrefix(c.ElasticSearchSettings.Version, "OpenSearch") {
+	// 	return fmt.Errorf("Incorrect engine version: %s. Must start with %q", c.ElasticSearchSettings.Version, "OpenSearch")
+	// }
 
 	if c.ElasticSearchSettings.SnapshotRepository == "" {
 		return fmt.Errorf("Empty SnapshotRepository. Must supply a value")