feat: add schematics consistency test features

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "go.sum|package-lock.json|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-10-11T12:56:06Z",
+  "generated_at": "2024-11-13T21:58:34Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -100,15 +100,15 @@
         "hashed_secret": "5996c731c43c6191af2324af0230bd65e723fcdb",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 357,
+        "line_number": 358,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "03e60e3e0d9675b19754e2a81bbb48a26af858e7",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 825,
+        "line_number": 826,
         "type": "Secret Keyword",
         "verified_result": null
       }
@@ -128,23 +128,23 @@
         "hashed_secret": "892bd503fb45f6fcafb1c7003d88291fc0b20208",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 270,
+        "line_number": 284,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "5da5a31d49370df43eff521b39c10db1466fae44",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 273,
+        "line_number": 287,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "d4c3d66fd0c38547a3c7a4c6bdc29c36911bc030",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 463,
+        "line_number": 488,
         "type": "Secret Keyword",
         "verified_result": null
       }
@@ -216,15 +216,15 @@
         "hashed_secret": "b4e929aa58c928e3e44d12e6f873f39cd8207a25",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 666,
+        "line_number": 520,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "16282376ddaaaf2bf60be9041a7504280f3f338b",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 679,
+        "line_number": 533,
         "type": "Secret Keyword",
         "verified_result": null
       }

cloudinfo/projects.go

@@ -3,8 +3,6 @@ package cloudinfo
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/IBM/go-sdk-core/v5/core"
-	project "github.com/IBM/project-go-sdk/projectv1"
 	"log"
 	"math/rand"
 	"os"
@@ -13,6 +11,9 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"github.com/IBM/go-sdk-core/v5/core"
+	project "github.com/IBM/project-go-sdk/projectv1"
 )
 
 // CreateProjectFromConfig creates a project with the given config
@@ -900,9 +901,13 @@ func (infoSvc *CloudInfoService) LookupMemberNameByID(stackDetails *project.Proj
 }
 
 // GetSchematicsJobLogsForMember gets the schematics job logs for a member
-func (infoSvc *CloudInfoService) GetSchematicsJobLogsForMember(member *project.ProjectConfig, memberName string) (details string, terraformLogs string) {
+func (infoSvc *CloudInfoService) GetSchematicsJobLogsForMember(member *project.ProjectConfig, memberName string, projectRegion string) (details string, terraformLogs string) {
 	var logMessage strings.Builder
 	var terraformLogMessage strings.Builder
+
+	// determine schematics geo location from project region
+	schematicsLocation := projectRegion[0:2]
+
 	logMessage.WriteString(fmt.Sprintf("Schematics job logs for member: %s", memberName))
 
 	if member.Schematics != nil && member.Schematics.WorkspaceCrn != nil {
@@ -935,7 +940,7 @@ func (infoSvc *CloudInfoService) GetSchematicsJobLogsForMember(member *project.P
 					jobURL := strings.Split(url, "/jobs?region=")[0]
 					jobURL = fmt.Sprintf("%s/log/%s", jobURL, jobID)
 					logMessage.WriteString(fmt.Sprintf("\nSchematics Job URL: %s", jobURL))
-					logs, errGetLogs := infoSvc.GetSchematicsJobLogsText(jobID)
+					logs, errGetLogs := infoSvc.GetSchematicsJobLogsText(jobID, schematicsLocation)
 					if errGetLogs != nil {
 						terraformLogMessage.WriteString(fmt.Sprintf("\nError getting job logs for Job ID: %s member: %s, error: %s", jobID, memberName, errGetLogs))
 					} else {
@@ -988,7 +993,7 @@ func (infoSvc *CloudInfoService) GetSchematicsJobLogsForMember(member *project.P
 				jobURL := strings.Split(url, "/jobs?region=")[0]
 				jobURL = fmt.Sprintf("%s/log/%s", jobURL, jobID)
 				logMessage.WriteString(fmt.Sprintf("\nSchematics Job URL: %s", jobURL))
-				logs, errGetLogs := infoSvc.GetSchematicsJobLogsText(jobID)
+				logs, errGetLogs := infoSvc.GetSchematicsJobLogsText(jobID, schematicsLocation)
 				if errGetLogs != nil {
 					terraformLogMessage.WriteString(fmt.Sprintf("\nError getting job logs for Job ID: %s member: %s, error: %s", jobID, memberName, errGetLogs))
 				} else {
@@ -1040,7 +1045,7 @@ func (infoSvc *CloudInfoService) GetSchematicsJobLogsForMember(member *project.P
 				jobURL := strings.Split(url, "/jobs?region=")[0]
 				jobURL = fmt.Sprintf("%s/log/%s", jobURL, jobID)
 				logMessage.WriteString(fmt.Sprintf("\nSchematics Job URL: %s", jobURL))
-				logs, errGetLogs := infoSvc.GetSchematicsJobLogsText(jobID)
+				logs, errGetLogs := infoSvc.GetSchematicsJobLogsText(jobID, schematicsLocation)
 				if errGetLogs != nil {
 					terraformLogMessage.WriteString(fmt.Sprintf("\nError getting job logs for Job ID: %s member: %s, error: %s", jobID, memberName, errGetLogs))
 				} else {

cloudinfo/schematics.go

@@ -2,16 +2,29 @@ package cloudinfo
 
 import (
 	"fmt"
+	"math/rand"
+	"net/url"
+	"strings"
+
 	"github.com/IBM/go-sdk-core/v5/core"
 	schematics "github.com/IBM/schematics-go-sdk/schematicsv1"
-	"io"
-	"net/http"
-	"strings"
-	"time"
+	"github.com/IBM/vpc-go-sdk/common"
 )
 
-func (infoSvc *CloudInfoService) GetSchematicsJobLogs(jobID string) (result *schematics.JobLog, response *core.DetailedResponse, err error) {
-	return infoSvc.schematicsService.ListJobLogs(
+// will return a previously configured schematics service based on location
+// error returned if location not initialized
+// location must be a valid geographical location supported by schematics: "us" or "eu"
+func (infoSvc *CloudInfoService) GetSchematicsServiceByLocation(location string) (schematicsService, error) {
+	service, isFound := infoSvc.schematicsServices[location]
+	if !isFound {
+		return nil, fmt.Errorf("could not find Schematics Service for location %s", location)
+	}
+
+	return service, nil
+}
+
+func (infoSvc *CloudInfoService) GetSchematicsJobLogs(jobID string, location string) (result *schematics.JobLog, response *core.DetailedResponse, err error) {
+	return infoSvc.schematicsServices[location].ListJobLogs(
 		&schematics.ListJobLogsOptions{
 			JobID: core.StringPtr(jobID),
 		},
@@ -21,56 +34,120 @@ func (infoSvc *CloudInfoService) GetSchematicsJobLogs(jobID string) (result *sch
 // GetSchematicsJobLogsText retrieves the logs of a Schematics job as a string
 // The logs are returned as a string, or an error if the operation failed
 // This is a temporary workaround until the Schematics GO SDK is fixed, ListJobLogs is broken as the response is text/plain and not application/json
-func (infoSvc *CloudInfoService) GetSchematicsJobLogsText(jobID string) (logs string, err error) {
-	const maxRetries = 3
-	const retryDelay = 2 * time.Second
-
-	url := fmt.Sprintf("https://schematics.cloud.ibm.com/v2/jobs/%s/logs", jobID)
-	var retryErrors []string
-
-	for attempt := 1; attempt <= maxRetries; attempt++ {
-		// Create the request
-		req, err := http.NewRequest("GET", url, nil)
-		if err != nil {
-			return "", fmt.Errorf("failed to create request: %v", err)
-		}
-
-		// Authenticate the request
-		err = infoSvc.authenticator.Authenticate(req)
-		if err != nil {
-			return "", fmt.Errorf("failed to authenticate: %v", err)
-		}
-
-		// Make the request
-		client := &http.Client{}
-		resp, err := client.Do(req)
-		if err != nil {
-			retryErrors = append(retryErrors, fmt.Sprintf("attempt %d: failed to make request: %v", attempt, err))
-			if attempt < maxRetries {
-				time.Sleep(retryDelay)
-				continue
-			}
-			return "", fmt.Errorf("exceeded maximum retries, attempt failures:\n%s", strings.Join(retryErrors, "\n"))
-		}
-		defer resp.Body.Close()
-
-		// Check if the response status is successful
-		if resp.StatusCode >= 200 && resp.StatusCode < 300 {
-			// Read the response body
-			body, err := io.ReadAll(resp.Body)
-			if err != nil {
-				return "", fmt.Errorf("failed to read response body: %v", err)
-			}
-			return string(body), nil
-		} else {
-			retryErrors = append(retryErrors, fmt.Sprintf("attempt %d: request failed with status code: %d", attempt, resp.StatusCode))
-			if attempt < maxRetries {
-				time.Sleep(retryDelay)
-				continue
-			}
-			return "", fmt.Errorf("exceeded maximum retries, attempt failures:\n%s", strings.Join(retryErrors, "\n"))
-		}
+// location must be a valid geographical location supported by schematics: "us" or "eu"
+func (infoSvc *CloudInfoService) GetSchematicsJobLogsText(jobID string, location string) (string, error) {
+
+	svc, svcErr := infoSvc.GetSchematicsServiceByLocation(location)
+	if svcErr != nil {
+		return "", fmt.Errorf("error getting schematics service for location %s:%w", location, svcErr)
+	}
+
+	// build up a REST API call for job logs
+	pathParamsMap := map[string]string{
+		"job_id": jobID,
+	}
+	builder := core.NewRequestBuilder(core.GET)
+	builder.EnableGzipCompression = svc.GetEnableGzipCompression()
+	_, builderErr := builder.ResolveRequestURL(svc.GetServiceURL(), `/v2/jobs/{job_id}/logs`, pathParamsMap)
+	if builderErr != nil {
+		return "", builderErr
 	}
+	sdkHeaders := common.GetSdkHeaders("schematics", "V1", "ListJobLogs")
+	for headerName, headerValue := range sdkHeaders {
+		builder.AddHeader(headerName, headerValue)
+	}
+	builder.AddHeader("Accept", "application/json")
+
+	request, buildErr := builder.Build()
+	if buildErr != nil {
+		return "", buildErr
+	}
+
+	// initialize the IBM Core HTTP service
+	baseService, baseSvcErr := core.NewBaseService(&core.ServiceOptions{
+		URL:           svc.GetServiceURL(),
+		Authenticator: infoSvc.authenticator,
+	})
+	if baseSvcErr != nil {
+		return "", baseSvcErr
+	}
+
+	// make the builder request call on the core http service, which is text/plain
+	// using response type "**string" to get raw text output
+	rawResponse := core.StringPtr("")
+	_, requestErr := baseService.Request(request, &rawResponse)
+	if requestErr != nil {
+		return "", requestErr
+	}
+
+	return *rawResponse, nil
+}
+
+// GetSchematicsJobFileData will download a specific job file and return a JobFileData structure.
+// Allowable values for fileType: state_file, plan_json
+// location must be a valid geographical location supported by schematics: "us" or "eu"
+func (infoSvc *CloudInfoService) GetSchematicsJobFileData(jobID string, fileType string, location string) (*schematics.JobFileData, error) {
+	// setup options
+	// file type Allowable values: [template_repo,readme_file,log_file,state_file,plan_json]
+	jobFileOptions := &schematics.GetJobFilesOptions{
+		JobID:    core.StringPtr(jobID),
+		FileType: core.StringPtr(fileType),
+	}
+
+	// get a service based on location
+	svc, svcErr := infoSvc.GetSchematicsServiceByLocation(location)
+	if svcErr != nil {
+		return nil, fmt.Errorf("error getting schematics service for location %s:%w", location, svcErr)
+	}
+
+	data, _, err := svc.GetJobFiles(jobFileOptions)
+
+	return data, err
+}
+
+// Returns a string of the unmarshalled `Terraform Plan JSON` produced by a schematics job
+// location must be a valid geographical location supported by schematics: "us" or "eu"
+func (infoSvc *CloudInfoService) GetSchematicsJobPlanJson(jobID string, location string) (string, error) {
+	// get the plan_json file for the job
+	data, dataErr := infoSvc.GetSchematicsJobFileData(jobID, "plan_json", location)
+
+	// check for multiple error conditions
+	if dataErr != nil {
+		return "", dataErr
+	}
+	if data == nil {
+		return "", fmt.Errorf("job file data object is nil, which is unexpected")
+	}
+	if data.FileContent == nil {
+		return "", fmt.Errorf("file content is nil, which is unexpected")
+	}
+
+	// extract the plan file content and return
+	contentPtr := data.FileContent
+
+	return *contentPtr, nil
+}
+
+// returns a random selected region that is valid for Schematics Workspace creation
+func GetRandomSchematicsLocation() string {
+	validLocations := GetSchematicsLocations()
+	randomIndex := rand.Intn(len(validLocations))
+	return validLocations[randomIndex]
+}
+
+// returns the appropriate schematics API endpoint based on specific region
+// the region can be geographic (us or eu) or specific (us-south)
+func GetSchematicServiceURLForRegion(region string) (string, error) {
+	// the service URLs are simply the region in front of base default
+
+	// first, get the default URL from official service
+	url, parseErr := url.Parse(schematics.DefaultServiceURL)
+	if parseErr != nil {
+		return "", fmt.Errorf("error parsing default schematics URL: %w", parseErr)
+	}
+
+	// prefix the region in front of existing host
+	url.Host = strings.ToLower(region) + "." + url.Host
 
-	return "", fmt.Errorf("exceeded maximum retries, attempt failures:\n%s", strings.Join(retryErrors, "\n"))
+	return url.String(), nil
 }