Skip to content

Commit

Permalink
feat: add option for an upgrade test using schematics (#913)<br> - Ad…
Browse files Browse the repository at this point in the history 
…ded new upgrade test feature, implemented by a new function `testschematic.RunSchematicUpgradeTest()`<br> - added new options to `testschematic.TestSchematicOptions` to configure and support new upgrade functionality

* fix: schematics test only print consistency log on fail

* refactor: move SkipUpgradeTest function to common

* feat: schematics upgrade test added

* feat: added option for a git token for schematics url tests
  • Loading branch information
@toddgiguere
toddgiguere authored Jan 16, 2025
1 parent 69b2e27 commit a50d580
Show file tree
Hide file tree
Showing 7 changed files with 325 additions and 101 deletions.
8 changes: 4 additions & 4 deletions .secrets.baseline
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"files": "go.sum|package-lock.json|^.secrets.baseline$",
"lines": null
},
"generated_at": "2024-11-13T21:58:34Z",
"generated_at": "2025-01-15T21:34:03Z",
"plugins_used": [
{
"name": "AWSKeyDetector"
Expand Down Expand Up @@ -164,7 +164,7 @@
"hashed_secret": "bff8f8143a073833d713e3c1821fe97661bc3cef",
"is_secret": false,
"is_verified": false,
"line_number": 315,
"line_number": 318,
"type": "Secret Keyword",
"verified_result": null
}
Expand Down Expand Up @@ -216,15 +216,15 @@
"hashed_secret": "b4e929aa58c928e3e44d12e6f873f39cd8207a25",
"is_secret": false,
"is_verified": false,
"line_number": 520,
"line_number": 481,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "16282376ddaaaf2bf60be9041a7504280f3f338b",
"is_secret": false,
"is_verified": false,
"line_number": 533,
"line_number": 494,
"type": "Secret Keyword",
"verified_result": null
}
Expand Down
42 changes: 42 additions & 0 deletions common/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,13 @@ import (
"os/exec"
"regexp"
"strings"
"testing"

"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/gruntwork-io/terratest/modules/logger"
"github.com/gruntwork-io/terratest/modules/random"
"golang.org/x/crypto/ssh"
)

Expand Down Expand Up @@ -385,3 +388,42 @@ func RetrievePrivateKey(sshPvtKey string) (interface{}, error) {
}
return key, err
}

// SkipUpgradeTest can determine if a terraform or schematics upgrade test should be skipped by analyzing
// the currently checked out git branch, looking for specific verbage in the commit messages.
func SkipUpgradeTest(testing *testing.T, source_repo string, source_branch string, branch string) bool {

// random string to use in remote name
remote := fmt.Sprintf("upstream-%s", strings.ToLower(random.UniqueId()))
logger.Log(testing, "Remote name:", remote)
// Set upstream to the source repo
remote_out, remote_err := exec.Command("/bin/sh", "-c", fmt.Sprintf("git remote add %s %s", remote, source_repo)).Output()
if remote_err != nil {
logger.Log(testing, "Add remote output:\n", remote_out)
logger.Log(testing, "Error adding upstream remote:\n", remote_err)
return false
}
// Fetch the source repo
fetch_out, fetch_err := exec.Command("/bin/sh", "-c", fmt.Sprintf("git fetch %s -f", remote)).Output()
if fetch_err != nil {
logger.Log(testing, "Fetch output:\n", fetch_out)
logger.Log(testing, "Error fetching upstream:\n", fetch_err)
return false
} else {
logger.Log(testing, "Fetch output:\n", fetch_out)
}
// Get all the commit messages from the PR branch
// NOTE: using the "origin" of the default branch as the start point, which will exist in a fresh
// clone even if the default branch has not been checked out or pulled.
cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("git log %s/%s..%s", remote, source_branch, branch))
out, _ := cmd.CombinedOutput()

fmt.Printf("Commit Messages (%s): \n%s", source_branch, string(out))
// Skip upgrade Test if BREAKING CHANGE OR SKIP UPGRADE TEST string found in commit messages
doNotRunUpgradeTest := false
if (strings.Contains(string(out), "BREAKING CHANGE") || strings.Contains(string(out), "SKIP UPGRADE TEST")) && !strings.Contains(string(out), "UNSKIP UPGRADE TEST") {
doNotRunUpgradeTest = true
}

return doNotRunUpgradeTest
}
41 changes: 1 addition & 40 deletions testhelper/tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,12 @@ package testhelper
import (
"fmt"
"os"
"os/exec"
"path"
"strings"
"testing"
"time"

"github.com/IBM/platform-services-go-sdk/resourcecontrollerv2"
"github.com/gruntwork-io/terratest/modules/random"

"github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/cloudinfo"

Expand All @@ -25,43 +23,6 @@ import (
"github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/common"
)

func (options *TestOptions) skipUpgradeTest(source_repo string, source_branch string, branch string) bool {

// random string to use in remote name
remote := fmt.Sprintf("upstream-%s", strings.ToLower(random.UniqueId()))
logger.Log(options.Testing, "Remote name:", remote)
// Set upstream to the source repo
remote_out, remote_err := exec.Command("/bin/sh", "-c", fmt.Sprintf("git remote add %s %s", remote, source_repo)).Output()
if remote_err != nil {
logger.Log(options.Testing, "Add remote output:\n", remote_out)
logger.Log(options.Testing, "Error adding upstream remote:\n", remote_err)
return false
}
// Fetch the source repo
fetch_out, fetch_err := exec.Command("/bin/sh", "-c", fmt.Sprintf("git fetch %s -f", remote)).Output()
if fetch_err != nil {
logger.Log(options.Testing, "Fetch output:\n", fetch_out)
logger.Log(options.Testing, "Error fetching upstream:\n", fetch_err)
return false
} else {
logger.Log(options.Testing, "Fetch output:\n", fetch_out)
}
// Get all the commit messages from the PR branch
// NOTE: using the "origin" of the default branch as the start point, which will exist in a fresh
// clone even if the default branch has not been checked out or pulled.
cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("git log %s/%s..%s", remote, source_branch, branch))
out, _ := cmd.CombinedOutput()

fmt.Printf("Commit Messages (%s): \n%s", source_branch, string(out))
// Skip upgrade Test if BREAKING CHANGE OR SKIP UPGRADE TEST string found in commit messages
doNotRunUpgradeTest := false
if (strings.Contains(string(out), "BREAKING CHANGE") || strings.Contains(string(out), "SKIP UPGRADE TEST")) && !strings.Contains(string(out), "UNSKIP UPGRADE TEST") {
doNotRunUpgradeTest = true
}

return doNotRunUpgradeTest
}

// Function to setup testing environment.
//
// Summary of settings:
Expand Down Expand Up @@ -385,7 +346,7 @@ func (options *TestOptions) RunTestUpgrade() (*terraform.PlanStruct, error) {
logger.Log(options.Testing, "Base Branch:", baseBranch)
}

if options.skipUpgradeTest(baseRepo, baseBranch, prBranch) {
if common.SkipUpgradeTest(options.Testing, baseRepo, baseBranch, prBranch) {
options.Testing.Log("Detected the string \"BREAKING CHANGE\" or \"SKIP UPGRADE TEST\" used in commit message, skipping upgrade Test.")
} else {
skipped = false
Expand Down
17 changes: 17 additions & 0 deletions testschematic/mock_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type schematicServiceMock struct {
mock.Mock
activities []schematics.WorkspaceActivity
failCreateWorkspace bool
failReplaceWorkspace bool
failDeleteWorkspace bool
failTemplateRepoUpload bool
failReplaceWorkspaceInputs bool
Expand All @@ -61,6 +62,7 @@ type iamAuthenticatorMock struct {
func mockSchematicServiceReset(mock *schematicServiceMock, options *TestSchematicOptions) {
mock.failCreateWorkspace = false
mock.failDeleteWorkspace = false
mock.failReplaceWorkspace = false
mock.failTemplateRepoUpload = false
mock.failReplaceWorkspaceInputs = false
mock.failListWorkspaceActivities = false
Expand Down Expand Up @@ -108,6 +110,21 @@ func (mock *schematicServiceMock) UpdateWorkspace(updateWorkspaceOptions *schema
return result, response, nil
}

func (mock *schematicServiceMock) ReplaceWorkspace(replaceWorkspaceOptions *schematics.ReplaceWorkspaceOptions) (*schematics.WorkspaceResponse, *core.DetailedResponse, error) {
if mock.failReplaceWorkspace {
return nil, &core.DetailedResponse{StatusCode: 404}, &schematicErrorMock{}
}

result := &schematics.WorkspaceResponse{
ID: core.StringPtr(mockWorkspaceID),
TemplateData: []schematics.TemplateSourceDataResponse{
{ID: core.StringPtr(mockTemplateID)},
},
}
response := &core.DetailedResponse{StatusCode: 200}
return result, response, nil
}

func (mock *schematicServiceMock) DeleteWorkspace(deleteWorkspaceOptions *schematics.DeleteWorkspaceOptions) (*string, *core.DetailedResponse, error) {
if mock.failDeleteWorkspace {
return nil, &core.DetailedResponse{StatusCode: 404}, &schematicErrorMock{}
Expand Down
109 changes: 109 additions & 0 deletions testschematic/schematics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (

// IBM schematics job types
const SchematicsJobTypeUpload = "TAR_WORKSPACE_UPLOAD"
const SchematicsJobTypeUpdate = "WORKSPACE_UPDATE"
const SchematicsJobTypePlan = "PLAN"
const SchematicsJobTypeApply = "APPLY"
const SchematicsJobTypeDestroy = "DESTROY"
Expand Down Expand Up @@ -53,6 +54,7 @@ type SchematicsApiSvcI interface {
PlanWorkspaceCommand(*schematics.PlanWorkspaceCommandOptions) (*schematics.WorkspaceActivityPlanResult, *core.DetailedResponse, error)
ApplyWorkspaceCommand(*schematics.ApplyWorkspaceCommandOptions) (*schematics.WorkspaceActivityApplyResult, *core.DetailedResponse, error)
DestroyWorkspaceCommand(*schematics.DestroyWorkspaceCommandOptions) (*schematics.WorkspaceActivityDestroyResult, *core.DetailedResponse, error)
ReplaceWorkspace(*schematics.ReplaceWorkspaceOptions) (*schematics.WorkspaceResponse, *core.DetailedResponse, error)
}

// interface for external IBMCloud IAM Authenticator api. Can be mocked for tests
Expand All @@ -69,12 +71,17 @@ type SchematicsTestService struct {
ApiAuthenticator IamAuthenticatorSvcI // the authenticator used for schematics api calls
WorkspaceID string // workspace ID used for tests
WorkspaceName string // name of workspace that was created for test
WorkspaceNameForLog string // combination of Name and ID useful for log consistency
WorkspaceLocation string // region the workspace was created in
TemplateID string // workspace template ID used for tests
TestOptions *TestSchematicOptions // additional testing options
TerraformTestStarted bool // keeps track of when actual Terraform resource testing has begin, used for proper test teardown logic
TerraformResourcesCreated bool // keeps track of when we start deploying resources, used for proper test teardown logic
CloudInfoService cloudinfo.CloudInfoServiceI // reference to a CloudInfoService resource
BaseTerraformRepo string // the URL of the origin git repository, typically the base that the PR will merge into, used for upgrade test
BaseTerraformRepoBranch string // the branch name of the main origin branch of the project (main or master), used for upgrade test
TestTerraformRepo string // the URL of the repo for the pull request, will be either origin or a fork
TestTerraformRepoBranch string // the branch of the test, usually the current checked out branch of the test run
}

// CreateAuthenticator will accept a valid IBM cloud API key, and
Expand Down Expand Up @@ -217,6 +224,108 @@ func (svc *SchematicsTestService) CreateTestWorkspace(name string, resourceGroup
return workspace, nil
}

// CreateUploadTarFile will create a tar file with terraform code, based on include patterns set in options.
// Returns the full tarball name that was created on local system (path included).
func (svc *SchematicsTestService) CreateUploadTarFile(projectPath string) (string, error) {
svc.TestOptions.Testing.Log("[SCHEMATICS] Creating TAR file")
tarballName, tarballErr := CreateSchematicTar(projectPath, &svc.TestOptions.TarIncludePatterns)
if tarballErr != nil {
return "", fmt.Errorf("error creating tar file: %w", tarballErr)
}

svc.TestOptions.Testing.Log("[SCHEMATICS] Uploading TAR file")
uploadErr := svc.UploadTarToWorkspace(tarballName)
if uploadErr != nil {
return tarballName, fmt.Errorf("error uploading tar file to workspace: %w - %s", uploadErr, svc.WorkspaceNameForLog)
}

// -------- UPLOAD TAR FILE ----------
// find the tar upload job
uploadJob, uploadJobErr := svc.FindLatestWorkspaceJobByName(SchematicsJobTypeUpload)
if uploadJobErr != nil {
return tarballName, fmt.Errorf("error finding the upload tar action: %w - %s", uploadJobErr, svc.WorkspaceNameForLog)
}
// wait for it to finish
uploadJobStatus, uploadJobStatusErr := svc.WaitForFinalJobStatus(*uploadJob.ActionID)
if uploadJobStatusErr != nil {
return tarballName, fmt.Errorf("error waiting for upload of tar to finish: %w - %s", uploadJobStatusErr, svc.WorkspaceNameForLog)
}
// check if complete
if uploadJobStatus != SchematicsJobStatusCompleted {
return tarballName, fmt.Errorf("tar upload has failed with status %s - %s", uploadJobStatus, svc.WorkspaceNameForLog)
}

return tarballName, nil
}

// SetTemplateRepoToBranch will call UpdateTestTemplateRepo with the appropriate URLs set at the service level
// for the branch of this test
func (svc *SchematicsTestService) SetTemplateRepoToBranch() error {
return svc.UpdateTestTemplateRepo(svc.TestTerraformRepo, svc.TestTerraformRepoBranch, svc.TestOptions.TemplateGitToken)
}

// SetTemplateRepoToBase will call UpdateTestTemplateRepo with the appropriate URLs set at the service level
// for the base (master or main) branch of this test, used in upgrade tests
func (svc *SchematicsTestService) SetTemplateRepoToBase() error {
return svc.UpdateTestTemplateRepo(svc.BaseTerraformRepo, svc.BaseTerraformRepoBranch, svc.TestOptions.TemplateGitToken)
}

// UpdateTestTemplateRepo will replace the workspace repository with a given github URL and branch
func (svc *SchematicsTestService) UpdateTestTemplateRepo(url string, branch string, token string) error {
svc.TestOptions.Testing.Logf("[SCHEMATICS] Setting template repository to repo %s (%s)", url, branch)
// set up repo update request
repoUpdateRequest := &schematics.TemplateRepoUpdateRequest{
URL: core.StringPtr(url),
Branch: core.StringPtr(branch),
}

// set up overall workspace options
replaceOptions := &schematics.ReplaceWorkspaceOptions{
WID: core.StringPtr(svc.WorkspaceID),
TemplateRepo: repoUpdateRequest,
}

// if supplied set a token for private repo
if len(token) > 0 {
replaceOptions.SetXGithubToken(token)
}

// now update template
var resp *core.DetailedResponse
var updateErr error
retries := 0
for {
_, resp, updateErr = svc.SchematicsApiSvc.ReplaceWorkspace(replaceOptions)
if svc.retryApiCall(updateErr, getDetailedResponseStatusCode(resp), retries) {
retries++
svc.TestOptions.Testing.Logf("[SCHEMATICS] RETRY ReplaceWorkspace, status code: %d", getDetailedResponseStatusCode(resp))
} else {
break
}
}
if updateErr != nil {
return updateErr
}

// -------- SETTING UP WORKSPACE WITH REPO ----------
// find the repository refresh job
replaceJob, replaceJobErr := svc.FindLatestWorkspaceJobByName(SchematicsJobTypeUpdate)
if replaceJobErr != nil {
return fmt.Errorf("error finding the update workspace action: %w - %s", replaceJobErr, svc.WorkspaceNameForLog)
}
// wait for it to finish
replaceJobStatus, replaceJobStatusErr := svc.WaitForFinalJobStatus(*replaceJob.ActionID)
if replaceJobStatusErr != nil {
return fmt.Errorf("error waiting for update of workspace to finish: %w - %s", replaceJobStatusErr, svc.WorkspaceNameForLog)
}
// check if complete
if replaceJobStatus != SchematicsJobStatusCompleted {
return fmt.Errorf("workspace update has failed with status %s - %s", replaceJobStatus, svc.WorkspaceNameForLog)
}

return nil
}

// UpdateTestTemplateVars will update an existing Schematics Workspace terraform template with a
// Variablestore, which will set terraform input variables for test runs.
func (svc *SchematicsTestService) UpdateTestTemplateVars(vars []TestSchematicTerraformVar) error {
Expand Down
28 changes: 27 additions & 1 deletion testschematic/test_options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ type TestSchematicOptions struct {
// to that value.
TemplateFolder string

// The Git auth token used by schematics to clone initial template source of project.
// If TAR upload option is not used, the default will be to use the Git URL of the test branch for initial schematics workload setup.
// If the test repo is private or protected, set this value to a Git token that is authorized to read/clone the repository.
// NOTE: you may also need to set the same token in a `NetrcSettings` as well for this scenario.
TemplateGitToken string

// Optional list of tags that will be applied to the Schematics Workspace instance
Tags []string

Expand Down Expand Up @@ -130,6 +136,21 @@ type TestSchematicOptions struct {
IgnoreDestroys testhelper.Exemptions
IgnoreUpdates testhelper.Exemptions

// Use these options to specify a base terraform repo and branch to use for upgrade tests.
// If not supplied, the default logic will be used to determine the base repo and branch.
// Will be overridden by environment variables BASE_TERRAFORM_REPO and BASE_TERRAFORM_BRANCH if set.
//
// For repositories that require authentication:
// - For HTTPS repositories, set the GIT_TOKEN environment variable to your Personal Access Token (PAT).
// - For SSH repositories, set the SSH_PRIVATE_KEY environment variable to your SSH private key value.
// If the SSH_PRIVATE_KEY environment variable is not set, the default SSH key located at ~/.ssh/id_rsa will be used.
// Ensure that the appropriate public key is added to the repository's list of authorized keys.
//
// BaseTerraformRepo: The URL of the base Terraform repository.
BaseTerraformRepo string
// BaseTerraformBranch: The branch within the base Terraform repository to use for upgrade tests.
BaseTerraformBranch string

// These optional fields can be used to override the default retry settings for making Schematics API calls.
// If SDK/API calls to Schematics result in errors, such as retrieving existing workspace details,
// the test framework will retry those calls for a set number of times, with a wait time between calls.
Expand All @@ -143,6 +164,11 @@ type TestSchematicOptions struct {
// By default the logs from schematics jobs will only be printed to the test log if there is a failure in the job.
// Set this value to `true` to have all schematics job logs (plan/apply/destroy) printed to the test log.
PrintAllSchematicsLogs bool

// This property will be set to true by the test when an upgrade test was performed.
// You can then inspect this value after the test run, if needed, to make further code decisions.
// NOTE: this is not an option field that is meant to be set from a unit test, it is informational only
IsUpgradeTest bool
}

type TestSchematicTerraformVar struct {
Expand Down Expand Up @@ -172,7 +198,7 @@ func (options *TestSchematicOptions) GetCheckConsistencyOptions() *testhelper.Ch
IgnoreAdds: options.IgnoreAdds,
IgnoreDestroys: options.IgnoreDestroys,
IgnoreUpdates: options.IgnoreUpdates,
IsUpgradeTest: false,
IsUpgradeTest: options.IsUpgradeTest,
}
}

Expand Down
Loading

0 comments on commit a50d580

Please sign in to comment.