util: unify metadata code (#890)

Remove duplicate methods and stop assuming instance attributes for all
calls, let the callers decide the metadata root level.

.gitignore

@@ -0,0 +1,2 @@
+imagetest/cmd/manager/manager
+imagetest/cmd/wrapper/wrapper

imagetest/build.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -x
+
+build_for() {
+  GOOS=$2
+  go build -C $1 ../... || { exit 1; }
+  go test -C $1 -c -o /dev/null -tags cit || { exit 1; }
+}
+
+build() {
+  build_for $1 linux
+  build_for $1 windows
+}
+
+if golint &> /dev/null; then
+  golint -set_exit_status ./...
+fi
+
+build cmd/manager/
+build cmd/wrapper/
+
+for suite in `find test_suites/* -maxdepth 0 -type d`; do
+  build $suite
+done

imagetest/cmd/wrapper/main.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"net/url"
 	"os/exec"
 	"strings"
@@ -19,9 +18,10 @@ import (
 
 // In special cases such as the shutdown script, the guest attribute match
 // on the first boot must have a different name than the usual guest attribute.
-func checkFirstBootSpecialGA() bool {
-	if _, err := utils.GetMetadataAttribute("shouldRebootDuringTest"); err == nil {
-		_, foundFirstBootGA := utils.GetMetadataGuestAttribute(utils.GuestAttributeTestNamespace + "/" + utils.FirstBootGAKey)
+func checkFirstBootSpecialGA(ctx context.Context) bool {
+	if _, err := utils.GetMetadata(ctx, "instance", "attributes", "shouldRebootDuringTest"); err == nil {
+		_, foundFirstBootGA := utils.GetMetadata(ctx, "instance", "guest-attributes",
+			utils.GuestAttributeTestNamespace, utils.FirstBootGAKey)
 		// if the special attribute to match the first boot of the shutdown script test is already set, foundFirstBootGA will be nil and we should use the regular guest attribute.
 		if foundFirstBootGA != nil {
 			return true
@@ -37,14 +37,16 @@ func main() {
 		log.Fatalf("failed to create cloud storage client: %v", err)
 	}
 	log.Printf("FINISHED-BOOTING")
-	firstBootSpecialAttribute := checkFirstBootSpecialGA()
+	firstBootSpecialAttribute := checkFirstBootSpecialGA(ctx)
 	// firstBootSpecialGA should be true if we need to match a different guest attribute than the usual guest attribute
 	defer func(ctx context.Context, firstBootSpecialGA bool) {
 		var err error
 		if firstBootSpecialGA {
-			err = utils.QueryMetadataGuestAttribute(ctx, utils.GuestAttributeTestNamespace, utils.FirstBootGAKey, http.MethodPut)
+			err = utils.PutMetadata(ctx, "instance", "guest-attributes", utils.GuestAttributeTestNamespace,
+				utils.FirstBootGAKey)
 		} else {
-			err = utils.QueryMetadataGuestAttribute(ctx, utils.GuestAttributeTestNamespace, utils.GuestAttributeTestKey, http.MethodPut)
+			err = utils.PutMetadata(ctx, "instance", "guest-attributes", utils.GuestAttributeTestNamespace,
+				utils.GuestAttributeTestKey)
 		}
 
 		if err != nil {
@@ -56,30 +58,30 @@ func main() {
 		}
 	}(ctx, firstBootSpecialAttribute)
 
-	daisyOutsPath, err := utils.GetMetadataAttribute("daisy-outs-path")
+	daisyOutsPath, err := utils.GetMetadata(ctx, "instance", "attributes", "daisy-outs-path")
 	if err != nil {
 		log.Fatalf("failed to get metadata daisy-outs-path: %v", err)
 	}
 	daisyOutsPath = daisyOutsPath + "/"
 
-	testPackageURL, err := utils.GetMetadataAttribute("_test_package_url")
+	testPackageURL, err := utils.GetMetadata(ctx, "instance", "attributes", "_test_package_url")
 	if err != nil {
 		log.Fatalf("failed to get metadata _test_package_url: %v", err)
 	}
 
-	resultsURL, err := utils.GetMetadataAttribute("_test_results_url")
+	resultsURL, err := utils.GetMetadata(ctx, "instance", "attributes", "_test_results_url")
 	if err != nil {
 		log.Fatalf("failed to get metadata _test_results_url: %v", err)
 	}
 
 	var testArguments = []string{"-test.v"}
 
-	testRun, err := utils.GetMetadataAttribute("_test_run")
+	testRun, err := utils.GetMetadata(ctx, "instance", "attributes", "_test_run")
 	if err == nil && testRun != "" {
 		testArguments = append(testArguments, "-test.run", testRun)
 	}
 
-	testPackage, err := utils.GetMetadataAttribute("_test_package_name")
+	testPackage, err := utils.GetMetadata(ctx, "instance", "attributes", "_test_package_name")
 	if err != nil {
 		log.Fatalf("failed to get metadata _test_package_name: %v", err)
 	}

imagetest/test_suites/disk/disk_resize_test.go

@@ -25,7 +25,7 @@ const (
 func TestDiskResize(t *testing.T) {
 	// TODO: test disk resizing on windows
 	utils.LinuxOnly(t)
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(utils.Context(t), "instance", "image")
 	if err != nil {
 		t.Fatalf("couldn't get image from metadata")
 	}

imagetest/test_suites/hostnamevalidation/hostname_validation_test.go

@@ -51,7 +51,7 @@ func testHostnameLinux(shortname string) error {
 }
 
 func TestHostname(t *testing.T) {
-	metadataHostname, err := utils.GetMetadata("hostname")
+	metadataHostname, err := utils.GetMetadata(utils.Context(t), "instance", "hostname")
 	if err != nil {
 		t.Fatalf(" still couldn't determine metadata hostname")
 	}
@@ -72,7 +72,7 @@ func TestHostname(t *testing.T) {
 
 // TestCustomHostname tests the 'fully qualified domain name'.
 func TestCustomHostname(t *testing.T) {
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(utils.Context(t), "instance", "image")
 	if err != nil {
 		t.Fatalf("Couldn't get image from metadata")
 	}
@@ -93,8 +93,9 @@ func TestCustomHostname(t *testing.T) {
 // TestFQDN tests the 'fully qualified domain name'.
 func TestFQDN(t *testing.T) {
 	utils.LinuxOnly(t)
+	ctx := utils.Context(t)
 	// TODO Zonal DNS is breaking this test case in EL9.
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(ctx, "instance", "image")
 	if err != nil {
 		t.Fatalf("Couldn't get image from metadata")
 	}
@@ -115,7 +116,7 @@ func TestFQDN(t *testing.T) {
 		t.Skip("Broken on EL9")
 	}
 
-	metadataHostname, err := utils.GetMetadata("hostname")
+	metadataHostname, err := utils.GetMetadata(ctx, "instance", "hostname")
 	if err != nil {
 		t.Fatalf("couldn't determine metadata hostname")
 	}
@@ -174,7 +175,7 @@ func TestHostKeysGeneratedOnce(t *testing.T) {
 		hashes = append(hashes, sshKeyHash{file, hash})
 	}
 
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(utils.Context(t), "instance", "image")
 	if err != nil {
 		t.Fatalf("Couldn't get image from metadata")
 	}
@@ -223,7 +224,8 @@ func TestHostKeysGeneratedOnce(t *testing.T) {
 
 func TestHostsFile(t *testing.T) {
 	utils.LinuxOnly(t)
-	image, err := utils.GetMetadata("image")
+	ctx := utils.Context(t)
+	image, err := utils.GetMetadata(ctx, "instance", "image")
 	if err != nil {
 		t.Fatalf("couldn't get image from metadata")
 	}
@@ -260,11 +262,11 @@ func TestHostsFile(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Couldn't read /etc/hosts")
 	}
-	ip, err := utils.GetMetadata("network-interfaces/0/ip")
+	ip, err := utils.GetMetadata(ctx, "instance", "network-interfaces", "0", "ip")
 	if err != nil {
 		t.Fatalf("Couldn't get ip from metadata")
 	}
-	hostname, err := utils.GetMetadata("hostname")
+	hostname, err := utils.GetMetadata(ctx, "instance", "hostname")
 	if err != nil {
 		t.Fatalf("Couldn't get hostname from metadata")
 	}

imagetest/test_suites/imageboot/image_boot_test.go

@@ -99,7 +99,7 @@ func lookForGuestAgentProcessWindows() error {
 	return fmt.Errorf("Guest Agent not found.")
 }
 
-func verifyBootTime() error {
+func verifyBootTime(t *testing.T) error {
 	// Reading the system uptime once both guest agent and sshd are found in the processes
 	uptimeData, err := os.ReadFile("/proc/uptime")
 	if err != nil {
@@ -116,7 +116,7 @@ func verifyBootTime() error {
 	//Validating the uptime against the allowed threshold value
 	var maxThreshold int
 
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(utils.Context(t), "instance", "image")
 	if err != nil {
 		return fmt.Errorf("couldn't get image from metadata")
 	}
@@ -272,7 +272,7 @@ func testWindowsGuestSecureBoot() error {
 }
 
 func TestStartTime(t *testing.T) {
-	metadata, err := utils.GetMetadataAttribute("start-time")
+	metadata, err := utils.GetMetadata(utils.Context(t), "instance", "attributes", "start-time")
 	if err != nil {
 		t.Fatalf("couldn't get start time from metadata")
 	}
@@ -311,7 +311,7 @@ func TestBootTime(t *testing.T) {
 			t.Fatalf("Failed to verify boot time: %v", err)
 		}
 	} else {
-		err := verifyBootTime()
+		err := verifyBootTime(t)
 		if err != nil {
 			t.Fatalf("Failed to verify boot time: %v", err)
 		}

imagetest/test_suites/licensevalidation/license_test.go

@@ -176,7 +176,7 @@ func isValidLicenseText(licenseCheck string) bool {
 }
 
 func TestArePackagesLegal(t *testing.T) {
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(utils.Context(t), "instance", "image")
 	if err != nil {
 		t.Fatalf("couldn't get image from metadata")
 	}
@@ -204,7 +204,7 @@ func TestArePackagesLegal(t *testing.T) {
 }
 
 func TestWindowsActivationStatus(t *testing.T) {
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(utils.Context(t), "instance", "image")
 	if err != nil {
 		t.Fatalf("Couldn't get image from metadata %v", err)
 	}

imagetest/test_suites/licensevalidation/linux_license_test.go

@@ -47,11 +47,11 @@ var imageLicenseCodeMap = map[string]string{
 
 func TestLinuxLicense(t *testing.T) {
 	// Assume only one license exist in image
-	licenseCode, err := utils.GetMetadata("licenses/0/id")
+	licenseCode, err := utils.GetMetadata(utils.Context(t), "instance", "licenses", "0", "id")
 	if err != nil {
 		t.Fatal("Failed to get license code metadata")
 	}
-	image, err := utils.GetMetadata("image")
+	image, err := utils.GetMetadata(utils.Context(t), "instance", "image")
 	if err != nil {
 		t.Fatal("Failed to get image metadata")
 	}

imagetest/test_suites/metadata/metadata_test.go

@@ -23,7 +23,7 @@ type Token struct {
 
 // TestTokenFetch test service-accounts token could be retrieved from metadata.
 func TestTokenFetch(t *testing.T) {
-	metadata, err := utils.GetMetadata("service-accounts/default/token")
+	metadata, err := utils.GetMetadata(utils.Context(t), "instance", "service-accounts", "default", "token")
 	if err != nil {
 		t.Fatalf("couldn't get token from metadata, err % v", err)
 	}
@@ -34,11 +34,11 @@ func TestTokenFetch(t *testing.T) {
 
 // TestMetaDataResponseHeaders verify that HTTP response headers do not include confidential data.
 func TestMetaDataResponseHeaders(t *testing.T) {
-	resp, err := utils.GetMetadataHTTPResponse("id")
+	_, headers, err := utils.GetMetadataWithHeaders(utils.Context(t), "instance", "id")
 	if err != nil {
 		t.Fatalf("couldn't get id from metadata, err % v", err)
 	}
-	for key, values := range resp.Header {
+	for key, values := range headers {
 		if key != "Metadata-Flavor" {
 			for _, v := range values {
 				if strings.Contains(strings.ToLower(v), "google") {

imagetest/test_suites/metadata/shutdown_script_test.go

@@ -19,8 +19,8 @@ const (
 )
 
 // TestShutdownScriptFailedLinux tests that a script failed execute doesn't crash the vm.
-func testShutdownScriptFailedLinux() error {
-	if _, err := utils.GetMetadataAttribute("shutdown-script"); err != nil {
+func testShutdownScriptFailedLinux(t *testing.T) error {
+	if _, err := utils.GetMetadata(utils.Context(t), "instance", "attributes", "shutdown-script"); err != nil {
 		return fmt.Errorf("couldn't get shutdown-script from metadata")
 	}
 
@@ -29,8 +29,8 @@ func testShutdownScriptFailedLinux() error {
 }
 
 // TestShutdownScriptFailedWindows tests that a script failed execute doesn't crash the vm.
-func testShutdownScriptFailedWindows() error {
-	if _, err := utils.GetMetadataAttribute("windows-shutdown-script-ps1"); err != nil {
+func testShutdownScriptFailedWindows(t *testing.T) error {
+	if _, err := utils.GetMetadata(utils.Context(t), "instance", "attributes", "windows-shutdown-script-ps1"); err != nil {
 		return fmt.Errorf("couldn't get windows-shutdown-script-ps1 from metadata")
 	}
 
@@ -73,7 +73,7 @@ func testShutdownScriptTimeWindows() error {
 // TestShutdownScripts verifies that the standard metadata script could run successfully
 // by checking the output content of the Shutdown script.
 func TestShutdownScripts(t *testing.T) {
-	result, err := utils.GetMetadataGuestAttribute("testing/result")
+	result, err := utils.GetMetadata(utils.Context(t), "instance", "guest-attributes", "testing", "result")
 	if err != nil {
 		t.Fatalf("failed to read shutdown script result key: %v", err)
 	}
@@ -85,19 +85,19 @@ func TestShutdownScripts(t *testing.T) {
 // Determine if the OS is Windows or Linux and run the appropriate failure test.
 func TestShutdownScriptsFailed(t *testing.T) {
 	if utils.IsWindows() {
-		if err := testShutdownScriptFailedWindows(); err != nil {
+		if err := testShutdownScriptFailedWindows(t); err != nil {
 			t.Fatalf("Shutdown script failure test failed with error: %v", err)
 		}
 	} else {
-		if err := testShutdownScriptFailedLinux(); err != nil {
+		if err := testShutdownScriptFailedLinux(t); err != nil {
 			t.Fatalf("Shutdown script failure test failed with error: %v", err)
 		}
 	}
 }
 
 // Determine if the OS is Windows or Linux and run the appropriate daemon test.
 func TestShutdownURLScripts(t *testing.T) {
-	result, err := utils.GetMetadataGuestAttribute("testing/result")
+	result, err := utils.GetMetadata(utils.Context(t), "instance", "guest-attributes", "testing", "result")
 	if err != nil {
 		t.Fatalf("failed to read shutdown script result key: %v", err)
 	}

imagetest/test_suites/metadata/startup_script_test.go

@@ -18,17 +18,17 @@ const (
 )
 
 // TestStartupScriptFailedLinux tests that a script failed execute doesn't crash the vm.
-func testStartupScriptFailedLinux() error {
-	if _, err := utils.GetMetadataAttribute("startup-script"); err != nil {
+func testStartupScriptFailedLinux(t *testing.T) error {
+	if _, err := utils.GetMetadata(utils.Context(t), "instance", "attributes", "startup-script"); err != nil {
 		return fmt.Errorf("couldn't get startup-script from metadata, %v", err)
 	}
 
 	return nil
 }
 
 // TestStartupScriptFailedWindows tests that a script failed execute doesn't crash the vm.
-func testStartupScriptFailedWindows() error {
-	if _, err := utils.GetMetadataAttribute("windows-startup-script-ps1"); err != nil {
+func testStartupScriptFailedWindows(t *testing.T) error {
+	if _, err := utils.GetMetadata(utils.Context(t), "instance", "attributes", "windows-startup-script-ps1"); err != nil {
 		return fmt.Errorf("couldn't get windows-startup-script-ps1 from metadata, %v", err)
 	}
 
@@ -72,7 +72,7 @@ func testDaemonScriptWindows() error {
 // TestStartupScripts verifies that the standard metadata script could run successfully
 // by checking the output content of the Startup script.
 func TestStartupScripts(t *testing.T) {
-	result, err := utils.GetMetadataGuestAttribute("testing/result")
+	result, err := utils.GetMetadata(utils.Context(t), "instance", "attributes", "testing", "result")
 	if err != nil {
 		t.Fatalf("failed to read startup script result key: %v", err)
 	}
@@ -84,11 +84,11 @@ func TestStartupScripts(t *testing.T) {
 // Determine if the OS is Windows or Linux and run the appropriate failure test.
 func TestStartupScriptsFailed(t *testing.T) {
 	if utils.IsWindows() {
-		if err := testStartupScriptFailedWindows(); err != nil {
+		if err := testStartupScriptFailedWindows(t); err != nil {
 			t.Fatalf("Startup script failure test failed with error: %v", err)
 		}
 	} else {
-		if err := testStartupScriptFailedLinux(); err != nil {
+		if err := testStartupScriptFailedLinux(t); err != nil {
 			t.Fatalf("Shutdown script failure test failed with error: %v", err)
 		}
 	}