Merge branch 'GoogleCloudPlatform:main' into extra-attributes-saml-su…

…pport

.ci/containers/build-environment/Dockerfile

@@ -1,4 +1,3 @@
-# Stage 1: Building Go dependencies
 FROM golang:1.23-bullseye AS builder
 
 # Set working directory
@@ -11,11 +10,6 @@ ADD "https://raw.githubusercontent.com/GoogleCloudPlatform/magic-modules/main/tp
 # Install the go dependencies
 RUN go mod download
 
-# Stage 2: Creating the final imag
-FROM ruby:3.1-bullseye
-
-# golang
-COPY --from=golang:1.23-bullseye /usr/local/go /usr/local/go
 ENV GOPATH /go
 ENV PATH /usr/local/go/bin:$PATH
 ENV PATH $GOPATH/bin:$PATH
@@ -38,12 +32,3 @@ RUN git config --global user.email "[email protected]"
 
 RUN go install golang.org/x/tools/cmd/goimports@d088b475e3360caabc032aaee1dc66351d4e729a
 RUN go install github.com/github/[email protected]+incompatible
-
-ADD "https://raw.githubusercontent.com/GoogleCloudPlatform/magic-modules/refs/heads/legacy-ruby/mmv1/Gemfile" Gemfile
-ADD "https://raw.githubusercontent.com/GoogleCloudPlatform/magic-modules/refs/heads/legacy-ruby/mmv1/Gemfile.lock" Gemfile.lock
-RUN bundle install
-RUN rm Gemfile Gemfile.lock
-
-# Copy Go dependencies from builder stage
-COPY --from=builder /go/pkg /go/pkg
-

.ci/gcb-test-failure-ticket.yaml

@@ -0,0 +1,19 @@
+---
+steps:
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: gcb-test-failure-ticket
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["TEAMCITY_TOKEN"]
+      args:
+        - 'collect-nightly-test-status'
+        - $_CUSTOM_DATE
+
+timeout: 3600s
+options:
+    machineType: 'N1_HIGHCPU_32'
+
+logsBucket: 'gs://cloudbuild-test-failure-ticket-logs'
+availableSecrets:
+  secretManager:
+    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
+      env: TEAMCITY_TOKEN

.ci/infra/terraform/main.tf

@@ -298,6 +298,7 @@ module "project-services" {
     "logging.googleapis.com",
     "looker.googleapis.com",
     "managedidentities.googleapis.com",
+    "managedkafka.googleapis.com",
     "memcache.googleapis.com",
     "memorystore.googleapis.com",
     "metastore.googleapis.com",
@@ -316,6 +317,7 @@ module "project-services" {
     "osconfig.googleapis.com",
     "oslogin.googleapis.com",
     "parallelstore.googleapis.com",
+    "parametermanager.googleapis.com",
     "privateca.googleapis.com",
     "privilegedaccessmanager.googleapis.com",
     "pubsub.googleapis.com",
@@ -467,6 +469,13 @@ resource "google_project_iam_member" "compute_agent_encrypter_decrypter" {
   member  = "serviceAccount:service-${google_project.proj.number}@compute-system.iam.gserviceaccount.com"
 }
 
+# TestAccColabRuntime_colabRuntimeBasicExample
+# TestAccColabRuntime_colabRuntimeFullExample
+resource "google_project_iam_member" "colab_admin_permissions" {
+  project = google_project.proj.project_id
+  role    = "roles/aiplatform.colabEnterpriseAdmin"
+  member  = "user:[email protected]"
+}
 
 data "google_organization" "org2" {
   organization = var.org2_id

.ci/magician/cloudstorage/bucket.go

@@ -0,0 +1,58 @@
+/*
+* Copyright 2025 Google LLC. All Rights Reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+ */
+package cloudstorage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"cloud.google.com/go/storage"
+)
+
+func (gcs *Client) WriteToGCSBucket(bucket, object, filePath string) error {
+	ctx := context.Background()
+
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return fmt.Errorf("storage.NewClient: %v", err)
+	}
+	defer client.Close()
+
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("os.Open: %w", err)
+	}
+	defer file.Close()
+
+	ctx, cancel := context.WithTimeout(ctx, time.Second*50)
+	defer cancel()
+
+	writer := client.Bucket(bucket).Object(object).NewWriter(ctx)
+	writer.ContentType = "application/json"
+
+	if _, err = io.Copy(writer, file); err != nil {
+		return fmt.Errorf("io.Copy: %w", err)
+	}
+	if err := writer.Close(); err != nil {
+		return fmt.Errorf("Writer.Close: %w", err)
+	}
+
+	fmt.Printf("File uploaded to bucket %s as %s\n", bucket, object)
+	return nil
+}

.ci/magician/cloudstorage/init.go

@@ -0,0 +1,23 @@
+/*
+* Copyright 2025 Google LLC. All Rights Reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+ */
+package cloudstorage
+
+type Client struct {
+}
+
+func NewClient() *Client {
+	return &Client{}
+}

.ci/magician/cmd/collect_nightly_test_status.go

@@ -0,0 +1,228 @@
+/*
+* Copyright 2025 Google LLC. All Rights Reserved.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+ */
+package cmd
+
+import (
+	"fmt"
+	"magician/cloudstorage"
+	"magician/provider"
+	"magician/teamcity"
+	utils "magician/utility"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+const (
+	NIGHTLY_DATA_BUCKET = "nightly-test-data"
+)
+
+var cntsRequiredEnvironmentVariables = [...]string{
+	"TEAMCITY_TOKEN",
+}
+
+type TestInfo struct {
+	Name         string `json:"name"`
+	Status       string `json:"status"`
+	Service      string `json:"service"`
+	ErrorMessage string `json:"error_message"`
+	LogLink      string `json"log_link`
+}
+
+// collectNightlyTestStatusCmd represents the collectNightlyTestStatus command
+var collectNightlyTestStatusCmd = &cobra.Command{
+	Use:   "collect-nightly-test-status",
+	Short: "Collects and stores nightly test status",
+	Long: `This command collects nightly test status, stores the data in JSON files and upload the files to GCS.
+
+
+	The command expects the following argument(s):
+	1. Custom test date in YYYY-MM-DD format. default: ""(current time when the job is executed)
+
+	It then performs the following operations:
+	1. Collects nightly test status of the execution day or the specified test date (if provided)
+	2. Stores the collected data in JSON files
+	3. Uploads the JSON files to GCS
+
+	The following environment variables are required:
+` + listCNTSRequiredEnvironmentVariables(),
+	Args: cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		env := make(map[string]string)
+		for _, ev := range cntsRequiredEnvironmentVariables {
+			val, ok := os.LookupEnv(ev)
+			if !ok {
+				return fmt.Errorf("did not provide %s environment variable", ev)
+			}
+			env[ev] = val
+		}
+
+		tc := teamcity.NewClient(env["TEAMCITY_TOKEN"])
+		gcs := cloudstorage.NewClient()
+
+		now := time.Now()
+
+		loc, err := time.LoadLocation("America/Los_Angeles")
+		if err != nil {
+			return fmt.Errorf("Error loading location: %s", err)
+		}
+		date := now.In(loc)
+		customDate := args[0]
+		// check if a specific date is provided
+		if customDate != "" {
+			parsedDate, err := time.Parse("2006-01-02", customDate) // input format YYYY-MM-DD
+			// Set the time to 6pm PT
+			date = time.Date(parsedDate.Year(), parsedDate.Month(), parsedDate.Day(), 18, 0, 0, 0, loc)
+			if err != nil {
+				return fmt.Errorf("invalid input time format: %w", err)
+			}
+		}
+
+		return execCollectNightlyTestStatus(date, tc, gcs)
+	},
+}
+
+func listCNTSRequiredEnvironmentVariables() string {
+	var result string
+	for i, ev := range cntsRequiredEnvironmentVariables {
+		result += fmt.Sprintf("\t%2d. %s\n", i+1, ev)
+	}
+	return result
+}
+
+func execCollectNightlyTestStatus(now time.Time, tc TeamcityClient, gcs CloudstorageClient) error {
+	lastday := now.AddDate(0, 0, -1)
+	formattedStartCut := lastday.Format(time.RFC3339)
+	formattedFinishCut := now.Format(time.RFC3339)
+	date := now.Format("2006-01-02")
+
+	err := createTestReport(provider.GA, tc, gcs, formattedStartCut, formattedFinishCut, date)
+	if err != nil {
+		return fmt.Errorf("Error getting GA nightly test status: %w", err)
+	}
+
+	err = createTestReport(provider.Beta, tc, gcs, formattedStartCut, formattedFinishCut, date)
+	if err != nil {
+		return fmt.Errorf("Error getting Beta nightly test status: %w", err)
+	}
+
+	return nil
+}
+
+func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs CloudstorageClient, formattedStartCut, formattedFinishCut, date string) error {
+	// Get all service test builds
+	builds, err := tc.GetBuilds(pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	if err != nil {
+		return err
+	}
+
+	var testInfoList []TestInfo
+	for _, build := range builds.Builds {
+		// Get service package name
+		serviceName, err := convertServiceName(build.BuildTypeId)
+		if err != nil {
+			return fmt.Errorf("failed to convert test service name for %s: %v", build.BuildTypeId, err)
+		}
+		// Skip sweeper package
+		if serviceName == "sweeper" {
+			continue
+		}
+
+		// Get test results
+		serviceTestResults, err := tc.GetTestResults(build)
+		if err != nil {
+			return fmt.Errorf("failed to get test results: %v", err)
+		}
+		if len(serviceTestResults.TestResults) == 0 {
+			fmt.Printf("Service %s has no tests\n", serviceName)
+			continue
+		}
+
+		for _, testResult := range serviceTestResults.TestResults {
+			var errorMessage string
+			// Get test debug log gcs link
+			logLink := fmt.Sprintf("https://storage.cloud.google.com/teamcity-logs/nightly/%s/%s/%s/debug-%s-%s-%s-%s.txt", pVersion.TeamCityNightlyProjectName(), date, build.Number, pVersion.ProviderName(), build.Number, strconv.Itoa(build.Id), testResult.Name)
+			// Get concise error message
+			if testResult.Status == "FAILURE" {
+				errorMessage = convertErrorMessage(testResult.ErrorMessage)
+			}
+			testInfoList = append(testInfoList, TestInfo{
+				Name:         testResult.Name,
+				Status:       testResult.Status,
+				Service:      serviceName,
+				ErrorMessage: errorMessage,
+				LogLink:      logLink,
+			})
+		}
+	}
+
+	// Write test status data to a JSON file
+	fmt.Println("Write test status")
+	testStatusFileName := fmt.Sprintf("%s-%s.json", date, pVersion.String())
+	err = utils.WriteToJson(testInfoList, testStatusFileName)
+	if err != nil {
+		return err
+	}
+
+	// Upload test status data file to gcs bucket
+	objectName := pVersion.String() + "/" + testStatusFileName
+	err = gcs.WriteToGCSBucket(NIGHTLY_DATA_BUCKET, objectName, testStatusFileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// convertServiceName extracts service package name from teamcity build type id
+// input: TerraformProviders_GoogleCloud_GOOGLE_NIGHTLYTESTS_GOOGLE_PACKAGE_SECRETMANAGER
+// output: secretmanager
+func convertServiceName(servicePath string) (string, error) {
+	idx := strings.LastIndex(servicePath, "_")
+
+	if idx != -1 {
+		return strings.ToLower(servicePath[idx+1:]), nil
+	}
+	return "", fmt.Errorf("wrong service path format for %s", servicePath)
+}
+
+// convertErrorMessage returns concise error message
+func convertErrorMessage(rawErrorMessage string) string {
+
+	startMarker := "------- Stdout: -------"
+	endMarker := "------- Stderr: -------"
+	startIndex := strings.Index(rawErrorMessage, startMarker)
+	endIndex := strings.Index(rawErrorMessage, endMarker)
+
+	if startIndex != -1 {
+		startIndex += len(startMarker)
+	} else {
+		startIndex = 0
+	}
+
+	if endIndex == -1 {
+		endIndex = len(rawErrorMessage)
+	}
+
+	return strings.TrimSpace(rawErrorMessage[startIndex:endIndex])
+}
+
+func init() {
+	rootCmd.AddCommand(collectNightlyTestStatusCmd)
+}