test migration

Signed-off-by: Wei Liu <[email protected]>

.github/workflows/e2e.yml

@@ -67,3 +67,20 @@ jobs:
           container_tool: docker
           SERVER_REPLICAS: 2
           MESSAGE_DRIVER_TYPE: grpc
+  migration:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: install ginkgo
+        run: go install github.com/onsi/ginkgo/v2/[email protected]
+      - name: Test E2E
+        run: |
+          make migration-test
+        env:
+          container_tool: docker
+          SERVER_REPLICAS: 2

Makefile

@@ -451,7 +451,7 @@ e2e-test/teardown:
 	./test/e2e/setup/e2e_teardown.sh
 .PHONY: e2e-test/teardown
 
-e2e-test: e2e-test/teardown e2e-test/setup
+e2e-test/run:
 	ginkgo -v --fail-fast --label-filter="!(e2e-tests-spec-resync-reconnect||e2e-tests-status-resync-reconnect)" \
 	--output-dir="${PWD}/test/e2e/report" --json-report=report.json --junit-report=report.xml \
 	${PWD}/test/e2e/pkg -- \
@@ -460,4 +460,11 @@ e2e-test: e2e-test/teardown e2e-test/setup
 	-server-kubeconfig=${PWD}/test/e2e/.kubeconfig \
 	-consumer-name=$(shell cat ${PWD}/test/e2e/.consumer_name) \
 	-agent-kubeconfig=${PWD}/test/e2e/.kubeconfig
+.PHONY: e2e-test/run
+
+e2e-test: e2e-test/teardown e2e-test/setup e2e-test/run
 .PHONY: e2e-test
+
+migration-test: e2e-test/teardown
+	./test/e2e/migration/test.sh
+.PHONY: migration-test

pkg/db/migrations.go

@@ -14,6 +14,10 @@ import (
 // For help writing migration steps, see the gorm documentation on migrations: http://doc.gorm.io/database.html#migration
 
 func Migrate(g2 *gorm.DB) error {
+	if err := migrations.CleanUpDirtyData(g2); err != nil {
+		return err
+	}
+
 	m := newGormigrate(g2)
 
 	if err := m.Migrate(); err != nil {

pkg/db/migrations/migration_structs.go

@@ -1,14 +1,19 @@
 package migrations
 
 import (
+	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
 	"gorm.io/gorm"
 
 	"github.com/go-gormigrate/gormigrate/v2"
+	"github.com/openshift-online/maestro/pkg/logger"
 )
 
+var log = logger.NewOCMLogger(context.Background())
+
 // gormigrate is a wrapper for gorm's migration functions that adds schema versioning and rollback capabilities.
 // For help writing migration steps, see the gorm documentation on migrations: http://doc.gorm.io/database.html#migration
 
@@ -38,6 +43,23 @@ var MigrationList = []*gormigrate.Migration{
 	alterEventInstances(),
 }
 
+// CleanUpDirtyData clean up the dirty data before migrating the tables.
+// When we add new constraints to old tables, we should especially consider the possibility of dirty data.
+func CleanUpDirtyData(db *gorm.DB) error {
+	// Clean up the dirty data from event_instances tale.
+	// the new constraints are added by migration `202412181141_alter_event_instances`,
+	// before do this migration, we should check and clean up dirty data
+	if err := cleanupUnreferencedData(db, "event_instances", "server_instances", "instance_id", "id"); err != nil {
+		return err
+	}
+
+	if err := cleanupUnreferencedData(db, "event_instances", "status_events", "event_id", "id"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Model represents the base model struct. All entities will have this struct embedded.
 type Model struct {
 	ID        string `gorm:"primary_key"`
@@ -69,6 +91,40 @@ func CreateFK(g2 *gorm.DB, fks ...fkMigration) error {
 	return nil
 }
 
+func cleanupUnreferencedData(db *gorm.DB, table, referencedTable, fk, pk string) error {
+	if !db.Migrator().HasTable(table) {
+		return nil
+	}
+
+	var nullIDs []sql.NullString
+	query := fmt.Sprintf(`
+        SELECT %s.%s
+        FROM %s
+        LEFT JOIN %s ON %s.%s = %s.%s
+        WHERE %s.%s IS NULL
+    `, table, fk, table, referencedTable, table, fk, referencedTable, pk, referencedTable, pk)
+
+	if err := db.Raw(query).Pluck(fk, &nullIDs).Error; err != nil {
+		return fmt.Errorf("failed to run query %s: %v", query, err)
+	}
+
+	var invalidIDs []string
+	for _, nullID := range nullIDs {
+		if nullID.Valid {
+			invalidIDs = append(invalidIDs, nullID.String)
+		}
+	}
+
+	if len(invalidIDs) != 0 {
+		log.Infof("clean up the dirty data from %s (fk=%s): %v", table, fk, invalidIDs)
+		if err := db.Table(table).Where(fk+" IN ?", invalidIDs).Delete(nil).Error; err != nil {
+			return fmt.Errorf("failed to delete dirty data: %v", err)
+		}
+	}
+
+	return nil
+}
+
 func fkName(model, dest string) string {
 	return fmt.Sprintf("fk_%s_%s", model, dest)
 }

test/e2e/migration/test.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -ex
+#
+# Copyright (c) 2023 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# get the last image tag from quay.io
+img_repo_api="https://quay.io/api/v1/repository/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro"
+img_registry="quay.io/redhat-user-workloads/maestro-rhtap-tenant"
+last_tag=$(curl -s -X GET "${img_repo_api}" | jq -s -c -r 'sort_by(.tags[].last_modified) | .[].tags[].name' | grep -E '^[a-z0-9]{40}$' | head -n 1)
+
+# use the last tag as the default commit sha
+commit_sha=${commit_sha:-"$last_tag"}
+
+output_dir="./_output/migration"
+
+rm -rf $output_dir
+mkdir -p $output_dir
+
+# run the e2e-test in the main repo with the commit sha
+git clone https://github.com/openshift-online/maestro.git "$output_dir/maestro"
+pushd $output_dir/maestro
+git checkout $commit_sha
+image_tag=$commit_sha external_image_registry=$img_registry internal_image_registry=$img_registry make e2e-test
+popd
+
+# copy the configurations
+cp $output_dir/maestro/test/e2e/.kubeconfig ./test/e2e/.kubeconfig
+cp $output_dir/maestro/test/e2e/.consumer_name ./test/e2e/.consumer_name
+cp $output_dir/maestro/test/e2e/.external_host_ip ./test/e2e/.external_host_ip
+# cp -r $output_dir/maestro/test/e2e/certs ./test/e2e/certs
+
+# run the e2e test in the current repo (upgrade)
+# make e2e-test/setup
+make e2e-test/run