From 9d64b7aadf2f028ee7ef136abe6a8d7b314b6bcb Mon Sep 17 00:00:00 2001
From: shunki-fujita <shunki-fujita@cybozu.co.jp>
Date: Fri, 5 Jul 2024 08:31:25 +0000
Subject: [PATCH] wip

---
 clustering/operations.go | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/clustering/operations.go b/clustering/operations.go
index 4c149ced..0c5380c6 100644
--- a/clustering/operations.go
+++ b/clustering/operations.go
@@ -149,14 +149,31 @@ func (p *managerProcess) clone(ctx context.Context, ss *StatusSet) (bool, error)
 func (p *managerProcess) switchover(ctx context.Context, ss *StatusSet) error {
 	log := logFromContext(ctx)
 	log.Info("begin switchover the primary", "current", ss.Primary, "next", ss.Candidate)
-
 	pdb := ss.DBOps[ss.Primary]
-	if err := pdb.SetReadOnly(ctx, true); err != nil {
-		return fmt.Errorf("failed to make instance %d read-only: %w", ss.Primary, err)
+	readOnly := false
+	for i := 0; i < 2; i++ {
+		ctxTimeout, cancel := context.WithTimeout(ctx, 15*time.Second)
+		defer cancel()
+		errCh := make(chan error)
+		go func() {
+			errCh <- pdb.SetReadOnly(ctxTimeout, true)
+		}()
+		err := <-errCh
+		if err != nil {
+			log.Error(err, "failed to set read-only mode", "instance", ss.Primary)
+		} else {
+			readOnly = true
+		}
+		time.Sleep(100 * time.Millisecond)
+		if err = pdb.KillConnections(ctx); err != nil {
+			return fmt.Errorf("failed to kill connections in instance %d: %w", ss.Primary, err)
+		}
+		if readOnly {
+			break
+		}
 	}
-	time.Sleep(100 * time.Millisecond)
-	if err := pdb.KillConnections(ctx); err != nil {
-		return fmt.Errorf("failed to kill connections in instance %d: %w", ss.Primary, err)
+	if !readOnly {
+		return fmt.Errorf("failed to set read-only mode in instance %d", ss.Primary)
 	}
 	pst, err := pdb.GetStatus(ctx)
 	if err != nil {