diff --git a/internal/net/grpc/status/status.go b/internal/net/grpc/status/status.go
index 5a21e298e41..12ec9087657 100644
--- a/internal/net/grpc/status/status.go
+++ b/internal/net/grpc/status/status.go
@@ -684,10 +684,7 @@ func withDetails(st *Status, err error, details ...any) *Status {
 			for _, msg := range ds {
 				r, ok := msg.(*errdetails.RetryInfo)
 				if ok && r != nil {
-					if m.GetRetryDelay().GetSeconds() == 0 &&
-						r.GetRetryDelay().GetSeconds() != 0 {
-						m.RetryDelay = r.GetRetryDelay()
-					} else if m.GetRetryDelay().GetSeconds() > r.GetRetryDelay().GetSeconds() {
+					if m.GetRetryDelay() == nil || r.GetRetryDelay().Seconds < m.GetRetryDelay().Seconds {
 						m.RetryDelay = r.GetRetryDelay()
 					}
 				}