CE changes for vault-31750 (#29303)

* ce changes for vault-31750

* add changelog

* make proto

* refactor naming

* clarify error message

* update changelog

* one more time

* make proto AGAIN

changelog/29303.txt

@@ -0,0 +1,3 @@
+```release-note:change
+core (enterprise): Add tracking of performance standbys by their HA node ID so that RPC connections can be more easily cleaned up when nodes are removed.
+```

physical/raft/raft.go

@@ -256,8 +256,9 @@ type RaftBackend struct {
 	// limits.
 	specialPathLimits map[string]uint64
 
-	removed         *atomic.Bool
-	removedCallback func()
+	removed              *atomic.Bool
+	removedCallback      func()
+	removedServerCleanup func(context.Context, string) (bool, error)
 }
 
 func (b *RaftBackend) IsNodeRemoved(ctx context.Context, nodeID string) (bool, error) {
@@ -284,6 +285,23 @@ func (b *RaftBackend) RemoveSelf() error {
 	return b.stableStore.SetUint64(removedKey, 1)
 }
 
+func (b *RaftBackend) SetRemovedServerCleanupFunc(f func(context.Context, string) (bool, error)) {
+	b.l.Lock()
+	b.removedServerCleanup = f
+	b.l.Unlock()
+}
+
+func (b *RaftBackend) RemovedServerCleanup(ctx context.Context, nodeID string) (bool, error) {
+	b.l.RLock()
+	defer b.l.RUnlock()
+
+	if b.removedServerCleanup != nil {
+		return b.removedServerCleanup(ctx, nodeID)
+	}
+
+	return false, nil
+}
+
 // LeaderJoinInfo contains information required by a node to join itself as a
 // follower to an existing raft cluster
 type LeaderJoinInfo struct {

physical/raft/raft_autopilot.go

@@ -666,6 +666,10 @@ func (d *Delegate) RemoveFailedServer(server *autopilot.Server) {
 		}
 
 		d.followerStates.Delete(string(server.ID))
+		_, err := d.RemovedServerCleanup(context.Background(), string(server.ID))
+		if err != nil {
+			d.logger.Error("failed to run cleanup", "error", err)
+		}
 	}()
 }
 

vault/logical_system_raft.go

@@ -268,6 +268,13 @@ func (b *SystemBackend) handleRaftRemovePeerUpdate() framework.OperationFunc {
 		}
 
 		b.Core.raftFollowerStates.Delete(serverID)
+		_, err := raftBackend.RemovedServerCleanup(ctx, serverID)
+		if err != nil {
+			// log the error but don't return it - we might get an error if we can't find the node in the cache, which
+			// is not an error condition in this instance.
+			b.logger.Info("attempted to remove node from perf standby cache but it failed, which might be fine", "server ID", serverID, "error", err)
+			return nil, nil
+		}
 
 		return nil, nil
 	}