feat(CosmosFullNode): Allow more configuration of p2p services (#331)

api/v1/cosmosfullnode_types.go

@@ -635,20 +635,26 @@ const (
 )
 
 type ServiceSpec struct {
-	// MaxSize number of p2p services to create for CometBFT peer exchange.
+	// Max number of external p2p services to create for CometBFT peer exchange.
 	// The public endpoint is set as the "p2p.external_address" in the config.toml.
+	// Controller creates p2p services for each pod so that every pod can peer with each other internally in the cluster.
+	// This setting allows you to control the number of p2p services exposed for peers outside of the cluster to use.
 	// If not set, defaults to 1.
 	// +kubebuilder:validation:Minimum:=0
 	// +optional
 	MaxP2PExternalAddresses *int32 `json:"maxP2PExternalAddresses"`
 
+	// Overrides for all P2P services that need external addresses.
+	// +optional
+	P2PTemplate ServiceOverridesSpec `json:"p2pTemplate"`
+
 	// Overrides for the single RPC service.
 	// +optional
-	RPCTemplate RPCServiceSpec `json:"rpcTemplate"`
+	RPCTemplate ServiceOverridesSpec `json:"rpcTemplate"`
 }
 
-// RPCServiceSpec allows some overrides for the created, single RPC service.
-type RPCServiceSpec struct {
+// ServiceOverridesSpec allows some overrides for the created, single RPC service.
+type ServiceOverridesSpec struct {
 	// +optional
 	Metadata Metadata `json:"metadata"`
 

config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml

@@ -5610,12 +5610,55 @@ spec:
                   exchange.
                 properties:
                   maxP2PExternalAddresses:
-                    description: MaxSize number of p2p services to create for CometBFT
-                      peer exchange. The public endpoint is set as the "p2p.external_address"
-                      in the config.toml. If not set, defaults to 1.
+                    description: Max number of external p2p services to create for
+                      CometBFT peer exchange. The public endpoint is set as the "p2p.external_address"
+                      in the config.toml. Controller creates p2p services for each
+                      pod so that every pod can peer with each other internally in
+                      the cluster. This setting allows you to control the number of
+                      p2p services exposed for peers outside of the cluster to use.
+                      If not set, defaults to 1.
                     format: int32
                     minimum: 0
                     type: integer
+                  p2pTemplate:
+                    description: Overrides for all P2P services that need external
+                      addresses.
+                    properties:
+                      externalTrafficPolicy:
+                        description: 'Sets endpoint and routing behavior. See: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#caveats-and-limitations-when-preserving-source-ips
+                          If not set, defaults to "Cluster".'
+                        enum:
+                        - Cluster
+                        - Local
+                        type: string
+                      metadata:
+                        description: Metadata is a subset of k8s object metadata.
+                        properties:
+                          annotations:
+                            additionalProperties:
+                              type: string
+                            description: Annotations are added to a resource. If there
+                              is a collision between annotations the Operator creates,
+                              the Operator annotations take precedence.
+                            type: object
+                          labels:
+                            additionalProperties:
+                              type: string
+                            description: Labels are added to a resource. If there
+                              is a collision between labels the Operator creates,
+                              the Operator labels take precedence.
+                            type: object
+                        type: object
+                      type:
+                        description: Describes ingress methods for a service. If not
+                          set, defaults to "ClusterIP".
+                        enum:
+                        - ClusterIP
+                        - NodePort
+                        - LoadBalancer
+                        - ExternalName
+                        type: string
+                    type: object
                   rpcTemplate:
                     description: Overrides for the single RPC service.
                     properties:

config/samples/cosmos_v1_cosmosfullnode_full.yaml

@@ -99,6 +99,14 @@ spec:
   service:
     # Creates N LoadBalancer services and exposes those public IPs in p2p.
     maxP2PExternalAddresses: 1
+    # Configures all p2p services that need external addresses.
+    p2pTemplate:
+      metadata:
+        labels:
+          extra: labels
+        annotations:
+          extra: annotations
+      type: NodePort
     # Configures the main service that load balances all RPC ports. Custom configuration likely necessary for Ingress.
     rpcTemplate:
       metadata:

internal/fullnode/service_builder.go

@@ -44,6 +44,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 			kube.InstanceLabel, instanceName(crd, ordinal),
 			kube.ComponentLabel, "p2p",
 		)
+		svc.Annotations = map[string]string{}
 
 		svc.Spec.Ports = []corev1.ServicePort{
 			{
@@ -56,8 +57,10 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 		svc.Spec.Selector = map[string]string{kube.InstanceLabel: instanceName(crd, ordinal)}
 
 		if i < maxExternal {
-			svc.Spec.Type = corev1.ServiceTypeLoadBalancer
-			svc.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal
+			preserveMergeInto(svc.Labels, crd.Spec.Service.P2PTemplate.Metadata.Labels)
+			preserveMergeInto(svc.Annotations, crd.Spec.Service.P2PTemplate.Metadata.Annotations)
+			svc.Spec.Type = *valOrDefault(crd.Spec.Service.P2PTemplate.Type, ptr(corev1.ServiceTypeLoadBalancer))
+			svc.Spec.ExternalTrafficPolicy = *valOrDefault(crd.Spec.Service.P2PTemplate.ExternalTrafficPolicy, ptr(corev1.ServiceExternalTrafficPolicyTypeLocal))
 		} else {
 			svc.Spec.Type = corev1.ServiceTypeClusterIP
 		}

internal/fullnode/service_builder_test.go

@@ -25,7 +25,7 @@ func TestBuildServices(t *testing.T) {
 		crd.Namespace = "test"
 		crd.Spec.ChainSpec.Network = "testnet"
 		crd.Spec.PodTemplate.Image = "terra:v6.0.0"
-		crd.Spec.Service.MaxP2PExternalAddresses = ptr(int32(0))
+
 		svcs := BuildServices(&crd)
 
 		require.Equal(t, 4, len(svcs)) // 3 p2p services + 1 rpc service
@@ -57,6 +57,11 @@ func TestBuildServices(t *testing.T) {
 				Selector: map[string]string{"app.kubernetes.io/instance": fmt.Sprintf("terra-%d", i)},
 				Type:     corev1.ServiceTypeClusterIP,
 			}
+			// By default, expose the first p2p service publicly.
+			if i == 0 {
+				wantSpec.Type = corev1.ServiceTypeLoadBalancer
+				wantSpec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal
+			}
 
 			require.Equal(t, wantSpec, p2p.Spec)
 		}
@@ -85,6 +90,79 @@ func TestBuildServices(t *testing.T) {
 		require.Empty(t, got.Spec.ExternalTrafficPolicy)
 	})
 
+	t.Run("zero p2p max external addresses", func(t *testing.T) {
+		crd := defaultCRD()
+		crd.Spec.Replicas = 3
+		crd.Spec.Service.MaxP2PExternalAddresses = ptr(int32(0))
+		// These overrides should be ignored.
+		crd.Spec.Service.P2PTemplate = cosmosv1.ServiceOverridesSpec{
+			Metadata: cosmosv1.Metadata{
+				Labels: map[string]string{"test": "should not see me"},
+			},
+			Type:                  ptr(corev1.ServiceTypeNodePort),
+			ExternalTrafficPolicy: ptr(corev1.ServiceExternalTrafficPolicyTypeLocal),
+		}
+
+		svcs := BuildServices(&crd)
+
+		gotP2P := lo.Filter(svcs, func(s diff.Resource[*corev1.Service], _ int) bool {
+			return s.Object().Labels[kube.ComponentLabel] == "p2p"
+		})
+
+		require.Equal(t, 3, len(gotP2P))
+		for i, svc := range gotP2P {
+			p2p := svc.Object()
+			require.Empty(t, p2p.Labels["test"])
+			require.Equal(t, corev1.ServiceTypeClusterIP, p2p.Spec.Type, i)
+			require.Empty(t, p2p.Spec.ExternalTrafficPolicy, i)
+		}
+	})
+
+	t.Run("p2p services with overrides", func(t *testing.T) {
+		crd := defaultCRD()
+		crd.Spec.Replicas = 2
+		crd.Name = "terra"
+		crd.Spec.Service.MaxP2PExternalAddresses = ptr(int32(2))
+		crd.Spec.Service.P2PTemplate = cosmosv1.ServiceOverridesSpec{
+			Metadata: cosmosv1.Metadata{
+				Labels:      map[string]string{"test": "value1", "app.kubernetes.io/name": "should not see me"},
+				Annotations: map[string]string{"test": "value2", "app.kubernetes.io/ordinal": "should not see me"},
+			},
+			Type:                  ptr(corev1.ServiceTypeNodePort),
+			ExternalTrafficPolicy: ptr(corev1.ServiceExternalTrafficPolicyTypeLocal),
+		}
+		svcs := BuildServices(&crd)
+
+		require.Equal(t, 3, len(svcs)) // 2 p2p services + 1 rpc service
+
+		for i, svc := range svcs[:2] {
+			p2p := svc.Object()
+			require.Equal(t, fmt.Sprintf("terra-p2p-%d", i), p2p.Name)
+
+			require.Equal(t, "value1", p2p.Labels["test"])
+			require.NotEqual(t, "should not see me", p2p.Labels["app.kubernetes.io/name"])
+
+			require.Equal(t, "value2", p2p.Annotations["test"])
+			require.NotEqual(t, "should not see me", p2p.Labels["app.kubernetes.io/ordinal"])
+
+			wantSpec := corev1.ServiceSpec{
+				Ports: []corev1.ServicePort{
+					{
+						Name:       "p2p",
+						Protocol:   corev1.ProtocolTCP,
+						Port:       26656,
+						TargetPort: intstr.FromString("p2p"),
+					},
+				},
+				Selector:              map[string]string{"app.kubernetes.io/instance": fmt.Sprintf("terra-%d", i)},
+				Type:                  corev1.ServiceTypeNodePort,
+				ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal,
+			}
+
+			require.Equal(t, wantSpec, p2p.Spec)
+		}
+	})
+
 	t.Run("rpc service", func(t *testing.T) {
 		crd := defaultCRD()
 		crd.Spec.Replicas = 1
@@ -156,7 +234,7 @@ func TestBuildServices(t *testing.T) {
 		crd.Namespace = "test"
 		crd.Spec.ChainSpec.Network = "testnet"
 		crd.Spec.PodTemplate.Image = "terra:v6.0.0"
-		crd.Spec.Service.RPCTemplate = cosmosv1.RPCServiceSpec{
+		crd.Spec.Service.RPCTemplate = cosmosv1.ServiceOverridesSpec{
 			Metadata: cosmosv1.Metadata{
 				Labels:      map[string]string{"label": "value", "app.kubernetes.io/name": "should not see me"},
 				Annotations: map[string]string{"test": "value"},