[RelayMiner]: add proxy.Ping(...) capability to test connectivity between relay servers and backend URLs

Tiltfile

@@ -224,16 +224,28 @@
 actor_number = 0
 for x in range(localnet_config["relayminers"]["count"]):
     actor_number = actor_number + 1
-    helm_resource(
-        "relayminer" + str(actor_number),
-        chart_prefix + "relayminer",
-        flags=[
+
+    flags = [
             "--values=./localnet/kubernetes/values-common.yaml",
             "--values=./localnet/kubernetes/values-relayminer-common.yaml",
             "--values=./localnet/kubernetes/values-relayminer-" + str(actor_number) + ".yaml",
             "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]),
             "--set=development.delve.enabled=" + str(localnet_config["relayminers"]["delve"]["enabled"]),
-        ],
+    ]
+
+    if localnet_config["rest"]["enabled"]:
+       flags.append("--values=./localnet/kubernetes/values-relayminer-" + str(actor_number) + "-rest" + ".yaml")
+
+    if localnet_config["ollama"]["enabled"]:
+       flags.append("--values=./localnet/kubernetes/values-relayminer-" + str(actor_number) + "-ollama" + ".yaml")
+
+    if localnet_config["rest"]["enabled"] and localnet_config["ollama"]["enabled"]:
+       flags.append("--values=./localnet/kubernetes/values-relayminer-" + str(actor_number) + "-all" + ".yaml")
+
+    helm_resource(
+        "relayminer" + str(actor_number),
+        chart_prefix + "relayminer",
+        flags=flags,
         image_deps=["poktrolld"],
         image_keys=[("image.repository", "image.tag")],
     )
@@ -257,6 +269,7 @@
             # Use with pprof like this: `go tool pprof -http=:3333 http://localhost:6070/debug/pprof/goroutine`
             str(6069 + actor_number)
             + ":6060",  # Relayminer pprof port. relayminer1 - exposes 6070, relayminer2 exposes 6071, etc.
+            str(7000 + actor_number) + ":8081", # Relayminer ping port. relayminer1 - exposes 7001, relayminer2 exposes 7002, ect.
         ],
     )
 

docusaurus/docs/operate/configs/relayminer_config.md

@@ -23,6 +23,7 @@ You can find a fully featured example configuration at [relayminer_config_full_e
   - [`smt_store_path`](#smt_store_path)
   - [`metrics`](#metrics)
   - [`pprof`](#pprof)
+  - [`ping`](#ping)
 - [Pocket node connectivity](#pocket-node-connectivity)
   - [`query_node_rpc_url`](#query_node_rpc_url)
   - [`query_node_grpc_url`](#query_node_grpc_url)
@@ -173,6 +174,21 @@ pprof:
 
 You can learn how to use that endpoint on the [Performance Troubleshooting](../../develop/developer_guide/performance_troubleshooting.md) page.
 
+### `ping`
+
+Configures a `ping` server to test the connectivity of every backend URLs. If
+all the backend URLs are reachable, the endpoint returns a 200 HTTP
+Code. Otherwise, if one or more backend URLs aren't reachable, the service
+returns an 500 HTTP Internal server error.
+
+Example configuration:
+
+```yaml
+ping:
+  enabled: true
+  addr: localhost:8081
+```
+
 ## Pocket node connectivity
 
 ```yaml

localnet/kubernetes/values-relayminer-1-all.yaml

@@ -0,0 +1,20 @@
+config:
+  suppliers:
+    - service_id: anvil
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://anvil:8547/
+        publicly_exposed_endpoints:
+          - relayminer1
+    - service_id: rest
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://rest:10000/
+        publicly_exposed_endpoints:
+          - relayminer1
+    - service_id: ollama
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://ollama:11434/
+        publicly_exposed_endpoints:
+          - relayminer1

localnet/kubernetes/values-relayminer-1-ollama.yaml

@@ -0,0 +1,14 @@
+config:
+  suppliers:
+    - service_id: anvil
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://anvil:8547/
+        publicly_exposed_endpoints:
+          - relayminer1
+    - service_id: ollama
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://ollama:11434/
+        publicly_exposed_endpoints:
+          - relayminer1

localnet/kubernetes/values-relayminer-1-rest.yaml

@@ -0,0 +1,14 @@
+config:
+  suppliers:
+    - service_id: anvil
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://anvil:8547/
+        publicly_exposed_endpoints:
+          - relayminer1
+    - service_id: rest
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://rest:10000/
+        publicly_exposed_endpoints:
+          - relayminer1

localnet/kubernetes/values-relayminer-1.yaml

@@ -11,15 +11,3 @@ config:
         backend_url: http://anvil:8547/
         publicly_exposed_endpoints:
           - relayminer1
-    - service_id: ollama
-      listen_url: http://0.0.0.0:8545
-      service_config:
-        backend_url: http://ollama:11434/
-        publicly_exposed_endpoints:
-          - relayminer1
-    - service_id: rest
-      listen_url: http://0.0.0.0:8545
-      service_config:
-        backend_url: http://rest:10000/
-        publicly_exposed_endpoints:
-          - relayminer1

localnet/kubernetes/values-relayminer-2-ollama.yaml

@@ -0,0 +1,14 @@
+config:
+  suppliers:
+    - service_id: anvil
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://anvil:8547/
+        publicly_exposed_endpoints:
+          - relayminer2
+    - service_id: ollama
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://ollama:11434/
+        publicly_exposed_endpoints:
+          - relayminer2

localnet/kubernetes/values-relayminer-2.yaml

@@ -11,9 +11,3 @@ config:
         backend_url: http://anvil:8547/
         publicly_exposed_endpoints:
           - relayminer2
-    - service_id: ollama
-      listen_url: http://0.0.0.0:8545
-      service_config:
-        backend_url: http://ollama:11434/
-        publicly_exposed_endpoints:
-          - relayminer2

localnet/kubernetes/values-relayminer-3-ollama.yaml

@@ -0,0 +1,14 @@
+config:
+  suppliers:
+    - service_id: anvil
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://anvil:8547/
+        publicly_exposed_endpoints:
+          - relayminer3
+    - service_id: ollama
+      listen_url: http://0.0.0.0:8545
+      service_config:
+        backend_url: http://ollama:11434/
+        publicly_exposed_endpoints:
+          - relayminer3

localnet/kubernetes/values-relayminer-3.yaml

@@ -11,9 +11,3 @@ config:
         backend_url: http://anvil:8547/
         publicly_exposed_endpoints:
           - relayminer3
-    - service_id: ollama
-      listen_url: http://0.0.0.0:8545
-      service_config:
-        backend_url: http://ollama:11434/
-        publicly_exposed_endpoints:
-          - relayminer3

localnet/kubernetes/values-relayminer-common.yaml

@@ -11,3 +11,6 @@ config:
   pprof:
     enabled: true
     addr: localhost:6060
+  ping:
+    enabled: true
+    addr: localhost:8081

localnet/poktrolld/config/relayminer_config.yaml

@@ -17,3 +17,6 @@ suppliers:
 pprof:
   enabled: false
   addr: localhost:6060
+ping:
+  enabled: false
+  addr: localhost:8082

localnet/poktrolld/config/relayminer_config_full_example.yaml

@@ -18,6 +18,12 @@ pprof:
   enabled: false
   addr: localhost:6060
 
+# Ping server configuration to test the connectivity of every
+# suppliers.[].service_config.backend_url
+ping:
+  enabled: false
+  addr: localhost:8081
+
 pocket_node:
   # Pocket node URL exposing the CometBFT JSON-RPC API.
   # Used by the Cosmos client SDK, event subscriptions, etc.

localnet/poktrolld/config/relayminer_config_localnet_vscode.yaml

@@ -41,3 +41,6 @@ suppliers:
 pprof:
   enabled: false
   addr: localhost:6070
+ping:
+  enabled: false
+  addr: localhost:8081

pkg/relayer/cmd/cmd.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -134,6 +135,17 @@ func runRelayer(cmd *cobra.Command, _ []string) error {
 		}
 	}
 
+	if relayMinerConfig.Ping.Enabled {
+		ln, err := net.Listen("tcp", relayMinerConfig.Ping.Addr)
+		if err != nil {
+			return fmt.Errorf("failed to listen ping server: %w", err)
+		}
+
+		if err := relayMiner.ServePing(ctx, ln); err != nil {
+			return fmt.Errorf("failed to start ping server: %w", err)
+		}
+	}
+
 	// Start the relay miner
 	logger.Info().Msg("Starting relay miner...")
 	if err := relayMiner.Start(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) {

pkg/relayer/config/relayminer_configs_reader.go

@@ -42,6 +42,11 @@ func ParseRelayMinerConfigs(configContent []byte) (*RelayMinerConfig, error) {
 		Addr:    yamlRelayMinerConfig.Pprof.Addr,
 	}
 
+	relayMinerConfig.Ping = &RelayMinerPingConfig{
+		Enabled: yamlRelayMinerConfig.Ping.Enabled,
+		Addr:    yamlRelayMinerConfig.Ping.Addr,
+	}
+
 	// Hydrate the pocket node urls
 	if err := relayMinerConfig.HydratePocketNodeUrls(&yamlRelayMinerConfig.PocketNode); err != nil {
 		return nil, err

pkg/relayer/config/types.go

@@ -24,6 +24,13 @@ type YAMLRelayMinerConfig struct {
 	Pprof                  YAMLRelayMinerPprofConfig      `yaml:"pprof"`
 	SmtStorePath           string                         `yaml:"smt_store_path"`
 	Suppliers              []YAMLRelayMinerSupplierConfig `yaml:"suppliers"`
+	Ping                   YAMLRelayMinerPingConfig       `yaml:"ping"`
+}
+
+// YAMLRelayMinerPingConfig represents the configuration to expose a ping server.
+type YAMLRelayMinerPingConfig struct {
+	Enabled bool   `yaml:"enabled"`
+	Addr    string `yaml:"addr"`
 }
 
 // YAMLRelayMinerPocketNodeConfig is the structure used to unmarshal the pocket
@@ -83,6 +90,14 @@ type RelayMinerConfig struct {
 	Pprof                  *RelayMinerPprofConfig
 	Servers                map[string]*RelayMinerServerConfig
 	SmtStorePath           string
+	Ping                   *RelayMinerPingConfig
+}
+
+// RelayMinerPingConfig is the structure resulting from parsing the ping
+// server configuration.
+type RelayMinerPingConfig struct {
+	Enabled bool
+	Addr    string
 }
 
 // RelayMinerPocketNodeConfig is the structure resulting from parsing the pocket

pkg/relayer/interface.go

@@ -72,6 +72,9 @@ type RelayerProxy interface {
 	// TODO_TECHDEBT(@red-0ne): This method should be moved out of the RelayerProxy interface
 	// that should not be responsible for signing relay responses.
 	SignRelayResponse(relayResponse *servicetypes.RelayResponse, supplierOperatorAddr string) error
+
+	// PingAll tests the connectivity between all the managed relay servers and their respective backend URLs.
+	PingAll(ctx context.Context) []error
 }
 
 type RelayerProxyOption func(RelayerProxy)
@@ -83,8 +86,14 @@ type RelayServer interface {
 
 	// Stop terminates the service server and returns an error if it fails.
 	Stop(ctx context.Context) error
+
+	// Ping tests the connection between the relay server and its backend URL.
+	Ping(ctx context.Context) error
 }
 
+// RelayServers aggregates a slice of RelayServer interface.
+type RelayServers []RelayServer
+
 // RelayerSessionsManager is responsible for managing the relayer's session lifecycles.
 // It handles the creation and retrieval of SMSTs (trees) for a given session, as
 // well as the respective and subsequent claim creation and proof submission.

pkg/relayer/proxy/proxy.go

@@ -139,14 +139,20 @@
 	if err := rp.BuildProvidedServices(ctx); err != nil {
 		return err
 	}
-
 	// Start the ring cache.
 	rp.ringCache.Start(ctx)
 
 	startGroup, ctx := errgroup.WithContext(ctx)
 
 	for _, relayServer := range rp.servers {
 		server := relayServer // create a new variable scoped to the anonymous function
+
+		// Ensure that each backing data node responds to a ping request
+		// (at least) before continuing operation.
+		if err := server.Ping(ctx); err != nil {
+			return err
+		}
+
 		startGroup.Go(func() error { return server.Start(ctx) })
 	}
 
@@ -187,3 +193,22 @@
 
 	return nil
 }
+
+// PingAll tests the connectivity between all the managed relay servers and their respective backend URLs.
+func (rp *relayerProxy) PingAll(ctx context.Context) []error {
+	var errs []error
+
+	for _, srv := range rp.servers {
+		if err := srv.Ping(ctx); err != nil {
+			rp.logger.Error().Err(err).
+				Msg("an unexpected error occured while pinging backend URL")
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+
+	return nil
+}

pkg/relayer/proxy/proxy_test.go

@@ -152,6 +152,11 @@ func TestRelayerProxy_StartAndStop(t *testing.T) {
 	// Block so relayerProxy has sufficient time to start
 	time.Sleep(100 * time.Millisecond)
 
+	errs := rp.PingAll(ctx)
+	for _, err := range errs {
+		require.NoError(t, err)
+	}
+
 	// Test that RelayerProxy is handling requests (ignoring the actual response content)
 	res, err := http.DefaultClient.Get(fmt.Sprintf("http://%s/", servicesConfigMap[defaultRelayMinerServer].ListenAddress))
 	require.NoError(t, err)