feat: add http routes and handlers to get/patch the memstore config

.golangci.yml

@@ -71,8 +71,7 @@ linters-settings:
     blocked:
       # List of blocked modules.
       # Default: []
-      modules:
-        - 
+      modules: []
   govet:
     # Enable all analyzers.
     # Default: false
@@ -133,9 +132,8 @@ linters:
     - durationcheck # checks for two durations multiplied together
     - errname # checks that sentinel errors are prefixed with the Err and error types are suffixed with the Error
     - errorlint # finds code that will cause problems with the error wrapping scheme introduced in Go 1.13
-    - execinquery # checks query string in Query function which reads your Go src files and warning it finds
     - exhaustive # checks exhaustiveness of enum switch statements
-    - exportloopref # checks for pointers to enclosing loop variables
+    - copyloopvar # checks for pointers to enclosing loop variables
     - forbidigo # forbids identifiers
     - funlen # tool for detection of long functions
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)

Makefile

@@ -11,7 +11,7 @@ LDFLAGSSTRING +=-X main.Date=$(BUILD_TIME)
 LDFLAGSSTRING +=-X main.Version=$(GIT_TAG)
 LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 
-E2EFUZZTEST = FUZZ=true go test ./e2e -fuzz -v -fuzztime=15m
+E2EFUZZTEST = FUZZ=true go test ./e2e -fuzz -v -fuzztime=5m
 
 .PHONY: eigenda-proxy
 eigenda-proxy:
@@ -23,7 +23,7 @@ docker-build:
 	@docker build -t ghcr.io/layr-labs/eigenda-proxy:dev .
 
 run-memstore-server:
-	./bin/eigenda-proxy --memstore.enabled --eigenda.cert-verification-disabled --eigenda.eth-rpc http://localhost:8545 --eigenda.svc-manager-addr 0x123 --metrics.enabled
+	./bin/eigenda-proxy --memstore.enabled --metrics.enabled
 
 disperse-test-blob:
 	curl -X POST -d my-blob-content http://127.0.0.1:3100/put/

cmd/server/entrypoint.go

@@ -6,7 +6,9 @@ import (
 	"fmt"
 
 	proxy_logging "github.com/Layr-Labs/eigenda-proxy/logging"
+	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore/memconfig"
 	"github.com/Layr-Labs/eigensdk-go/logging"
+	"github.com/gorilla/mux"
 
 	"github.com/Layr-Labs/eigenda-proxy/flags"
 	"github.com/Layr-Labs/eigenda-proxy/metrics"
@@ -47,9 +49,15 @@ func StartProxySvr(cliCtx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to create store: %w", err)
 	}
+
 	server := server.NewServer(cliCtx.String(flags.ListenAddrFlagName), cliCtx.Int(flags.PortFlagName), sm, log, m)
+	r := mux.NewRouter()
+	server.RegisterRoutes(r)
+	if cfg.EigenDAConfig.MemstoreEnabled {
+		memconfig.NewHandlerHTTP(log, cfg.EigenDAConfig.MemstoreConfig).RegisterMemstoreConfigHandlers(r)
+	}
 
-	if err := server.Start(); err != nil {
+	if err := server.Start(r); err != nil {
 		return fmt.Errorf("failed to start the DA server: %w", err)
 	}
 

e2e/setup.go

@@ -12,14 +12,15 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/server"
 	"github.com/Layr-Labs/eigenda-proxy/store"
-	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
+	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore/memconfig"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/Layr-Labs/eigenda/encoding/kzg"
 	"github.com/Layr-Labs/eigensdk-go/logging"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"golang.org/x/exp/rand"
@@ -207,11 +208,10 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
 			},
 		},
 		MemstoreEnabled: testCfg.UseMemory,
-		MemstoreConfig: memstore.Config{
+		MemstoreConfig: memconfig.NewSafeConfig(memconfig.Config{
 			BlobExpiration:   testCfg.Expiration,
 			MaxBlobSizeBytes: maxBlobLengthBytes,
-		},
-
+		}),
 		StorageConfig: store.Config{
 			AsyncPutWorkers: testCfg.WriteThreadCount,
 		},
@@ -274,7 +274,9 @@ func CreateTestSuite(testSuiteCfg server.CLIConfig) (TestSuite, func()) {
 	proxySvr := server.NewServer(host, 0, sm, log, m)
 
 	log.Info("Starting proxy server...")
-	err = proxySvr.Start()
+	r := mux.NewRouter()
+	proxySvr.RegisterRoutes(r)
+	err = proxySvr.Start(r)
 	if err != nil {
 		panic(err)
 	}

server/config.go

@@ -9,13 +9,14 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/store"
 	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
+	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore/memconfig"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
 )
 
 type Config struct {
 	EdaClientConfig clients.EigenDAClientConfig
-	MemstoreConfig  memstore.Config
+	MemstoreConfig  *memconfig.SafeConfig
 	StorageConfig   store.Config
 	VerifierConfig  verify.Config
 	PutRetries      uint

server/config_test.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/Layr-Labs/eigenda-proxy/common"
-	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
+	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore/memconfig"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/Layr-Labs/eigenda/encoding/kzg"
@@ -42,9 +42,9 @@ func validCfg() *Config {
 			EthConfirmationDepth: 12,
 		},
 		MemstoreEnabled: true,
-		MemstoreConfig: memstore.Config{
+		MemstoreConfig: memconfig.NewSafeConfig(memconfig.Config{
 			BlobExpiration: 25 * time.Minute,
-		},
+		}),
 	}
 }
 

server/handlers_test.go

@@ -99,7 +99,7 @@ func TestHandlerGet(t *testing.T) {
 			r := mux.NewRouter()
 			// enable this logger to help debug tests
 			server := NewServer("localhost", 0, mockStorageMgr, testLogger, metrics.NoopMetrics)
-			server.registerRoutes(r)
+			server.RegisterRoutes(r)
 			r.ServeHTTP(rec, req)
 
 			require.Equal(t, tt.expectedCode, rec.Code)
@@ -170,7 +170,7 @@ func TestHandlerPutSuccess(t *testing.T) {
 			r := mux.NewRouter()
 			// enable this logger to help debug tests
 			server := NewServer("localhost", 0, mockStorageMgr, testLogger, metrics.NoopMetrics)
-			server.registerRoutes(r)
+			server.RegisterRoutes(r)
 			r.ServeHTTP(rec, req)
 
 			require.Equal(t, tt.expectedCode, rec.Code)
@@ -255,7 +255,7 @@ func TestHandlerPutErrors(t *testing.T) {
 				r := mux.NewRouter()
 				// enable this logger to help debug tests
 				server := NewServer("localhost", 0, mockStorageMgr, testLogger, metrics.NoopMetrics)
-				server.registerRoutes(r)
+				server.RegisterRoutes(r)
 				r.ServeHTTP(rec, req)
 
 				require.Equal(t, tt.expectedHTTPCode, rec.Code)

server/load_store.go

@@ -85,12 +85,6 @@ func LoadStoreManager(ctx context.Context, cfg CLIConfig, log logging.Logger, m
 		return nil, fmt.Errorf("failed to create verifier: %w", err)
 	}
 
-	if vCfg.VerifyCerts {
-		log.Info("Certificate verification with Ethereum enabled")
-	} else {
-		log.Warn("Verification disabled")
-	}
-
 	// create EigenDA backend store
 	var eigenDA common.GeneratedKeyStore
 	if cfg.EigenDAConfig.MemstoreEnabled {
@@ -109,7 +103,7 @@ func LoadStoreManager(ctx context.Context, cfg CLIConfig, log logging.Logger, m
 			verifier,
 			log,
 			&eigenda.StoreConfig{
-				MaxBlobSizeBytes:     cfg.EigenDAConfig.MemstoreConfig.MaxBlobSizeBytes,
+				MaxBlobSizeBytes:     cfg.EigenDAConfig.MemstoreConfig.MaxBlobSizeBytes(),
 				EthConfirmationDepth: cfg.EigenDAConfig.VerifierConfig.EthConfirmationDepth,
 				StatusQueryTimeout:   cfg.EigenDAConfig.EdaClientConfig.StatusQueryTimeout,
 				PutRetries:           cfg.EigenDAConfig.PutRetries,

server/routing.go

@@ -14,7 +14,7 @@ const (
 	routingVarNameCommitTypeByteHex = "commit_type_byte_hex"
 )
 
-func (svr *Server) registerRoutes(r *mux.Router) {
+func (svr *Server) RegisterRoutes(r *mux.Router) {
 	subrouterGET := r.Methods("GET").PathPrefix("/get").Subrouter()
 	// std commitments (for nitro)
 	subrouterGET.HandleFunc("/"+

server/routing_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/metrics"
 	"github.com/Layr-Labs/eigenda-proxy/mocks"
 	"github.com/golang/mock/gomock"
+	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/require"
 )
 
@@ -21,7 +22,8 @@ func TestRouting(t *testing.T) {
 
 	m := metrics.NewMetrics("default")
 	server := NewServer("localhost", 8080, mockRouter, testLogger, m)
-	err := server.Start()
+	r := mux.NewRouter()
+	err := server.Start(r)
 	require.NoError(t, err)
 
 	tests := []struct {

server/server.go

@@ -53,9 +53,7 @@ func NewServer(host string, port int, sm store.IManager, log logging.Logger,
 	}
 }
 
-func (svr *Server) Start() error {
-	r := mux.NewRouter()
-	svr.registerRoutes(r)
+func (svr *Server) Start(r *mux.Router) error {
 	svr.httpServer.Handler = r
 
 	listener, err := net.Listen("tcp", svr.endpoint)

store/generated_key/memstore/README.md

@@ -0,0 +1,49 @@
+# Memstore Backend
+
+The Memstore backend is a simple in-memory key-value store that is meant to replace a real EigenDA backend (talking to the disperser) for testing and development purposes. It is **never** recommended for production use.
+
+## Usage
+
+```bash
+./bin/eigenda-proxy --memstore.enabled
+```
+
+## Configuration
+
+See [memconfig/config.go](./memconfig/config.go) for the configuration options.
+These can all be set via their respective flags or environment variables. Run `./bin/eigenda-proxy --help | grep memstore` to see these.
+
+## Config REST API
+
+The Memstore backend also provides a REST API for changing the configuration at runtime. This is useful for testing different configurations without restarting the proxy.
+
+The API consists of GET and PATCH methods on the `/memstore/config` resource.
+
+### Get the current configuration
+
+```bash
+$ curl http://localhost:3100/memstore/config | jq
+{
+  "MaxBlobSizeBytes": 16777216,
+  "BlobExpiration": "25m0s",
+  "PutLatency": "0s",
+  "GetLatency": "0s",
+  "PutReturnsFailoverError": false
+}
+```
+
+### Set a configuration option
+
+The PATCH request allows to patch the configuration. This allows only sending a subset of the configuration options. The other fields will be left intact.
+
+```bash
+$ curl -X PATCH http://localhost:3100/memstore/config -d '{"PutReturnsFailoverError": true}'
+{"MaxBlobSizeBytes":16777216,"BlobExpiration":"25m0s","PutLatency":"0s","GetLatency":"0s","PutReturnsFailoverError":true}
+```
+
+One can of course still build a jq pipe to produce the same result (although still using PATCH instead of PUT since that is the only method available):
+```bash
+$ curl http://localhost:3100/memstore/config | \
+  jq '.PutLatency = "5s" | .GetLatency = "2s"' | \
+  curl -X PATCH http://localhost:3100/memstore/config -d @-
+```

store/generated_key/memstore/cli.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore/memconfig"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/urfave/cli/v2"
 )
@@ -98,16 +99,17 @@ func CLIFlags(envPrefix, category string) []cli.Flag {
 	}
 }
 
-func ReadConfig(ctx *cli.Context) Config {
-	return Config{
-		// TODO: there has to be a better way to get MaxBlobLengthBytes
-		// right now we get it from the verifier cli, but there's probably a way to share flags more nicely?
-		// maybe use a duplicate but hidden flag in memstore category, and set it using the action by reading
-		// from the other flag?
-		MaxBlobSizeBytes:        verify.MaxBlobLengthBytes,
-		BlobExpiration:          ctx.Duration(ExpirationFlagName),
-		PutLatency:              ctx.Duration(PutLatencyFlagName),
-		GetLatency:              ctx.Duration(GetLatencyFlagName),
-		PutReturnsFailoverError: ctx.Bool(PutReturnsFailoverErrorFlagName),
-	}
+func ReadConfig(ctx *cli.Context) *memconfig.SafeConfig {
+	return memconfig.NewSafeConfig(
+		memconfig.Config{
+			// TODO: there has to be a better way to get MaxBlobLengthBytes
+			// right now we get it from the verifier cli, but there's probably a way to share flags more nicely?
+			// maybe use a duplicate but hidden flag in memstore category, and set it using the action by reading
+			// from the other flag?
+			MaxBlobSizeBytes:        verify.MaxBlobLengthBytes,
+			BlobExpiration:          ctx.Duration(ExpirationFlagName),
+			PutLatency:              ctx.Duration(PutLatencyFlagName),
+			GetLatency:              ctx.Duration(GetLatencyFlagName),
+			PutReturnsFailoverError: ctx.Bool(PutReturnsFailoverErrorFlagName),
+		})
 }