From 2a930729222c00e51e4a97bc47621618f998e013 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 24 Oct 2024 12:57:28 -0400 Subject: [PATCH] Add additional context to the max size errors received occasionally on backup --- internal/cmd/backup.go | 38 ++++++++++++++++++++++-- internal/cmd/backup_test.go | 58 +++++++++++++++++++++++++++++++++++-- internal/cmd/cmd.go | 2 +- 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/internal/cmd/backup.go b/internal/cmd/backup.go index 95bf910..92e0d38 100644 --- a/internal/cmd/backup.go +++ b/internal/cmd/backup.go @@ -7,6 +7,7 @@ import ( "io" "os" "regexp" + "strconv" "strings" "time" @@ -22,6 +23,8 @@ import ( "github.com/spf13/cobra" "golang.org/x/exp/constraints" "golang.org/x/exp/maps" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/authzed/zed/internal/client" "github.com/authzed/zed/internal/commands" @@ -249,7 +252,7 @@ func backupCreateCmdFunc(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() schemaResp, err := c.ReadSchema(ctx, &v1.ReadSchemaRequest{}) if err != nil { - return fmt.Errorf("error reading schema: %w", err) + return fmt.Errorf("error reading schema: %w", addSizeErrInfo(err)) } else if schemaResp.ReadAt == nil { return fmt.Errorf("`backup` is not supported on this version of SpiceDB") } @@ -284,7 +287,7 @@ func backupCreateCmdFunc(cmd *cobra.Command, args []string) (err error) { }, }) if err != nil { - return fmt.Errorf("error exporting relationships: %w", err) + return fmt.Errorf("error exporting relationships: %w", addSizeErrInfo(err)) } relationshipReadStart := time.Now() @@ -299,7 +302,7 @@ func backupCreateCmdFunc(cmd *cobra.Command, args []string) (err error) { relsResp, err := relationshipStream.Recv() if err != nil { if !errors.Is(err, io.EOF) { - return fmt.Errorf("error receiving relationships: %w", err) + return fmt.Errorf("error receiving relationships: %w", addSizeErrInfo(err)) } break } @@ -619,3 +622,32 @@ func rewriteLegacy(schema string) string { schema = string(missingAllowedTypes.ReplaceAll([]byte(schema), []byte("\n/* deleted missing allowed type error */"))) return string(shortRelations.ReplaceAll([]byte(schema), []byte("\n/* deleted short relation name */"))) } + +var sizeErrorRegEx = regexp.MustCompile(`received message larger than max \((\d+) vs. (\d+)\)`) + +func addSizeErrInfo(err error) error { + if err == nil { + return nil + } + + code := status.Code(err) + if code != codes.ResourceExhausted { + return err + } + + if !strings.Contains(err.Error(), "received message larger than max") { + return err + } + + matches := sizeErrorRegEx.FindStringSubmatch(err.Error()) + if len(matches) != 3 { + return fmt.Errorf("%w: set flag --max-message-size=bytecounthere to increase the maximum allowable size", err) + } + + necessaryByteCount, err := strconv.Atoi(matches[1]) + if err != nil { + return fmt.Errorf("%w: set flag --max-message-size=bytecounthere to increase the maximum allowable size", err) + } + + return fmt.Errorf("%w: set flag --max-message-size=%d to increase the maximum allowable size", err, 2*necessaryByteCount) +} diff --git a/internal/cmd/backup_test.go b/internal/cmd/backup_test.go index 41cc77b..746aead 100644 --- a/internal/cmd/backup_test.go +++ b/internal/cmd/backup_test.go @@ -2,19 +2,23 @@ package cmd import ( "context" + "errors" "os" "path/filepath" "strings" "testing" - "github.com/authzed/zed/internal/client" - zedtesting "github.com/authzed/zed/internal/testing" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" "github.com/authzed/spicedb/pkg/tuple" "github.com/google/uuid" "github.com/rs/zerolog" "github.com/stretchr/testify/require" + + "github.com/authzed/zed/internal/client" + zedtesting "github.com/authzed/zed/internal/testing" ) func init() { @@ -379,3 +383,53 @@ func TestBackupRestoreCmdFunc(t *testing.T) { require.NoError(t, rrCli.CloseSend()) require.Equal(t, "test/resource:1#reader@test/user:1", tuple.MustStringRelationship(rrResp.Relationship)) } + +func TestAddSizeErrInfo(t *testing.T) { + tcs := []struct { + name string + err error + expectedError string + }{ + { + name: "error is nil", + err: nil, + expectedError: "", + }, + { + name: "error is not a size error", + err: errors.New("some error"), + expectedError: "some error", + }, + { + name: "error has correct code, wrong message", + err: status.New(codes.ResourceExhausted, "foobar").Err(), + expectedError: "foobar", + }, + { + name: "error has correct message, wrong code", + err: status.New(codes.Unauthenticated, "received message larger than max").Err(), + expectedError: "received message larger than max", + }, + { + name: "error has correct code and message", + err: status.New(codes.ResourceExhausted, "received message larger than max").Err(), + expectedError: "set flag --max-message-size=bytecounthere", + }, + { + name: "error has correct code and message with additional info", + err: status.New(codes.ResourceExhausted, "received message larger than max (1234 vs. 45)").Err(), + expectedError: "set flag --max-message-size=2468", + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + err := addSizeErrInfo(tc.err) + if tc.expectedError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.expectedError) + } + }) + } +} diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 2f55784..c526f8d 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -75,7 +75,7 @@ func Run() { rootCmd.PersistentFlags().Bool("no-verify-ca", false, "do not attempt to verify the server's certificate chain and host name") rootCmd.PersistentFlags().Bool("debug", false, "enable debug logging") rootCmd.PersistentFlags().String("request-id", "", "optional id to send along with SpiceDB requests for tracing") - rootCmd.PersistentFlags().Int("max-message-size", 0, "maximum size in bytes (defaults to 4mb) of a gRPC message that can be sent or received by zed") + rootCmd.PersistentFlags().Int("max-message-size", 0, "maximum size *in bytes* (defaults to 4_194_304 bytes ~= 4MB) of a gRPC message that can be sent or received by zed") _ = rootCmd.PersistentFlags().MarkHidden("debug") // This cannot return its error. versionCmd := &cobra.Command{