From b54a1ac01ddd1bf413997842f2081ff359c9645c Mon Sep 17 00:00:00 2001 From: iurii Date: Tue, 21 Jan 2025 21:56:36 +0200 Subject: [PATCH] tmp commit to fetch(dump) some shares in GOB format --- go.mod | 5 +- go.sum | 5 + migrations/migration_5_share_gob_to_ssz.go | 142 +++++++++++++++++++-- 3 files changed, 142 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 3e58ac2202..c480f86909 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,10 @@ require ( tailscale.com v1.72.0 ) -require github.com/felixge/httpsnoop v1.0.4 // indirect +require ( + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/sanity-io/litter v1.5.6 // indirect +) require ( github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect diff --git a/go.sum b/go.sum index 5ead0985b9..9179cffd7b 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,7 @@ github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXk github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -619,6 +620,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -683,6 +685,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sanity-io/litter v1.5.6 h1:hCFycYzhRnW4niFbbmR7QKdmds69PbVa/sNmEN5euSU= +github.com/sanity-io/litter v1.5.6/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= @@ -745,6 +749,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/migrations/migration_5_share_gob_to_ssz.go b/migrations/migration_5_share_gob_to_ssz.go index b10162975c..a8d8d7a282 100644 --- a/migrations/migration_5_share_gob_to_ssz.go +++ b/migrations/migration_5_share_gob_to_ssz.go @@ -1,8 +1,12 @@ package migrations import ( + "bytes" "context" + "encoding/hex" "fmt" + "github.com/sanity-io/litter" + "github.com/ssvlabs/ssv-spec/types" "github.com/ssvlabs/ssv/storage/basedb" "go.uber.org/zap" ) @@ -16,17 +20,35 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{ // storagePrefix is a base prefix we use when storing shares var storagePrefix = []byte("operator/") - // sets is a bunch of updates this migration will need to perform, we cannot do them all in a - // single transaction (because there is a limit on how large a single transaction can be) so - // we'll use SetMany func that will split up the data we want to update into batches committing + // sharesSSZEncoded is a bunch of updates this migration will need to perform, we cannot do them + // all in a single transaction (because there is a limit on how large a single transaction can be) + // so we'll use SetMany func that will split up the data we want to update into batches committing // each batch in a separate transaction. I guess that makes this migration non-atomic. - sets := make([]basedb.Obj, 0) + sharesSSZEncoded := make([]basedb.Obj, 0) + // TODO + shareIDTesting := "" + + //sharesGOB := make(map[string]*storageShareGOB) + sharesGOB := make(map[string][]*storageShareGOB) err := opt.Db.GetAll(append(storagePrefix, sharesPrefixGOB...), func(i int, obj basedb.Obj) error { shareGOB := &storageShareGOB{} if err := shareGOB.Decode(obj.Value); err != nil { return fmt.Errorf("decode gob share: %w", err) } + // TODO - using 0 for now to see how the real data looks like, + // but that will likely result into collisions + sID := shareID(shareGOB.ValidatorPubKey, 0) + if shareIDTesting == "" { + shareIDTesting = sID + } + if sID == shareIDTesting { + litter.Dump(shareGOB) + litter.Dump(obj.Value) + litter.Dump(hex.EncodeToString(obj.Value)) + } + //sID := shareID(shareGOB.ValidatorPubKey, shareGOB.OperatorID) + sharesGOB[sID] = append(sharesGOB[sID], shareGOB) share, err := storageShareGOBToSpecShare(shareGOB) if err != nil { return fmt.Errorf("convert storage share to spec share: %w", err) @@ -37,7 +59,7 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{ if err != nil { return fmt.Errorf("encode ssz share: %w", err) } - sets = append(sets, basedb.Obj{ + sharesSSZEncoded = append(sharesSSZEncoded, basedb.Obj{ Key: key, Value: value, }) @@ -47,14 +69,40 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{ return fmt.Errorf("GetAll: %w", err) } - if err := opt.Db.SetMany(storagePrefix, len(sets), func(i int) (basedb.Obj, error) { - return sets[i], nil + if err := opt.Db.SetMany(storagePrefix, len(sharesSSZEncoded), func(i int) (basedb.Obj, error) { + return sharesSSZEncoded[i], nil }); err != nil { return fmt.Errorf("SetMany: %w", err) } - // TODO - do not complete this migration for now, we'll complete it once - // additional sanity-checks are added here + totalGOB := len(sharesGOB) + totalSSZ := 0 + if err := opt.Db.GetAll(append(storagePrefix, sharesPrefixSSZ...), func(i int, obj basedb.Obj) error { + shareSSZ := &storageShareSSZ{} + err := shareSSZ.Decode(obj.Value) + if err != nil { + return fmt.Errorf("decode ssz share: %w", err) + } + // TODO this check is disabled for now until we know what to use as shareID + //sID := shareID(shareSSZ.ValidatorPubKey, shareSSZ.OperatorID) + //shareGOB := sharesGOB[sID] + //if !matchGOBvsSSZ(shareGOB, shareSSZ) { + // return fmt.Errorf( + // "GOB share doesn't match corresponding SSZ share, GOB: %s, SSZ: %s", + // litter.Sdump(shareGOB), + // litter.Sdump(shareSSZ), + // ) + //} + totalSSZ++ + return nil + }); err != nil { + return fmt.Errorf("GetMany: %w", err) + } + + if totalSSZ != totalGOB { + return fmt.Errorf("total SSZ shares count %d doesn't match GOB shares count %d", totalSSZ, totalGOB) + } + //if err := opt.Db.DropPrefix(append(storagePrefix, sharesPrefixGOB...)); err != nil { // return fmt.Errorf("DropPrefix: %w", err) //} @@ -67,3 +115,79 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{ return nil }, } + +func shareID(validatorPubkey []byte, operatorID types.OperatorID) string { + // TODO is this the correct way to uniquely identify share ? + return fmt.Sprintf("%s_%d", validatorPubkey, operatorID) +} + +func matchGOBvsSSZ(shareGOB *storageShareGOB, shareSSZ *storageShareSSZ) bool { + // note, ssz share no longer has OperatorID field + + if !bytes.Equal(shareGOB.ValidatorPubKey, shareSSZ.ValidatorPubKey) { + return false + } + if !bytes.Equal(shareGOB.SharePubKey, shareSSZ.SharePubKey) { + return false + } + if len(shareGOB.Committee) != len(shareSSZ.Committee) { + return false + } + for i, committeeGOB := range shareGOB.Committee { + committeeSSZ := shareSSZ.Committee[i] + if committeeGOB.OperatorID != committeeSSZ.OperatorID { + return false + } + if !bytes.Equal(committeeGOB.PubKey, committeeSSZ.PubKey) { + return false + } + } + if shareGOB.Quorum != shareSSZ.Quorum { + return false + } + if shareGOB.PartialQuorum != shareSSZ.PartialQuorum { + return false + } + if shareGOB.DomainType != shareSSZ.DomainType { + return false + } + if shareGOB.FeeRecipientAddress != shareSSZ.FeeRecipientAddress { + return false + } + if !bytes.Equal(shareGOB.Graffiti, shareSSZ.Graffiti) { + return false + } + + if shareGOB.OwnerAddress != shareSSZ.OwnerAddress { + return false + } + if shareGOB.Liquidated != shareSSZ.Liquidated { + return false + } + + // finally, check Beacon metadata matches + if shareGOB.BeaconMetadata == nil { + if shareSSZ.ValidatorIndex != 0 { + return false + } + if shareSSZ.Status != 0 { + return false + } + if shareSSZ.ActivationEpoch != 0 { + return false + } + // note, ssz share no longer has Balance field + return true + } + if uint64(shareGOB.BeaconMetadata.Index) != shareSSZ.ValidatorIndex { + return false + } + if int(shareGOB.BeaconMetadata.Status) != int(shareSSZ.Status) { + return false + } + if uint64(shareGOB.BeaconMetadata.ActivationEpoch) != shareSSZ.ActivationEpoch { + return false + } + + return true +}