tmp commit to fetch(dump) some shares in GOB format

go.mod

@@ -56,7 +56,10 @@ require (
 	tailscale.com v1.72.0
 )
 
-require github.com/felixge/httpsnoop v1.0.4 // indirect
+require (
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/sanity-io/litter v1.5.6 // indirect
+)
 
 require (
 	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect

go.sum

@@ -108,6 +108,7 @@ github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXk
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
 github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
+github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -619,6 +620,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -683,6 +685,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sanity-io/litter v1.5.6 h1:hCFycYzhRnW4niFbbmR7QKdmds69PbVa/sNmEN5euSU=
+github.com/sanity-io/litter v1.5.6/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
@@ -745,6 +749,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

migrations/migration_5_share_gob_to_ssz.go

@@ -1,8 +1,12 @@
 package migrations
 
 import (
+	"bytes"
 	"context"
+	"encoding/hex"
 	"fmt"
+	"github.com/sanity-io/litter"
+	"github.com/ssvlabs/ssv-spec/types"
 	"github.com/ssvlabs/ssv/storage/basedb"
 	"go.uber.org/zap"
 )
@@ -16,17 +20,35 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{
 		// storagePrefix is a base prefix we use when storing shares
 		var storagePrefix = []byte("operator/")
 
-		// sets is a bunch of updates this migration will need to perform, we cannot do them all in a
-		// single transaction (because there is a limit on how large a single transaction can be) so
-		// we'll use SetMany func that will split up the data we want to update into batches committing
+		// sharesSSZEncoded is a bunch of updates this migration will need to perform, we cannot do them
+		// all in a single transaction (because there is a limit on how large a single transaction can be)
+		// so we'll use SetMany func that will split up the data we want to update into batches committing
 		// each batch in a separate transaction. I guess that makes this migration non-atomic.
-		sets := make([]basedb.Obj, 0)
+		sharesSSZEncoded := make([]basedb.Obj, 0)
 
+		// TODO
+		shareIDTesting := ""
+
+		//sharesGOB := make(map[string]*storageShareGOB)
+		sharesGOB := make(map[string][]*storageShareGOB)
 		err := opt.Db.GetAll(append(storagePrefix, sharesPrefixGOB...), func(i int, obj basedb.Obj) error {
 			shareGOB := &storageShareGOB{}
 			if err := shareGOB.Decode(obj.Value); err != nil {
 				return fmt.Errorf("decode gob share: %w", err)
 			}
+			// TODO - using 0 for now to see how the real data looks like,
+			// but that will likely result into collisions
+			sID := shareID(shareGOB.ValidatorPubKey, 0)
+			if shareIDTesting == "" {
+				shareIDTesting = sID
+			}
+			if sID == shareIDTesting {
+				litter.Dump(shareGOB)
+				litter.Dump(obj.Value)
+				litter.Dump(hex.EncodeToString(obj.Value))
+			}
+			//sID := shareID(shareGOB.ValidatorPubKey, shareGOB.OperatorID)
+			sharesGOB[sID] = append(sharesGOB[sID], shareGOB)
 			share, err := storageShareGOBToSpecShare(shareGOB)
 			if err != nil {
 				return fmt.Errorf("convert storage share to spec share: %w", err)
@@ -37,7 +59,7 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{
 			if err != nil {
 				return fmt.Errorf("encode ssz share: %w", err)
 			}
-			sets = append(sets, basedb.Obj{
+			sharesSSZEncoded = append(sharesSSZEncoded, basedb.Obj{
 				Key:   key,
 				Value: value,
 			})
@@ -47,14 +69,40 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{
 			return fmt.Errorf("GetAll: %w", err)
 		}
 
-		if err := opt.Db.SetMany(storagePrefix, len(sets), func(i int) (basedb.Obj, error) {
-			return sets[i], nil
+		if err := opt.Db.SetMany(storagePrefix, len(sharesSSZEncoded), func(i int) (basedb.Obj, error) {
+			return sharesSSZEncoded[i], nil
 		}); err != nil {
 			return fmt.Errorf("SetMany: %w", err)
 		}
 
-		// TODO - do not complete this migration for now, we'll complete it once
-		// additional sanity-checks are added here
+		totalGOB := len(sharesGOB)
+		totalSSZ := 0
+		if err := opt.Db.GetAll(append(storagePrefix, sharesPrefixSSZ...), func(i int, obj basedb.Obj) error {
+			shareSSZ := &storageShareSSZ{}
+			err := shareSSZ.Decode(obj.Value)
+			if err != nil {
+				return fmt.Errorf("decode ssz share: %w", err)
+			}
+			// TODO this check is disabled for now until we know what to use as shareID
+			//sID := shareID(shareSSZ.ValidatorPubKey, shareSSZ.OperatorID)
+			//shareGOB := sharesGOB[sID]
+			//if !matchGOBvsSSZ(shareGOB, shareSSZ) {
+			//	return fmt.Errorf(
+			//		"GOB share doesn't match corresponding SSZ share, GOB: %s, SSZ: %s",
+			//		litter.Sdump(shareGOB),
+			//		litter.Sdump(shareSSZ),
+			//	)
+			//}
+			totalSSZ++
+			return nil
+		}); err != nil {
+			return fmt.Errorf("GetMany: %w", err)
+		}
+
+		if totalSSZ != totalGOB {
+			return fmt.Errorf("total SSZ shares count %d doesn't match GOB shares count %d", totalSSZ, totalGOB)
+		}
+
 		//if err := opt.Db.DropPrefix(append(storagePrefix, sharesPrefixGOB...)); err != nil {
 		//	return fmt.Errorf("DropPrefix: %w", err)
 		//}
@@ -67,3 +115,79 @@ var migration_5_change_share_format_from_gob_to_ssz = Migration{
 		return nil
 	},
 }
+
+func shareID(validatorPubkey []byte, operatorID types.OperatorID) string {
+	// TODO is this the correct way to uniquely identify share ?
+	return fmt.Sprintf("%s_%d", validatorPubkey, operatorID)
+}
+
+func matchGOBvsSSZ(shareGOB *storageShareGOB, shareSSZ *storageShareSSZ) bool {
+	// note, ssz share no longer has OperatorID field
+
+	if !bytes.Equal(shareGOB.ValidatorPubKey, shareSSZ.ValidatorPubKey) {
+		return false
+	}
+	if !bytes.Equal(shareGOB.SharePubKey, shareSSZ.SharePubKey) {
+		return false
+	}
+	if len(shareGOB.Committee) != len(shareSSZ.Committee) {
+		return false
+	}
+	for i, committeeGOB := range shareGOB.Committee {
+		committeeSSZ := shareSSZ.Committee[i]
+		if committeeGOB.OperatorID != committeeSSZ.OperatorID {
+			return false
+		}
+		if !bytes.Equal(committeeGOB.PubKey, committeeSSZ.PubKey) {
+			return false
+		}
+	}
+	if shareGOB.Quorum != shareSSZ.Quorum {
+		return false
+	}
+	if shareGOB.PartialQuorum != shareSSZ.PartialQuorum {
+		return false
+	}
+	if shareGOB.DomainType != shareSSZ.DomainType {
+		return false
+	}
+	if shareGOB.FeeRecipientAddress != shareSSZ.FeeRecipientAddress {
+		return false
+	}
+	if !bytes.Equal(shareGOB.Graffiti, shareSSZ.Graffiti) {
+		return false
+	}
+
+	if shareGOB.OwnerAddress != shareSSZ.OwnerAddress {
+		return false
+	}
+	if shareGOB.Liquidated != shareSSZ.Liquidated {
+		return false
+	}
+
+	// finally, check Beacon metadata matches
+	if shareGOB.BeaconMetadata == nil {
+		if shareSSZ.ValidatorIndex != 0 {
+			return false
+		}
+		if shareSSZ.Status != 0 {
+			return false
+		}
+		if shareSSZ.ActivationEpoch != 0 {
+			return false
+		}
+		// note, ssz share no longer has Balance field
+		return true
+	}
+	if uint64(shareGOB.BeaconMetadata.Index) != shareSSZ.ValidatorIndex {
+		return false
+	}
+	if int(shareGOB.BeaconMetadata.Status) != int(shareSSZ.Status) {
+		return false
+	}
+	if uint64(shareGOB.BeaconMetadata.ActivationEpoch) != shareSSZ.ActivationEpoch {
+		return false
+	}
+
+	return true
+}