From 45d128106f6de2a853fa92507a50c3fa80621b34 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 20 Feb 2024 16:29:15 -0800 Subject: [PATCH] Add a sequence number to advertisements The sequence number is optional to allow backwards compatability with previous existing advertisements that do not have a sequence number. --- ingest/schema/envelope.go | 21 ++++++++++++++++++--- ingest/schema/envelope_test.go | 3 ++- ingest/schema/schema.ipldsch | 2 ++ ingest/schema/types.go | 8 ++++++++ ingest/schema/types_test.go | 2 ++ 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/ingest/schema/envelope.go b/ingest/schema/envelope.go index 7ce3105..10a0017 100644 --- a/ingest/schema/envelope.go +++ b/ingest/schema/envelope.go @@ -12,6 +12,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/record" "github.com/multiformats/go-multihash" + "github.com/multiformats/go-varint" ) var log = logging.Logger("indexer/schema") @@ -93,9 +94,14 @@ func signaturePayload(ad *Advertisement, oldFormat bool) ([]byte, error) { addrsLen += len(addr) } + var seqLen int + if ad.SeqNum != nil { + seqLen = varint.UvarintSize(*ad.SeqNum) + } + // Signature data is previousID+entries+metadata+isRm var sigBuf bytes.Buffer - sigBuf.Grow(len(bindex) + len(ent) + len(ad.Provider) + addrsLen + len(ad.Metadata) + 1) + sigBuf.Grow(len(bindex) + len(ent) + len(ad.Provider) + addrsLen + len(ad.Metadata) + 1 + seqLen) sigBuf.Write(bindex) sigBuf.Write(ent) sigBuf.WriteString(ad.Provider) @@ -108,6 +114,9 @@ func signaturePayload(ad *Advertisement, oldFormat bool) ([]byte, error) { } else { sigBuf.WriteByte(0) } + if ad.SeqNum != nil { + sigBuf.Write(varint.ToUvarint(*ad.SeqNum)) + } // Generates the old (incorrect) data payload used for signature. This is // only for compatibility with existing advertisements that have the old @@ -147,8 +156,11 @@ func extendedProviderSignaturePayload(ad *Advertisement, p *Provider) ([]byte, e for _, addr := range p.Addresses { addrsLen += len(addr) } - - sigBuf.Grow(len(bindex) + len(ent) + len(ad.Provider) + len(ad.ContextID) + len(p.ID) + addrsLen + len(p.Metadata) + 1) + var seqLen int + if ad.SeqNum != nil { + seqLen = varint.UvarintSize(*ad.SeqNum) + } + sigBuf.Grow(len(bindex) + len(ent) + len(ad.Provider) + len(ad.ContextID) + len(p.ID) + addrsLen + len(p.Metadata) + 1 + seqLen) sigBuf.Write(bindex) sigBuf.Write(ent) sigBuf.WriteString(ad.Provider) @@ -163,6 +175,9 @@ func extendedProviderSignaturePayload(ad *Advertisement, p *Provider) ([]byte, e } else { sigBuf.WriteByte(0) } + if ad.SeqNum != nil { + sigBuf.Write(varint.ToUvarint(*ad.SeqNum)) + } return multihash.Sum(sigBuf.Bytes(), multihash.SHA2_256, -1) } diff --git a/ingest/schema/envelope_test.go b/ingest/schema/envelope_test.go index 8e24e1f..5b2e184 100644 --- a/ingest/schema/envelope_test.go +++ b/ingest/schema/envelope_test.go @@ -51,7 +51,7 @@ func testSignAndVerify(t *testing.T, signer func(*stischema.Advertisement, crypt require.NoError(t, err) elnk, err := lsys.Store(ipld.LinkContext{}, stischema.Linkproto, node) require.NoError(t, err) - + seq := uint64(123) adv := stischema.Advertisement{ Provider: "12D3KooWKRyzVWW6ChFjQjK4miCty85Niy48tpPV95XdKu1BcvMA", Addresses: []string{ @@ -60,6 +60,7 @@ func testSignAndVerify(t *testing.T, signer func(*stischema.Advertisement, crypt Entries: elnk, ContextID: []byte("test-context-id"), Metadata: []byte("test-metadata"), + SeqNum: &seq, } err = signer(&adv, priv) require.NoError(t, err) diff --git a/ingest/schema/schema.ipldsch b/ingest/schema/schema.ipldsch index 57b1825..ef573df 100644 --- a/ingest/schema/schema.ipldsch +++ b/ingest/schema/schema.ipldsch @@ -64,6 +64,8 @@ type Advertisement struct { Metadata Bytes # IsRm specifies whether this advertisement represents the content are no longer retrievalbe fom the provider. IsRm Bool + # SeqNum is a monotonically increasing sequence number that gives the advertisement's distance in the chain. + SeqNum optional Int # ExtendedProvider might optionally specify a set of providers where the ad entries are available from. # See: https://github.com/ipni/storetheindex/blob/main/doc/ingest.md#extendedprovider ExtendedProvider optional ExtendedProvider diff --git a/ingest/schema/types.go b/ingest/schema/types.go index bbfa213..5f61370 100644 --- a/ingest/schema/types.go +++ b/ingest/schema/types.go @@ -40,6 +40,7 @@ type ( ContextID []byte Metadata []byte IsRm bool + SeqNum *uint64 ExtendedProvider *ExtendedProvider } EntryChunk struct { @@ -100,6 +101,13 @@ func (a Advertisement) PreviousCid() cid.Cid { return a.PreviousID.(cidlink.Link).Cid } +func (a Advertisement) Sequence() uint64 { + if a.SeqNum == nil { + return 0 + } + return uint64(*a.SeqNum) +} + func (a Advertisement) Validate() error { if len(a.ContextID) > MaxContextIDLen { return errors.New("context id too long") diff --git a/ingest/schema/types_test.go b/ingest/schema/types_test.go index 3e6ced9..3e9e544 100644 --- a/ingest/schema/types_test.go +++ b/ingest/schema/types_test.go @@ -269,6 +269,7 @@ func Test_LinkLoadNoEntries(t *testing.T) { func generateAdvertisement() *stischema.Advertisement { mhs := test.RandomMultihashes(7) prev := ipld.Link(cidlink.Link{Cid: cid.NewCidV1(cid.Raw, mhs[0])}) + seq := uint64(54321) return &stischema.Advertisement{ PreviousID: prev, Provider: mhs[1].String(), @@ -280,6 +281,7 @@ func generateAdvertisement() *stischema.Advertisement { Metadata: mhs[5], Signature: mhs[6], IsRm: false, + SeqNum: &seq, } }