Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(shwap): Update availability to work with shwap #3600

Merged
merged 4 commits into from
Aug 1, 2024
Merged

feat(shwap): Update availability to work with shwap #3600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2e05c59
shwap availability integration
walldiss Jul 25, 2024
579ca32
Merge branch 'shwap' into shwap-availability
walldiss Jul 31, 2024
3ae0db2
review
walldiss Aug 1, 2024
fd7f04c
wrap non-nil
walldiss Aug 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 15 additions & 16 deletions share/availability/full/availability.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,13 @@ import (
"errors"
"fmt"

"github.com/filecoin-project/dagstore"
logging "github.com/ipfs/go-log/v2"

"github.com/celestiaorg/celestia-node/header"
"github.com/celestiaorg/celestia-node/share"
"github.com/celestiaorg/celestia-node/share/eds"
"github.com/celestiaorg/celestia-node/share/eds/byzantine"
"github.com/celestiaorg/celestia-node/share/ipld"
"github.com/celestiaorg/celestia-node/share/shwap"
"github.com/celestiaorg/celestia-node/store"
)

var log = logging.Logger("share/full")
Expand All @@ -21,14 +20,14 @@ var log = logging.Logger("share/full")
// recovery technique. It is considered "full" because it is required
// to download enough shares to fully reconstruct the data square.
type ShareAvailability struct {
store *eds.Store
getter share.Getter
store *store.Store
getter shwap.Getter
}

// NewShareAvailability creates a new full ShareAvailability.
func NewShareAvailability(
store *eds.Store,
getter share.Getter,
store *store.Store,
getter shwap.Getter,
) *ShareAvailability {
return &ShareAvailability{
store: store,
Expand All @@ -40,8 +39,12 @@ func NewShareAvailability(
// enough Shares from the network.
func (fa *ShareAvailability) SharesAvailable(ctx context.Context, header *header.ExtendedHeader) error {
dah := header.DAH
// short-circuit if the given root is an empty data square, to avoid datastore hit
// if the data square is empty, we can safely link the header height in the store to an empty EDS.
if share.DataHash(dah.Hash()).IsEmptyEDS() {
err := fa.store.Put(ctx, dah, header.Height(), share.EmptyEDS())
if err != nil {
return fmt.Errorf("put empty EDS: %w", err)
}
return nil
}

Expand All @@ -54,29 +57,25 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, header *header
}

// a hack to avoid loading the whole EDS in mem if we store it already.
if ok, _ := fa.store.Has(ctx, dah.Hash()); ok {
if ok, _ := fa.store.HasByHeight(ctx, header.Height()); ok {
return nil
}

adder := ipld.NewProofsAdder(len(dah.RowRoots), false)
ctx = ipld.CtxWithProofsAdder(ctx, adder)
defer adder.Purge()

eds, err := fa.getter.GetEDS(ctx, header)
if err != nil {
if errors.Is(err, context.Canceled) {
return err
}
log.Errorw("availability validation failed", "root", dah.String(), "err", err.Error())
var byzantineErr *byzantine.ErrByzantine
if errors.Is(err, share.ErrNotFound) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) {
if errors.Is(err, shwap.ErrNotFound) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) {
return share.ErrNotAvailable
}
return err
}

err = fa.store.Put(ctx, dah.Hash(), eds)
if err != nil && !errors.Is(err, dagstore.ErrShardExists) {
err = fa.store.Put(ctx, dah, header.Height(), eds)
if err != nil {
return fmt.Errorf("full availability: failed to store eds: %w", err)
}
return nil
Expand Down
90 changes: 52 additions & 38 deletions share/availability/full/availability_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,75 +3,89 @@ package full
import (
"context"
"testing"
"time"

"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/celestiaorg/celestia-node/header/headertest"
"github.com/celestiaorg/celestia-node/share"
availability_test "github.com/celestiaorg/celestia-node/share/availability/test"
"github.com/celestiaorg/celestia-node/share/eds/edstest"
"github.com/celestiaorg/celestia-node/share/mocks"
"github.com/celestiaorg/celestia-node/share/shwap"
"github.com/celestiaorg/celestia-node/share/shwap/getters/mock"
"github.com/celestiaorg/celestia-node/store"
)

func TestShareAvailableOverMocknet_Full(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
func TestSharesAvailable(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()

net := availability_test.NewTestDAGNet(ctx, t)
_, root := RandNode(net, 32)

eh := headertest.RandExtendedHeaderWithRoot(t, root)
nd := Node(net)
net.ConnectAll()
// RandServiceWithSquare creates a NewShareAvailability inside, so we can test it
eds := edstest.RandEDS(t, 16)
roots, err := share.NewAxisRoots(eds)
eh := headertest.RandExtendedHeaderWithRoot(t, roots)

err := nd.SharesAvailable(ctx, eh)
assert.NoError(t, err)
}
getter := mock.NewMockGetter(gomock.NewController(t))
getter.EXPECT().GetEDS(gomock.Any(), eh).Return(eds, nil)

func TestSharesAvailable_Full(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := store.NewStore(store.DefaultParameters(), t.TempDir())
avail := NewShareAvailability(store, getter)
err = avail.SharesAvailable(ctx, eh)
require.NoError(t, err)

// RandServiceWithSquare creates a NewShareAvailability inside, so we can test it
getter, roots := GetterWithRandSquare(t, 16)
// Check if the store has the root
has, err := store.HasByHash(ctx, roots.Hash())
require.NoError(t, err)
require.True(t, has)

eh := headertest.RandExtendedHeaderWithRoot(t, roots)
avail := TestAvailability(t, getter)
err := avail.SharesAvailable(ctx, eh)
assert.NoError(t, err)
// Check if the store has the root linked to the height
has, err = store.HasByHeight(ctx, eh.Height())
require.NoError(t, err)
require.True(t, has)
}

func TestSharesAvailable_StoresToEDSStore(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
func TestSharesAvailable_StoredEds(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()

// RandServiceWithSquare creates a NewShareAvailability inside, so we can test it
getter, roots := GetterWithRandSquare(t, 16)
eds := edstest.RandEDS(t, 4)
roots, err := share.NewAxisRoots(eds)
eh := headertest.RandExtendedHeaderWithRoot(t, roots)
avail := TestAvailability(t, getter)
err := avail.SharesAvailable(ctx, eh)
assert.NoError(t, err)
require.NoError(t, err)

store, err := store.NewStore(store.DefaultParameters(), t.TempDir())
avail := NewShareAvailability(store, nil)

err = store.Put(ctx, roots, eh.Height(), eds)
require.NoError(t, err)

has, err := avail.store.Has(ctx, roots.Hash())
assert.NoError(t, err)
assert.True(t, has)
has, err := store.HasByHeight(ctx, eh.Height())
require.NoError(t, err)
require.True(t, has)

err = avail.SharesAvailable(ctx, eh)
require.NoError(t, err)

has, err = store.HasByHeight(ctx, eh.Height())
require.NoError(t, err)
require.True(t, has)
}

func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) {
func TestSharesAvailable_ErrNotAvailable(t *testing.T) {
ctrl := gomock.NewController(t)
getter := mocks.NewMockGetter(ctrl)
ctx, cancel := context.WithCancel(context.Background())
getter := mock.NewMockGetter(ctrl)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()

eds := edstest.RandEDS(t, 4)
roots, err := share.NewAxisRoots(eds)
eh := headertest.RandExtendedHeaderWithRoot(t, roots)
require.NoError(t, err)
avail := TestAvailability(t, getter)

errors := []error{share.ErrNotFound, context.DeadlineExceeded}
store, err := store.NewStore(store.DefaultParameters(), t.TempDir())
avail := NewShareAvailability(store, getter)

errors := []error{shwap.ErrNotFound, context.DeadlineExceeded}
for _, getterErr := range errors {
getter.EXPECT().GetEDS(gomock.Any(), gomock.Any()).Return(nil, getterErr)
err := avail.SharesAvailable(ctx, eh)
Expand Down
Loading
Loading