diff --git a/vms/proposervm/post_fork_block.go b/vms/proposervm/post_fork_block.go index 707b6dc327c7..2406a974a963 100644 --- a/vms/proposervm/post_fork_block.go +++ b/vms/proposervm/post_fork_block.go @@ -33,7 +33,12 @@ func (b *postForkBlock) Accept(ctx context.Context) error { func (b *postForkBlock) acceptOuterBlk() error { // Update in-memory references b.status = choices.Accepted - b.vm.lastAcceptedTime = b.Timestamp() + + // following backfill introduction we may store past + // blocks, hence safeguard lastAcceptedTime + if b.Timestamp().After(b.vm.lastAcceptedTime) { + b.vm.lastAcceptedTime = b.Timestamp() + } return b.vm.acceptPostForkBlock(b) } diff --git a/vms/proposervm/state/block_height_index.go b/vms/proposervm/state/block_height_index.go index b60fca0c363d..3005fdefff11 100644 --- a/vms/proposervm/state/block_height_index.go +++ b/vms/proposervm/state/block_height_index.go @@ -19,8 +19,9 @@ var ( heightPrefix = []byte("height") metadataPrefix = []byte("metadata") - forkKey = []byte("fork") - checkpointKey = []byte("checkpoint") + forkKey = []byte("fork") + checkpointKey = []byte("checkpoint") + latestBackfilledKey = []byte("latestBackfilled") ) type HeightIndexGetter interface { @@ -32,12 +33,16 @@ type HeightIndexGetter interface { // Fork height is stored when the first post-fork block/option is accepted. // Before that, fork height won't be found. GetForkHeight() (uint64, error) + + GetLastBackfilledBlkID() (ids.ID, error) } type HeightIndexWriter interface { SetForkHeight(height uint64) error SetBlockIDAtHeight(height uint64, blkID ids.ID) error DeleteBlockIDAtHeight(height uint64) error + + SetLastBackfilledBlkID(blkID ids.ID) error } // A checkpoint is the blockID of the next block to be considered @@ -139,3 +144,11 @@ func (hi *heightIndex) SetCheckpoint(blkID ids.ID) error { func (hi *heightIndex) DeleteCheckpoint() error { return hi.metadataDB.Delete(checkpointKey) } + +func (hi *heightIndex) SetLastBackfilledBlkID(blkID ids.ID) error { + return database.PutID(hi.metadataDB, latestBackfilledKey, blkID) +} + +func (hi *heightIndex) GetLastBackfilledBlkID() (ids.ID, error) { + return database.GetID(hi.metadataDB, latestBackfilledKey) +} diff --git a/vms/proposervm/state/mock_state.go b/vms/proposervm/state/mock_state.go index fcd266f2d790..6afc847b960e 100644 --- a/vms/proposervm/state/mock_state.go +++ b/vms/proposervm/state/mock_state.go @@ -182,6 +182,21 @@ func (mr *MockStateMockRecorder) GetLastAccepted() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastAccepted", reflect.TypeOf((*MockState)(nil).GetLastAccepted)) } +// GetLastBackfilledBlkID mocks base method. +func (m *MockState) GetLastBackfilledBlkID() (ids.ID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLastBackfilledBlkID") + ret0, _ := ret[0].(ids.ID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLastBackfilledBlkID indicates an expected call of GetLastBackfilledBlkID. +func (mr *MockStateMockRecorder) GetLastBackfilledBlkID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastBackfilledBlkID", reflect.TypeOf((*MockState)(nil).GetLastBackfilledBlkID)) +} + // GetMinimumHeight mocks base method. func (m *MockState) GetMinimumHeight() (uint64, error) { m.ctrl.T.Helper() @@ -266,3 +281,17 @@ func (mr *MockStateMockRecorder) SetLastAccepted(arg0 interface{}) *gomock.Call mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastAccepted", reflect.TypeOf((*MockState)(nil).SetLastAccepted), arg0) } + +// SetLastBackfilledBlkID mocks base method. +func (m *MockState) SetLastBackfilledBlkID(arg0 ids.ID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLastBackfilledBlkID", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLastBackfilledBlkID indicates an expected call of SetLastBackfilledBlkID. +func (mr *MockStateMockRecorder) SetLastBackfilledBlkID(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastBackfilledBlkID", reflect.TypeOf((*MockState)(nil).SetLastBackfilledBlkID), arg0) +} diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go new file mode 100644 index 000000000000..73c3fb8d74f5 --- /dev/null +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -0,0 +1,1108 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package proposervm + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/memdb" + "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow" + "github.com/ava-labs/avalanchego/snow/choices" + "github.com/ava-labs/avalanchego/snow/consensus/snowman" + "github.com/ava-labs/avalanchego/snow/engine/common" + "github.com/ava-labs/avalanchego/snow/engine/snowman/block" + "github.com/ava-labs/avalanchego/snow/snowtest" + "github.com/ava-labs/avalanchego/vms/proposervm/summary" + + statelessblock "github.com/ava-labs/avalanchego/vms/proposervm/block" +) + +// Post Fork section +func TestBlockBackfillEnabledPostFork(t *testing.T) { + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + // 1. Accept a State summary + var ( + forkHeight = uint64(100) + stateSummaryHeight = uint64(2023) + proVMParentStateSummaryBlk = ids.GenerateTestID() + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) + + ctx := context.Background() + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + // 2. Check that block backfilling is enabled looking at innerVM + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proVMParentStateSummaryBlk, blkID) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) +} + +func TestBlockBackfillPostForkSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(100) + blkCount = 12 + startBlkHeight = uint64(1492) + + // create a list of consecutive blocks and build state summary of top of them + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID == blk.ID() { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + lowestblk := innerBlks[0] + for _, blk := range innerBlks { + if blk.Height() < lowestblk.Height() { + lowestblk = blk + } + } + return lowestblk.Parent(), lowestblk.Height() - 1, nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[0].Parent(), nextBlkID) + require.Equal(proBlks[0].Height()-1, nextBlkHeight) + + // check proBlocks have been indexed + for _, blk := range proBlks { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } +} + +func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(100) + blkCount = 10 + startBlkHeight = uint64(1492) + + // create a list of consecutive blocks and build state summary of top of them + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + + blkID, height, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proTopBlk.ID(), blkID) + require.Equal(proTopBlk.Height(), height) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + + // simulate that lower half of backfilled blocks won't be accepted by innerVM + idx := len(innerBlks) / 2 + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID != blk.ID() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return nil, database.ErrNotFound + } + return blk, nil + } + return nil, database.ErrNotFound + } + + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + // assume lowest half blocks fails verification + return innerBlks[idx].ID(), innerBlks[idx].Height(), nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[idx].ID(), nextBlkID) + require.Equal(proBlks[idx].Height(), nextBlkHeight) + + // check only upper half of blocks have been indexed + for i, blk := range proBlks { + if i <= idx { + _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.ErrorIs(err, database.ErrNotFound) + + _, err = vm.GetBlock(ctx, blk.ID()) + require.ErrorIs(err, database.ErrNotFound) + } else { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } + } +} + +// Across Fork section +func TestBlockBackfillEnabledAcrossFork(t *testing.T) { + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + // 1. Accept a State summary + var ( + forkHeight = uint64(50) + stateSummaryHeight = forkHeight + 1 + proVMParentStateSummaryBlk = ids.GenerateTestID() + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + // 2. Check that block backfilling is enabled looking at innerVM + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proVMParentStateSummaryBlk, blkID) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) +} + +func TestBlockBackfillAcrossForkSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(100) + blkCount = 4 + startBlkHeight = forkHeight - uint64(blkCount)/2 + + // create a list of consecutive blocks and build state summary of top of them + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID == blk.ID() { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + lowestblk := innerBlks[0] + for _, blk := range innerBlks { + if blk.Height() < lowestblk.Height() { + lowestblk = blk + } + } + return lowestblk.Parent(), lowestblk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height == blk.Height() { + return blk.ID(), nil + } + } + return ids.Empty, database.ErrNotFound + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[0].Parent(), nextBlkID) + require.Equal(proBlks[0].Height()-1, nextBlkHeight) + + // check proBlocks have been indexed + for _, blk := range proBlks { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } +} + +func TestBlockBackfillAcrossForkPartialSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(100) + blkCount = 8 + startBlkHeight = forkHeight - uint64(blkCount)/2 + + // simulate that the bottom [idxFailure] blocks will fail + // being pushed in innerVM + idxFailure = 3 + + // create a list of consecutive blocks and build state summary of top of them + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + + blkID, height, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proTopBlk.ID(), blkID) + require.Equal(proTopBlk.Height(), height) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + + // simulate that lower half of backfilled blocks won't be accepted by innerVM + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID != blk.ID() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idxFailure].Height() { + return nil, database.ErrNotFound + } + return blk, nil + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height != blk.Height() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idxFailure].Height() { + return ids.Empty, database.ErrNotFound + } + return blk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + // assume lowest half blocks fails verification + return innerBlks[idxFailure].ID(), innerBlks[idxFailure].Height(), nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[idxFailure].ID(), nextBlkID) + require.Equal(proBlks[idxFailure].Height(), nextBlkHeight) + + // check only upper half of blocks have been indexed + for i, blk := range proBlks { + if i <= idxFailure { + _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.ErrorIs(err, database.ErrNotFound) + + _, err = vm.GetBlock(ctx, blk.ID()) + require.ErrorIs(err, database.ErrNotFound) + } else { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } + } +} + +// Pre Fork section +func TestBlockBackfillEnabledPreFork(t *testing.T) { + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + // 1. Accept a State summary + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: 100, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + // 2. Check that block backfilling is enabled looking at innerVM + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + innerVM.BackfillBlocksEnabledF = func(_ context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(innerStateSyncedBlk.Parent(), blkID) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) +} + +func TestBlockBackfillPreForkSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(2000) + blkCount = 8 + startBlkHeight = uint64(100) + + // create a list of consecutive blocks and build state summary of top of them + // proBlks should all be preForkBlocks + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + preForkTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(preForkTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID == blk.ID() { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height == blk.Height() { + return blk.ID(), nil + } + } + return ids.Empty, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + lowestblk := innerBlks[0] + for _, blk := range innerBlks { + if blk.Height() < lowestblk.Height() { + lowestblk = blk + } + } + return lowestblk.Parent(), lowestblk.Height() - 1, nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[0].Parent(), nextBlkID) + require.Equal(proBlks[0].Height()-1, nextBlkHeight) + + // check proBlocks have been indexed + for _, blk := range proBlks { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } +} + +func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(2000) + blkCount = 10 + startBlkHeight = uint64(100) + + // create a list of consecutive blocks and build state summary of top of them + // proBlks should all be preForkBlocks + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + preForkTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(preForkTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + // simulate that lower half of backfilled blocks won't be accepted by innerVM + idx := len(innerBlks) / 2 + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID != blk.ID() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return nil, database.ErrNotFound + } + return blk, nil + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height != blk.Height() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return ids.Empty, database.ErrNotFound + } + return blk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + // assume lowest half blocks fails verification + return innerBlks[idx].ID(), innerBlks[idx].Height(), nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[idx].ID(), nextBlkID) + require.Equal(proBlks[idx].Height(), nextBlkHeight) + + // check only upper half of blocks have been indexed + for i, blk := range proBlks { + if i <= idx { + _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.ErrorIs(err, database.ErrNotFound) + + _, err = vm.GetBlock(ctx, blk.ID()) + require.ErrorIs(err, database.ErrNotFound) + } else { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } + } +} + +func createTestBlocks( + t *testing.T, + vm *VM, + forkHeight uint64, + blkCount int, + startBlkHeight uint64, +) ( + []snowman.Block, // proposerVM blocks + []snowman.Block, // inner VM blocks +) { + require := require.New(t) + var ( + latestInnerBlkID = ids.GenerateTestID() + + dummyBlkTime = time.Now() + dummyPChainHeight = startBlkHeight / 2 + + innerBlks = make([]snowman.Block, 0, blkCount) + proBlks = make([]snowman.Block, 0, blkCount) + ) + for idx := 0; idx < blkCount; idx++ { + blkHeight := startBlkHeight + uint64(idx) + + rndBytes := ids.GenerateTestID() + innerBlkTop := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.GenerateTestID(), + StatusV: choices.Processing, + }, + BytesV: rndBytes[:], + ParentV: latestInnerBlkID, + HeightV: startBlkHeight + uint64(idx), + } + latestInnerBlkID = innerBlkTop.ID() + innerBlks = append(innerBlks, innerBlkTop) + + if blkHeight < forkHeight { + proBlks = append(proBlks, &preForkBlock{ + vm: vm, + Block: innerBlkTop, + }) + } else { + latestProBlkID := ids.GenerateTestID() + if len(proBlks) != 0 { + latestProBlkID = proBlks[len(proBlks)-1].ID() + } + statelessChild, err := statelessblock.BuildUnsigned( + latestProBlkID, + dummyBlkTime, + dummyPChainHeight, + innerBlkTop.Bytes(), + ) + require.NoError(err) + proBlkTop := &postForkBlock{ + SignedBlock: statelessChild, + postForkCommonComponents: postForkCommonComponents{ + vm: vm, + innerBlk: innerBlkTop, + status: choices.Processing, + }, + } + proBlks = append(proBlks, proBlkTop) + } + } + return proBlks, innerBlks +} + +func createPostForkStateSummary( + t *testing.T, + vm *VM, + forkHeight uint64, + proVMParentStateSummaryBlk ids.ID, + innerVM *fullVM, + innerSummary *block.TestStateSummary, + innerBlk *snowman.TestBlock, +) block.StateSummary { + require := require.New(t) + + pchainHeight := innerBlk.Height() / 2 + slb, err := statelessblock.Build( + proVMParentStateSummaryBlk, + innerBlk.Timestamp(), + pchainHeight, + vm.StakingCertLeaf, + innerBlk.Bytes(), + vm.ctx.ChainID, + vm.StakingLeafSigner, + ) + require.NoError(err) + + statelessSummary, err := summary.Build(forkHeight, slb.Bytes(), innerSummary.Bytes()) + require.NoError(err) + + innerVM.ParseStateSummaryF = func(ctx context.Context, summaryBytes []byte) (block.StateSummary, error) { + require.Equal(innerSummary.BytesV, summaryBytes) + return innerSummary, nil + } + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + require.Equal(innerBlk.Bytes(), b) + return innerBlk, nil + } + + summary, err := vm.ParseStateSummary(context.Background(), statelessSummary.Bytes()) + require.NoError(err) + return summary +} + +func setupBlockBackfillingVM( + t *testing.T, + toEngineCh chan<- common.Message, +) ( + *fullVM, + *VM, +) { + require := require.New(t) + + innerVM := &fullVM{ + TestVM: &block.TestVM{ + TestVM: common.TestVM{ + T: t, + }, + }, + TestStateSyncableVM: &block.TestStateSyncableVM{ + T: t, + }, + } + + // signal height index is complete + innerVM.VerifyHeightIndexF = func(context.Context) error { + return nil + } + + // load innerVM expectations + innerGenesisBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'G', 'e', 'n', 'e', 's', 'i', 's', 'I', 'D'}, + }, + HeightV: 0, + BytesV: []byte("genesis state"), + } + + innerVM.InitializeF = func(_ context.Context, _ *snow.Context, _ database.Database, + _ []byte, _ []byte, _ []byte, ch chan<- common.Message, + _ []*common.Fx, _ common.AppSender, + ) error { + return nil + } + innerVM.VerifyHeightIndexF = func(context.Context) error { + return nil + } + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerGenesisBlk.ID(), nil + } + innerVM.GetBlockF = func(context.Context, ids.ID) (snowman.Block, error) { + return innerGenesisBlk, nil + } + + // createVM + vm := New( + innerVM, + Config{ + ActivationTime: time.Time{}, + MinimumPChainHeight: 0, + MinBlkDelay: DefaultMinBlockDelay, + NumHistoricalBlocks: DefaultNumHistoricalBlocks, + StakingLeafSigner: pTestSigner, + StakingCertLeaf: pTestCert, + }, + ) + + ctx := snowtest.Context(t, snowtest.CChainID) + ctx.NodeID = ids.NodeIDFromCert(pTestCert) + + require.NoError(vm.Initialize( + context.Background(), + ctx, + memdb.New(), + innerGenesisBlk.Bytes(), + nil, + nil, + toEngineCh, + nil, + nil, + )) + + return innerVM, vm +} diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 06379a1797fe..cdb81e167453 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -5,12 +5,16 @@ package proposervm import ( "context" + "errors" "fmt" + "sort" "go.uber.org/zap" + "golang.org/x/exp/maps" "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow/consensus/snowman" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "github.com/ava-labs/avalanchego/vms/proposervm/summary" ) @@ -161,10 +165,184 @@ func (vm *VM) buildStateSummary(ctx context.Context, innerSummary block.StateSum }, nil } -func (*VM) BackfillBlocksEnabled(context.Context) (ids.ID, uint64, error) { - return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled +func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, uint64, error) { + if vm.ssVM == nil { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + + _, innerBlkHeight, err := vm.ssVM.BackfillBlocksEnabled(ctx) + if err != nil { + return ids.Empty, 0, fmt.Errorf("failed checking that block backfilling is enabled in innerVM: %w", err) + } + + return vm.nextBlockBackfillData(ctx, innerBlkHeight) } -func (*VM) BackfillBlocks(context.Context, [][]byte) (ids.ID, uint64, error) { - return ids.Empty, 0, block.ErrStopBlockBackfilling +func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, uint64, error) { + blks := make(map[uint64]Block) + + // 1. Parse + for i, blkBytes := range blksBytes { + blk, err := vm.parseBlock(ctx, blkBytes) + if err != nil { + return ids.Empty, 0, fmt.Errorf("failed parsing backfilled block, index %d, %w", i, err) + } + blks[blk.Height()] = blk + } + + // 2. Validate blocks, checking that they are continguous + blkHeights := maps.Keys(blks) + sort.Slice(blkHeights, func(i, j int) bool { + return blkHeights[i] < blkHeights[j] + }) + + var ( + topBlk = blks[blkHeights[len(blkHeights)-1]] + topIdx = len(blkHeights) - 2 + ) + + // vm.latestBackfilledBlock is non nil only if proposerVM has forked + if vm.latestBackfilledBlock != ids.Empty { + latestBackfilledBlk, err := vm.getBlock(ctx, vm.latestBackfilledBlock) + if err != nil { + return ids.Empty, 0, fmt.Errorf( + "failed retrieving latest backfilled block, %s, %w, %w", + vm.latestBackfilledBlock, + err, + block.ErrInternalBlockBackfilling, + ) + } + + topBlk = latestBackfilledBlk + topIdx = len(blkHeights) - 1 + } + + for i := topIdx; i >= 0; i-- { + blk := blks[blkHeights[i]] + if topBlk.Parent() != blk.ID() { + return ids.Empty, 0, fmt.Errorf("unexpected backfilled block %s, expected child' parent is %s", blk.ID(), topBlk.Parent()) + } + if err := blk.acceptOuterBlk(); err != nil { + return ids.Empty, 0, fmt.Errorf( + "failed indexing backfilled block, blkID %s, %w, %w", + blk.ID(), + err, + block.ErrInternalBlockBackfilling, + ) + } + topBlk = blk + } + + // 3. Backfill inner blocks to innerVM + innerBlksBytes := make([][]byte, 0, len(blksBytes)) + for _, blk := range blks { + innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) + } + _, nextInnerBlkHeight, err := vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) + switch { + case errors.Is(err, block.ErrStopBlockBackfilling): + return ids.Empty, 0, err // done backfilling + case errors.Is(err, block.ErrInternalBlockBackfilling): + return ids.Empty, 0, err + case err == nil: + // check proposerVM and innerVM alignment + default: + // non-internal error in innerVM, check proposerVM and innerVM alignment + } + + // 4. Check alignment + for _, blk := range blks { + innerBlkID := blk.getInnerBlk().ID() + switch _, err := vm.ChainVM.GetBlock(ctx, innerBlkID); err { + case nil: + continue + case database.ErrNotFound: + if err := vm.revertBackfilledBlock(blk); err != nil { + return ids.Empty, 0, fmt.Errorf("failed reverting backfilled VM block from height index %s, %w", blk.ID(), err) + } + default: + return ids.Empty, 0, fmt.Errorf( + "failed checking innerVM block %s, %w, %w", + innerBlkID, + err, + block.ErrInternalBlockBackfilling, + ) + } + } + + return vm.nextBlockBackfillData(ctx, nextInnerBlkHeight) +} + +func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) (ids.ID, uint64, error) { + childBlkHeight := innerBlkHeight + 1 + childBlkID, err := vm.GetBlockIDAtHeight(ctx, childBlkHeight) + if err != nil { + return ids.Empty, 0, fmt.Errorf( + "failed retrieving proposer block ID at height %d: %w, %w", + childBlkHeight, + err, + block.ErrInternalBlockBackfilling, + ) + } + + var childBlk snowman.Block + childBlk, err = vm.getPostForkBlock(ctx, childBlkID) + switch err { + case nil: + vm.latestBackfilledBlock = childBlkID + if err := vm.State.SetLastBackfilledBlkID(childBlkID); err != nil { + return ids.Empty, 0, fmt.Errorf( + "failed storing last backfilled block ID: %w, %w", + err, + block.ErrInternalBlockBackfilling, + ) + } + if err := vm.db.Commit(); err != nil { + return ids.Empty, 0, fmt.Errorf( + "failed committing backfilled blocks reversal: %w, %w", + err, + block.ErrInternalBlockBackfilling, + ) + } + case database.ErrNotFound: + // proposerVM may not be active yet. + childBlk, err = vm.getPreForkBlock(ctx, childBlkID) + if err != nil { + return ids.Empty, + 0, + fmt.Errorf("failed retrieving innerVM block %s: %w, %w", + childBlkID, + err, + block.ErrInternalBlockBackfilling, + ) + } + default: + return ids.Empty, 0, fmt.Errorf( + "failed retrieving proposer block %s: %w, %w", + childBlkID, + err, + block.ErrInternalBlockBackfilling, + ) + } + + return childBlk.Parent(), childBlk.Height() - 1, nil +} + +func (vm *VM) revertBackfilledBlock(blk Block) error { + if err := vm.State.DeleteBlock(blk.ID()); err != nil { + return fmt.Errorf( + "failed reverting backfilled VM block %s: %w, %w", + blk.ID(), + err, + block.ErrInternalBlockBackfilling) + } + if err := vm.State.DeleteBlockIDAtHeight(blk.Height()); err != nil { + return fmt.Errorf( + "failed reverting backfilled VM block from height index %s: %w, %w", + blk.ID(), + err, + block.ErrInternalBlockBackfilling, + ) + } + return nil } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 99dc045be66d..e447e7aeba12 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -122,6 +122,11 @@ type VM struct { // lastAcceptedHeight is set to the last accepted PostForkBlock's height. lastAcceptedHeight uint64 + + // latestBackfilledBlock track the latest post fork block + // indexed via block backfilling. Will be ids.Empty if proposerVM + // fork is not active + latestBackfilledBlock ids.ID } // New performs best when [minBlkDelay] is whole seconds. This is because block @@ -237,6 +242,10 @@ func (vm *VM) Initialize( return err } + if err := vm.repairBlockBackfilling(ctx); err != nil { + return err + } + forkHeight, err := vm.getForkHeight() switch err { case nil: @@ -255,6 +264,52 @@ func (vm *VM) Initialize( return nil } +func (vm *VM) repairBlockBackfilling(ctx context.Context) error { + bottomBlkID, err := vm.State.GetLastBackfilledBlkID() + switch { + case errors.Is(err, database.ErrNotFound): + // vm never backfilled blocks. + return nil + case err != nil: + return fmt.Errorf("failed loading last backfilled block ID %s, %w", bottomBlkID, err) + default: + // check alignment + } + + for { + blk, err := vm.getBlock(ctx, bottomBlkID) + if err != nil { + return fmt.Errorf("failed retrieving latest backfilled block, %s: %w", bottomBlkID, err) + } + + var ( + innerBlkID = blk.getInnerBlk().ID() + childBlkHeight = blk.Height() + 1 + ) + + _, err = vm.ChainVM.GetBlock(ctx, innerBlkID) + switch err { + case nil: + if err := vm.db.Commit(); err != nil { + return fmt.Errorf("failed committing backfilled blocks reversal, %w", err) + } + return nil // proposerVM and innerVM aligned + case database.ErrNotFound: + if err := vm.revertBackfilledBlock(blk); err != nil { + return err + } + childBlkID, err := vm.GetBlockIDAtHeight(ctx, childBlkHeight) + if err != nil { + return fmt.Errorf("failed retrieving blkID at height %d, while repairing backfilled blocks: %w", childBlkHeight, err) + } + bottomBlkID = childBlkID + default: + return fmt.Errorf("failed retrieving inner vm blk id %s, while repairing backfilled blocks: %w", innerBlkID, err) + } + vm.latestBackfilledBlock = bottomBlkID + } +} + // shutdown ops then propagate shutdown to innerVM func (vm *VM) Shutdown(ctx context.Context) error { vm.onShutdown() @@ -303,6 +358,10 @@ func (vm *VM) BuildBlock(ctx context.Context) (snowman.Block, error) { } func (vm *VM) ParseBlock(ctx context.Context, b []byte) (snowman.Block, error) { + return vm.parseBlock(ctx, b) +} + +func (vm *VM) parseBlock(ctx context.Context, b []byte) (Block, error) { if blk, err := vm.parsePostForkBlock(ctx, b); err == nil { return blk, nil } @@ -835,13 +894,17 @@ func (vm *VM) acceptPostForkBlock(blk PostForkBlock) error { height := blk.Height() blkID := blk.ID() - vm.lastAcceptedHeight = height - delete(vm.verifiedBlocks, blkID) + // following backfill introduction we may store past + // blocks, hence safeguard last accepted block data + if height >= vm.lastAcceptedHeight { + vm.lastAcceptedHeight = height + delete(vm.verifiedBlocks, blkID) - // Persist this block, its height index, and its status - if err := vm.State.SetLastAccepted(blkID); err != nil { - return err + if err := vm.State.SetLastAccepted(blkID); err != nil { + return err + } } + if err := vm.State.PutBlock(blk.getStatelessBlk(), choices.Accepted); err != nil { return err }