From a2511c84063f4894b6af2f174cc32b07a8f64127 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 00:07:25 +0100 Subject: [PATCH 01/25] moved up proposerVM block backfilling --- vms/proposervm/state_syncable_vm.go | 25 ++++--------------------- vms/proposervm/vm.go | 4 ---- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 270b0f7b5788..32e78444b6c7 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -161,27 +161,10 @@ func (vm *VM) buildStateSummary(ctx context.Context, innerSummary block.StateSum }, nil } -func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, error) { - if vm.ssVM == nil { - return ids.Empty, nil - } - - return vm.ssVM.BackfillBlocksEnabled(ctx) +func (*VM) BackfillBlocksEnabled(context.Context) (ids.ID, error) { + return ids.Empty, block.ErrBlockBackfillingNotEnabled } -func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, error) { - innerBlksBytes := make([][]byte, 0, len(blksBytes)) - for i, blkBytes := range blksBytes { - blk, err := vm.parseProposerBlock(ctx, blkBytes) - if err != nil { - return ids.Empty, fmt.Errorf("failed parsing backfilled block, index %d, %w", i, err) - } - // TODO: consider validating that block is at least below last accepted block - // or that block has not been stored yet - if err := blk.acceptOuterBlk(); err != nil { - return ids.Empty, fmt.Errorf("failed indexing backfilled block, index %d, %w", i, err) - } - innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) - } - return vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) +func (*VM) BackfillBlocks(context.Context, [][]byte) (ids.ID, error) { + return ids.Empty, block.ErrStopBlockBackfilling } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 1bfdf13da0a5..a101be86f574 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -327,10 +327,6 @@ func (vm *VM) BuildBlock(ctx context.Context) (snowman.Block, error) { } func (vm *VM) ParseBlock(ctx context.Context, b []byte) (snowman.Block, error) { - return vm.parseProposerBlock(ctx, b) -} - -func (vm *VM) parseProposerBlock(ctx context.Context, b []byte) (Block, error) { if blk, err := vm.parsePostForkBlock(ctx, b); err == nil { return blk, nil } From 24e58baf2f5648c21c32a94f28a7a180e955fd09 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 00:09:36 +0100 Subject: [PATCH 02/25] proposerVM scaffolding --- vms/proposervm/state_syncable_vm.go | 25 +++++++++++++++++++++---- vms/proposervm/vm.go | 4 ++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 32e78444b6c7..270b0f7b5788 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -161,10 +161,27 @@ func (vm *VM) buildStateSummary(ctx context.Context, innerSummary block.StateSum }, nil } -func (*VM) BackfillBlocksEnabled(context.Context) (ids.ID, error) { - return ids.Empty, block.ErrBlockBackfillingNotEnabled +func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, error) { + if vm.ssVM == nil { + return ids.Empty, nil + } + + return vm.ssVM.BackfillBlocksEnabled(ctx) } -func (*VM) BackfillBlocks(context.Context, [][]byte) (ids.ID, error) { - return ids.Empty, block.ErrStopBlockBackfilling +func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, error) { + innerBlksBytes := make([][]byte, 0, len(blksBytes)) + for i, blkBytes := range blksBytes { + blk, err := vm.parseProposerBlock(ctx, blkBytes) + if err != nil { + return ids.Empty, fmt.Errorf("failed parsing backfilled block, index %d, %w", i, err) + } + // TODO: consider validating that block is at least below last accepted block + // or that block has not been stored yet + if err := blk.acceptOuterBlk(); err != nil { + return ids.Empty, fmt.Errorf("failed indexing backfilled block, index %d, %w", i, err) + } + innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) + } + return vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index a101be86f574..1bfdf13da0a5 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -327,6 +327,10 @@ func (vm *VM) BuildBlock(ctx context.Context) (snowman.Block, error) { } func (vm *VM) ParseBlock(ctx context.Context, b []byte) (snowman.Block, error) { + return vm.parseProposerBlock(ctx, b) +} + +func (vm *VM) parseProposerBlock(ctx context.Context, b []byte) (Block, error) { if blk, err := vm.parsePostForkBlock(ctx, b); err == nil { return blk, nil } From dad9abeea4135362fa1139b2252177e66e9a50a7 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 00:20:56 +0100 Subject: [PATCH 03/25] proposerVM hardened BackfillBlocks checks --- vms/proposervm/state_syncable_vm.go | 30 +++++++++++++++++++++++++---- vms/proposervm/vm.go | 3 +++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 270b0f7b5788..a9c904a3b84d 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -13,6 +13,8 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "github.com/ava-labs/avalanchego/vms/proposervm/summary" + + safemath "github.com/ava-labs/avalanchego/utils/math" ) func (vm *VM) StateSyncEnabled(ctx context.Context) (bool, error) { @@ -166,22 +168,42 @@ func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, error) { return ids.Empty, nil } - return vm.ssVM.BackfillBlocksEnabled(ctx) + innerBlkID, err := vm.ssVM.BackfillBlocksEnabled(ctx) + if err != nil { + return ids.Empty, err + } + + innerBlk, err := vm.ChainVM.GetBlock(ctx, innerBlkID) + if err != nil { + return ids.Empty, err + } + + vm.latestBlockBackfillingHeight = innerBlk.Height() + return innerBlkID, nil } func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, error) { - innerBlksBytes := make([][]byte, 0, len(blksBytes)) + var ( + innerBlksBytes = make([][]byte, 0, len(blksBytes)) + minBlocksHeight = vm.latestBlockBackfillingHeight + ) + for i, blkBytes := range blksBytes { blk, err := vm.parseProposerBlock(ctx, blkBytes) if err != nil { return ids.Empty, fmt.Errorf("failed parsing backfilled block, index %d, %w", i, err) } - // TODO: consider validating that block is at least below last accepted block - // or that block has not been stored yet + if blk.Height() > vm.latestBlockBackfillingHeight { + return ids.Empty, fmt.Errorf("") + } + minBlocksHeight = safemath.Min(minBlocksHeight, blk.Height()) + if err := blk.acceptOuterBlk(); err != nil { return ids.Empty, fmt.Errorf("failed indexing backfilled block, index %d, %w", i, err) } innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) } + + vm.lastAcceptedHeight = minBlocksHeight return vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 1bfdf13da0a5..e154d4f6c655 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -133,6 +133,9 @@ type VM struct { // lastAcceptedHeight is set to the last accepted PostForkBlock's height. lastAcceptedHeight uint64 + + // blockBackfilling attributes + latestBlockBackfillingHeight uint64 } // New performs best when [minBlkDelay] is whole seconds. This is because block From 95961ceaf31e1a0dc1efeb64358dfcb4fda90632 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 00:44:05 +0100 Subject: [PATCH 04/25] fixed merge --- vms/proposervm/vm.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 4c5adf63c08b..87a5221a3fb5 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -330,6 +330,10 @@ func (vm *VM) BuildBlock(ctx context.Context) (snowman.Block, error) { } func (vm *VM) ParseBlock(ctx context.Context, b []byte) (snowman.Block, error) { + return parseProposerBlock(ctx, b) +} + +func (vm *VM) parseProposerBlock(ctx context.Context, b []byte) (Block, error) { if blk, err := vm.parsePostForkBlock(ctx, b); err == nil { return blk, nil } From 0f5b503a092738c47205c015e32183ab86aec453 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 00:44:16 +0100 Subject: [PATCH 05/25] fixed merge --- vms/proposervm/vm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 87a5221a3fb5..e154d4f6c655 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -330,7 +330,7 @@ func (vm *VM) BuildBlock(ctx context.Context) (snowman.Block, error) { } func (vm *VM) ParseBlock(ctx context.Context, b []byte) (snowman.Block, error) { - return parseProposerBlock(ctx, b) + return vm.parseProposerBlock(ctx, b) } func (vm *VM) parseProposerBlock(ctx context.Context, b []byte) (Block, error) { From fe215b2cf14e80163a9aae82d6f211d8c892f35d Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 00:58:08 +0100 Subject: [PATCH 06/25] cleanup --- vms/proposervm/state_syncable_vm.go | 23 +++++------------------ vms/proposervm/vm.go | 3 --- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index a9c904a3b84d..71752233866b 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -13,8 +13,6 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "github.com/ava-labs/avalanchego/vms/proposervm/summary" - - safemath "github.com/ava-labs/avalanchego/utils/math" ) func (vm *VM) StateSyncEnabled(ctx context.Context) (bool, error) { @@ -173,30 +171,19 @@ func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, error) { return ids.Empty, err } - innerBlk, err := vm.ChainVM.GetBlock(ctx, innerBlkID) - if err != nil { - return ids.Empty, err - } - - vm.latestBlockBackfillingHeight = innerBlk.Height() + // vvv This is wrong vvv It returns innerVM BlockID, instead of proposerVM blockID return innerBlkID, nil } func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, error) { - var ( - innerBlksBytes = make([][]byte, 0, len(blksBytes)) - minBlocksHeight = vm.latestBlockBackfillingHeight - ) - + innerBlksBytes := make([][]byte, 0, len(blksBytes)) for i, blkBytes := range blksBytes { blk, err := vm.parseProposerBlock(ctx, blkBytes) if err != nil { return ids.Empty, fmt.Errorf("failed parsing backfilled block, index %d, %w", i, err) } - if blk.Height() > vm.latestBlockBackfillingHeight { - return ids.Empty, fmt.Errorf("") - } - minBlocksHeight = safemath.Min(minBlocksHeight, blk.Height()) + + // TODO: introduce validation if err := blk.acceptOuterBlk(); err != nil { return ids.Empty, fmt.Errorf("failed indexing backfilled block, index %d, %w", i, err) @@ -204,6 +191,6 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, e innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) } - vm.lastAcceptedHeight = minBlocksHeight + // vvv This is wrong vvv It returns innerVM BlockID, instead of proposerVM blockID return vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index e154d4f6c655..1bfdf13da0a5 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -133,9 +133,6 @@ type VM struct { // lastAcceptedHeight is set to the last accepted PostForkBlock's height. lastAcceptedHeight uint64 - - // blockBackfilling attributes - latestBlockBackfillingHeight uint64 } // New performs best when [minBlkDelay] is whole seconds. This is because block From 5beab36be9e89b0fb3e888a1d63f968e3729155a Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 10:52:30 +0100 Subject: [PATCH 07/25] wip: proposerVM block backfilling fixes --- vms/proposervm/scheduler/scheduler.go | 13 +++-- vms/proposervm/scheduler/scheduler_test.go | 36 +++++++----- vms/proposervm/state_syncable_vm.go | 67 ++++++++++++++++++---- vms/proposervm/vm.go | 8 ++- 4 files changed, 93 insertions(+), 31 deletions(-) diff --git a/vms/proposervm/scheduler/scheduler.go b/vms/proposervm/scheduler/scheduler.go index 5946a67b9b77..91ef6d09e546 100644 --- a/vms/proposervm/scheduler/scheduler.go +++ b/vms/proposervm/scheduler/scheduler.go @@ -9,6 +9,7 @@ import ( "go.uber.org/zap" "github.com/ava-labs/avalanchego/snow/engine/common" + "github.com/ava-labs/avalanchego/utils" "github.com/ava-labs/avalanchego/utils/logging" ) @@ -37,15 +38,18 @@ type scheduler struct { // from telling the engine to call its VM's BuildBlock method until the // given time newBuildBlockTime chan time.Time + + stateSyncDone *utils.Atomic[bool] } -func New(log logging.Logger, toEngine chan<- common.Message) (Scheduler, chan<- common.Message) { +func New(log logging.Logger, toEngine chan<- common.Message, stateSyncDone *utils.Atomic[bool]) (Scheduler, chan<- common.Message) { vmToEngine := make(chan common.Message, cap(toEngine)) return &scheduler{ log: log, fromVM: vmToEngine, toEngine: toEngine, newBuildBlockTime: make(chan time.Time), + stateSyncDone: stateSyncDone, }, vmToEngine } @@ -74,9 +78,10 @@ waitloop: for { select { - case msg := <-s.fromVM: - // Give the engine the message from the VM asking the engine to - // build a block + case msg := <-s.fromVM: // Pass the message up to the engine + if msg == common.StateSyncDone { + s.stateSyncDone.Set(true) + } select { case s.toEngine <- msg: default: diff --git a/vms/proposervm/scheduler/scheduler_test.go b/vms/proposervm/scheduler/scheduler_test.go index 821a36883e90..e83cec258ac9 100644 --- a/vms/proposervm/scheduler/scheduler_test.go +++ b/vms/proposervm/scheduler/scheduler_test.go @@ -10,14 +10,18 @@ import ( "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/snow/engine/common" + "github.com/ava-labs/avalanchego/utils" "github.com/ava-labs/avalanchego/utils/logging" ) func TestDelayFromNew(t *testing.T) { - toEngine := make(chan common.Message, 10) - startTime := time.Now().Add(50 * time.Millisecond) + var ( + toEngine = make(chan common.Message, 10) + startTime = time.Now().Add(50 * time.Millisecond) + dummyStateSyncDone = utils.Atomic[bool]{} + ) - s, fromVM := New(logging.NoLog{}, toEngine) + s, fromVM := New(logging.NoLog{}, toEngine, &dummyStateSyncDone) defer s.Close() go s.Dispatch(startTime) @@ -28,11 +32,14 @@ func TestDelayFromNew(t *testing.T) { } func TestDelayFromSetTime(t *testing.T) { - toEngine := make(chan common.Message, 10) - now := time.Now() - startTime := now.Add(50 * time.Millisecond) - - s, fromVM := New(logging.NoLog{}, toEngine) + var ( + toEngine = make(chan common.Message, 10) + now = time.Now() + startTime = now.Add(50 * time.Millisecond) + dummyStateSyncDone = utils.Atomic[bool]{} + ) + + s, fromVM := New(logging.NoLog{}, toEngine, &dummyStateSyncDone) defer s.Close() go s.Dispatch(now) @@ -45,11 +52,14 @@ func TestDelayFromSetTime(t *testing.T) { } func TestReceipt(*testing.T) { - toEngine := make(chan common.Message, 10) - now := time.Now() - startTime := now.Add(50 * time.Millisecond) - - s, fromVM := New(logging.NoLog{}, toEngine) + var ( + toEngine = make(chan common.Message, 10) + now = time.Now() + startTime = now.Add(50 * time.Millisecond) + dummyStateSyncDone = utils.Atomic[bool]{} + ) + + s, fromVM := New(logging.NoLog{}, toEngine, &dummyStateSyncDone) defer s.Close() go s.Dispatch(now) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 71752233866b..70b3d3c8457a 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -162,35 +162,80 @@ func (vm *VM) buildStateSummary(ctx context.Context, innerSummary block.StateSum } func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, error) { - if vm.ssVM == nil { + if vm.ssVM == nil || !vm.stateSyncDone.Get() { return ids.Empty, nil } + // if vm implements Snowman++, a block height index must be available + // to support state sync + if err := vm.VerifyHeightIndex(ctx); err != nil { + return ids.Empty, fmt.Errorf("could not verify height index: %w", err) + } + innerBlkID, err := vm.ssVM.BackfillBlocksEnabled(ctx) if err != nil { - return ids.Empty, err + return ids.Empty, fmt.Errorf("failed checking that block backfilling is enabled in innerVM: %w", err) } - // vvv This is wrong vvv It returns innerVM BlockID, instead of proposerVM blockID - return innerBlkID, nil + innerBlk, err := vm.ChainVM.GetBlock(ctx, innerBlkID) + if err != nil { + return ids.Empty, fmt.Errorf("failed retrieving block ID %s from innerVM: %w", innerBlkID, err) + } + + return vm.GetBlockIDAtHeight(ctx, innerBlk.Height()) } func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, error) { - innerBlksBytes := make([][]byte, 0, len(blksBytes)) + // if vm implements Snowman++, a block height index must be available + // to support state sync + if err := vm.VerifyHeightIndex(ctx); err != nil { + return ids.Empty, fmt.Errorf("could not verify height index: %w", err) + } + + var ( + blks = make([]Block, 0, len(blksBytes)) + innerBlksBytes = make([][]byte, 0, len(blksBytes)) + ) + for i, blkBytes := range blksBytes { - blk, err := vm.parseProposerBlock(ctx, blkBytes) + blk, err := vm.parseBlock(ctx, blkBytes) if err != nil { return ids.Empty, fmt.Errorf("failed parsing backfilled block, index %d, %w", i, err) } // TODO: introduce validation - if err := blk.acceptOuterBlk(); err != nil { - return ids.Empty, fmt.Errorf("failed indexing backfilled block, index %d, %w", i, err) - } + blks = append(blks, blk) innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) } - // vvv This is wrong vvv It returns innerVM BlockID, instead of proposerVM blockID - return vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) + nextInnerBlkID, err := vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) + if err == nil { + // no error signals at least a single block has been accepted by the innerVM. (not necessarily all of them) + // Find out which blocks have been accepted and store them. + // Should the process err while looping there will be innerVM blocks not indexed by proposerVM. Repair will be + // carried out upon restart. + for _, blk := range blks { + _, err := vm.ChainVM.GetBlock(ctx, blk.getInnerBlk().ID()) + if err != nil { + continue + } + if err := blk.acceptOuterBlk(); err != nil { + return ids.Empty, fmt.Errorf("failed indexing backfilled block, blkID %s, %w", blk.ID(), err) + } + } + } + + // Finally map innerVM next block ID to proposerVM one + innerBlk, err := vm.ChainVM.GetBlock(ctx, nextInnerBlkID) + if err != nil { + return ids.Empty, fmt.Errorf("failed retrieving inner vm next block: %w", err) + } + + // vvv is this correct? When have I height indexed this block??? vvv If it errs it may be one of the blocks in the list + nextBlkID, err := vm.GetBlockIDAtHeight(ctx, innerBlk.Height()) + if err != nil { + return ids.Empty, fmt.Errorf("failed mapping innerVM next block ID to proposerVM one: %w", err) + } + return nextBlkID, err } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 1bfdf13da0a5..a8c17efb7cab 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -133,6 +133,8 @@ type VM struct { // lastAcceptedHeight is set to the last accepted PostForkBlock's height. lastAcceptedHeight uint64 + + stateSyncDone utils.Atomic[bool] } // New performs best when [minBlkDelay] is whole seconds. This is because block @@ -220,7 +222,7 @@ func (vm *VM) Initialize( indexerState := state.New(indexerDB) vm.hIndexer = indexer.NewHeightIndexer(vm, vm.ctx.Log, indexerState) - scheduler, vmToEngine := scheduler.New(vm.ctx.Log, toEngine) + scheduler, vmToEngine := scheduler.New(vm.ctx.Log, toEngine, &vm.stateSyncDone) vm.Scheduler = scheduler vm.toScheduler = vmToEngine @@ -327,10 +329,10 @@ func (vm *VM) BuildBlock(ctx context.Context) (snowman.Block, error) { } func (vm *VM) ParseBlock(ctx context.Context, b []byte) (snowman.Block, error) { - return vm.parseProposerBlock(ctx, b) + return vm.parseBlock(ctx, b) } -func (vm *VM) parseProposerBlock(ctx context.Context, b []byte) (Block, error) { +func (vm *VM) parseBlock(ctx context.Context, b []byte) (Block, error) { if blk, err := vm.parsePostForkBlock(ctx, b); err == nil { return blk, nil } From 57cd3bed8b8a35a341df7b86bae92dd8e94e1984 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Tue, 31 Oct 2023 15:12:22 +0100 Subject: [PATCH 08/25] wip: adding UTs --- .../state_sync_block_backfilling_test.go | 210 ++++++++++++++++++ vms/proposervm/state_syncable_vm.go | 8 +- 2 files changed, 211 insertions(+), 7 deletions(-) create mode 100644 vms/proposervm/state_sync_block_backfilling_test.go diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go new file mode 100644 index 000000000000..09c432298903 --- /dev/null +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -0,0 +1,210 @@ +// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package proposervm + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/manager" + "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow" + "github.com/ava-labs/avalanchego/snow/choices" + "github.com/ava-labs/avalanchego/snow/consensus/snowman" + "github.com/ava-labs/avalanchego/snow/engine/common" + "github.com/ava-labs/avalanchego/snow/engine/snowman/block" + "github.com/ava-labs/avalanchego/version" + "github.com/ava-labs/avalanchego/vms/proposervm/summary" + + statelessblock "github.com/ava-labs/avalanchego/vms/proposervm/block" +) + +func TestBlockBackfillEnabled(t *testing.T) { + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + // 1. Accept a State summary + stateSummaryHeight := uint64(2023) + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary, stateSummaryBlkID := createTestStateSummary(t, innerVM, vm, innerSummary, innerStateSyncedBlk) + + // Block backfilling not enabled before state sync is accepted + ctx := context.Background() + _, err := vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + _, err = stateSummary.Accept(ctx) + require.NoError(err) + + // 2. Signal to the ProposerVM that state sync is done + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + // Block backfilling not enabled before innerVM declares state sync done + _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + // Block backfilling not enabled until innerVM says so + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, error) { + return ids.Empty, block.ErrBlockBackfillingNotEnabled + } + _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + // 3. Finally check that block backfilling is done + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + blkID, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(stateSummaryBlkID, blkID) +} + +func createTestStateSummary( + t *testing.T, + innerVM *fullVM, + vm *VM, + innerSummary *block.TestStateSummary, + innerBlk *snowman.TestBlock, +) (block.StateSummary, ids.ID) { + require := require.New(t) + slb, err := statelessblock.Build( + vm.preferred, + innerBlk.Timestamp(), + 100, // pChainHeight, + vm.stakingCertLeaf, + innerBlk.Bytes(), + vm.ctx.ChainID, + vm.stakingLeafSigner, + ) + require.NoError(err) + + statelessSummary, err := summary.Build(innerSummary.Height()-1, slb.Bytes(), innerSummary.Bytes()) + require.NoError(err) + + innerVM.ParseStateSummaryF = func(ctx context.Context, summaryBytes []byte) (block.StateSummary, error) { + require.Equal(innerSummary.BytesV, summaryBytes) + return innerSummary, nil + } + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + require.Equal(innerBlk.Bytes(), b) + return innerBlk, nil + } + + summary, err := vm.ParseStateSummary(context.Background(), statelessSummary.Bytes()) + require.NoError(err) + return summary, slb.ID() +} + +func setupBlockBackfillingVM(t *testing.T, toEngineCh chan<- common.Message) (*fullVM, *VM, chan<- common.Message) { + require := require.New(t) + + innerVM := &fullVM{ + TestVM: &block.TestVM{ + TestVM: common.TestVM{ + T: t, + }, + }, + TestStateSyncableVM: &block.TestStateSyncableVM{ + T: t, + }, + } + + // signal height index is complete + innerVM.VerifyHeightIndexF = func(context.Context) error { + return nil + } + + // load innerVM expectations + innerGenesisBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'G', 'e', 'n', 'e', 's', 'i', 's', 'I', 'D'}, + }, + HeightV: 0, + BytesV: []byte("genesis state"), + } + + toProVMChannel := make(chan<- common.Message) + innerVM.InitializeF = func(_ context.Context, _ *snow.Context, _ manager.Manager, + _ []byte, _ []byte, _ []byte, ch chan<- common.Message, + _ []*common.Fx, _ common.AppSender, + ) error { + toProVMChannel = ch + return nil + } + innerVM.VerifyHeightIndexF = func(context.Context) error { + return nil + } + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerGenesisBlk.ID(), nil + } + innerVM.GetBlockF = func(context.Context, ids.ID) (snowman.Block, error) { + return innerGenesisBlk, nil + } + + // createVM + dbManager := manager.NewMemDB(version.Semantic1_0_0) + dbManager = dbManager.NewPrefixDBManager([]byte{}) + + vm := New( + innerVM, + time.Time{}, + 0, + DefaultMinBlockDelay, + DefaultNumHistoricalBlocks, + pTestSigner, + pTestCert, + ) + + ctx := snow.DefaultContextTest() + ctx.NodeID = ids.NodeIDFromCert(pTestCert) + + require.NoError(vm.Initialize( + context.Background(), + ctx, + dbManager, + innerGenesisBlk.Bytes(), + nil, + nil, + toEngineCh, + nil, + nil, + )) + + return innerVM, vm, toProVMChannel +} diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 70b3d3c8457a..1cc82723e0cf 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -163,13 +163,7 @@ func (vm *VM) buildStateSummary(ctx context.Context, innerSummary block.StateSum func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, error) { if vm.ssVM == nil || !vm.stateSyncDone.Get() { - return ids.Empty, nil - } - - // if vm implements Snowman++, a block height index must be available - // to support state sync - if err := vm.VerifyHeightIndex(ctx); err != nil { - return ids.Empty, fmt.Errorf("could not verify height index: %w", err) + return ids.Empty, block.ErrBlockBackfillingNotEnabled } innerBlkID, err := vm.ssVM.BackfillBlocksEnabled(ctx) From d1d5ad0466c9b0aebf5e71c5e488581c82f3d44f Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Wed, 1 Nov 2023 17:07:07 +0100 Subject: [PATCH 09/25] wip: adding UTs --- .../state_sync_block_backfilling_test.go | 195 +++++++++++++++++- vms/proposervm/state_syncable_vm.go | 6 - 2 files changed, 186 insertions(+), 15 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index 1690f311738f..a7e5ebb3f4d4 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -4,6 +4,7 @@ package proposervm import ( + "bytes" "context" "testing" "time" @@ -33,7 +34,11 @@ func TestBlockBackfillEnabled(t *testing.T) { }() // 1. Accept a State summary - stateSummaryHeight := uint64(2023) + var ( + stateSummaryHeight = uint64(2023) + proVMParentStateSummaryBlk = ids.GenerateTestID() + ) + innerSummary := &block.TestStateSummary{ IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, HeightV: stateSummaryHeight, @@ -46,7 +51,7 @@ func TestBlockBackfillEnabled(t *testing.T) { HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary, stateSummaryParentBlkID := createTestStateSummary(t, innerVM, vm, innerSummary, innerStateSyncedBlk) + stateSummary := createTestStateSummary(t, vm, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) // Block backfilling not enabled before state sync is accepted ctx := context.Background() @@ -79,32 +84,204 @@ func TestBlockBackfillEnabled(t *testing.T) { fromInnerVMCh <- common.StateSyncDone <-toEngineCh - // Block backfilling not enabled until innerVM says so + // 3. Finally check that block backfilling is enabled looking at innerVM innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled } _, _, err = vm.BackfillBlocksEnabled(ctx) require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - // 3. Finally check that block backfilling is done innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) - require.Equal(stateSummaryParentBlkID, blkID) + require.Equal(proVMParentStateSummaryBlk, blkID) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) +} + +func TestBlockBackfill(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + blkCount = 1 + startBlkHeight = uint64(1492) + proBlks, innerBlks = createTestBlocks(t, vm, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createTestStateSummary(t, vm, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + idx := -1 + for i, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + idx = i + break + } + } + if idx == -1 { + return nil, database.ErrNotFound + } + return innerBlks[idx], nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + idx := -1 + for i, blk := range innerBlks { + if blkID == blk.ID() { + idx = i + break + } + } + if idx == -1 { + return nil, database.ErrNotFound + } + return innerBlks[idx], nil + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + lowestblk := innerBlks[0] + for _, blk := range innerBlks { + if blk.Height() < lowestblk.Height() { + lowestblk = blk + } + } + return lowestblk.Parent(), lowestblk.Height() - 1, nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[0].Parent(), nextBlkID) + require.Equal(proBlks[0].Height()-1, nextBlkHeight) +} + +func createTestBlocks( + t *testing.T, + vm *VM, + blkCount int, + startBlkHeight uint64, +) ( + []snowman.Block, // proposerVM blocks + []snowman.Block, // inner VM blocks +) { + require := require.New(t) + var ( + latestInnerBlkID = ids.GenerateTestID() + latestProBlkID = ids.GenerateTestID() + + dummyBlkTime = time.Now() + dummyPChainHeight = uint64(1492) + + innerBlks = make([]snowman.Block, 0, blkCount) + proBlks = make([]snowman.Block, 0, blkCount) + ) + for idx := 0; idx < blkCount; idx++ { + rndBytes := ids.GenerateTestID() + innerBlkTop := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.GenerateTestID(), + StatusV: choices.Processing, + }, + BytesV: rndBytes[:], + ParentV: latestInnerBlkID, + HeightV: startBlkHeight + uint64(idx), + } + latestInnerBlkID = innerBlkTop.ID() + innerBlks = append(innerBlks, innerBlkTop) + + statelessChild, err := statelessblock.BuildUnsigned( + latestProBlkID, + dummyBlkTime, + dummyPChainHeight, + innerBlkTop.Bytes(), + ) + require.NoError(err) + proBlkTop := &postForkBlock{ + SignedBlock: statelessChild, + postForkCommonComponents: postForkCommonComponents{ + vm: vm, + innerBlk: innerBlkTop, + status: choices.Processing, + }, + } + latestProBlkID = proBlkTop.ID() + proBlks = append(proBlks, proBlkTop) + } + return proBlks, innerBlks } func createTestStateSummary( t *testing.T, - innerVM *fullVM, vm *VM, + proVMParentStateSummaryBlk ids.ID, + innerVM *fullVM, innerSummary *block.TestStateSummary, innerBlk *snowman.TestBlock, -) (block.StateSummary, ids.ID) { +) block.StateSummary { require := require.New(t) slb, err := statelessblock.Build( - vm.preferred, + proVMParentStateSummaryBlk, innerBlk.Timestamp(), 100, // pChainHeight, vm.stakingCertLeaf, @@ -128,7 +305,7 @@ func createTestStateSummary( summary, err := vm.ParseStateSummary(context.Background(), statelessSummary.Bytes()) require.NoError(err) - return summary, slb.ParentID() + return summary } func setupBlockBackfillingVM(t *testing.T, toEngineCh chan<- common.Message) (*fullVM, *VM, chan<- common.Message) { diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 4cc456f25008..88b7825bb341 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -175,12 +175,6 @@ func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, uint64, error) } func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, uint64, error) { - // if vm implements Snowman++, a block height index must be available - // to support state sync - if err := vm.VerifyHeightIndex(ctx); err != nil { - return ids.Empty, 0, fmt.Errorf("could not verify height index: %w", err) - } - var ( blks = make([]Block, 0, len(blksBytes)) innerBlksBytes = make([][]byte, 0, len(blksBytes)) From 6ded29a7d8081a89d605eb77238b99f1d423efb9 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Wed, 1 Nov 2023 23:59:36 +0100 Subject: [PATCH 10/25] wip: adding UTs --- .../state_sync_block_backfilling_test.go | 11 +++++++---- vms/proposervm/state_syncable_vm.go | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index a7e5ebb3f4d4..d73895c4e7bd 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -35,6 +35,7 @@ func TestBlockBackfillEnabled(t *testing.T) { // 1. Accept a State summary var ( + forkHeight = uint64(100) stateSummaryHeight = uint64(2023) proVMParentStateSummaryBlk = ids.GenerateTestID() ) @@ -51,7 +52,7 @@ func TestBlockBackfillEnabled(t *testing.T) { HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary := createTestStateSummary(t, vm, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) + stateSummary := createTestStateSummary(t, vm, proVMParentStateSummaryBlk, forkHeight, innerVM, innerSummary, innerStateSyncedBlk) // Block backfilling not enabled before state sync is accepted ctx := context.Background() @@ -115,7 +116,8 @@ func TestBlockBackfill(t *testing.T) { }() var ( - blkCount = 1 + forkHeight = uint64(100) + blkCount = 10 startBlkHeight = uint64(1492) proBlks, innerBlks = createTestBlocks(t, vm, blkCount, startBlkHeight) @@ -137,7 +139,7 @@ func TestBlockBackfill(t *testing.T) { HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary := createTestStateSummary(t, vm, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + stateSummary := createTestStateSummary(t, vm, proTopBlk.ID(), forkHeight, innerVM, innerSummary, innerStateSyncedBlk) innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil @@ -275,6 +277,7 @@ func createTestStateSummary( t *testing.T, vm *VM, proVMParentStateSummaryBlk ids.ID, + forkHeight uint64, innerVM *fullVM, innerSummary *block.TestStateSummary, innerBlk *snowman.TestBlock, @@ -291,7 +294,7 @@ func createTestStateSummary( ) require.NoError(err) - statelessSummary, err := summary.Build(innerSummary.Height()-1, slb.Bytes(), innerSummary.Bytes()) + statelessSummary, err := summary.Build(forkHeight, slb.Bytes(), innerSummary.Bytes()) require.NoError(err) innerVM.ParseStateSummaryF = func(ctx context.Context, summaryBytes []byte) (block.StateSummary, error) { diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 88b7825bb341..c90371031ab1 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -6,8 +6,10 @@ package proposervm import ( "context" "fmt" + "sort" "go.uber.org/zap" + "golang.org/x/exp/maps" "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/ids" @@ -176,7 +178,7 @@ func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, uint64, error) func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, uint64, error) { var ( - blks = make([]Block, 0, len(blksBytes)) + blks = make(map[uint64]Block) innerBlksBytes = make([][]byte, 0, len(blksBytes)) ) @@ -188,7 +190,7 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u // TODO: introduce validation - blks = append(blks, blk) + blks[blk.Height()] = blk innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) } @@ -198,7 +200,15 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u // Find out which blocks have been accepted and store them. // Should the process err while looping there will be innerVM blocks not indexed by proposerVM. Repair will be // carried out upon restart. - for _, blk := range blks { + + // Make sure to sort blocks by height to accept them in the right order + blkHeights := maps.Keys(blks) + sort.Slice(blkHeights, func(i, j int) bool { + return blkHeights[i] < blkHeights[j] // sort in ascending order + }) + + for _, h := range blkHeights { + blk := blks[h] _, err := vm.ChainVM.GetBlock(ctx, blk.getInnerBlk().ID()) if err != nil { continue From 2550a87ed35060467b66873ecc241199bffab7eb Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Thu, 2 Nov 2023 00:20:18 +0100 Subject: [PATCH 11/25] nit --- .../state_sync_block_backfilling_test.go | 179 ++++++++++++++++-- 1 file changed, 158 insertions(+), 21 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index d73895c4e7bd..05eb2bcd3456 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -106,7 +106,7 @@ func TestBlockBackfillEnabled(t *testing.T) { require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) } -func TestBlockBackfill(t *testing.T) { +func TestBlockBackfillSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) @@ -116,9 +116,11 @@ func TestBlockBackfill(t *testing.T) { }() var ( - forkHeight = uint64(100) - blkCount = 10 - startBlkHeight = uint64(1492) + forkHeight = uint64(100) + blkCount = 10 + startBlkHeight = uint64(1492) + + // create a list of consecutive blocks and build state summary of top of them proBlks, innerBlks = createTestBlocks(t, vm, blkCount, startBlkHeight) innerTopBlk = innerBlks[len(innerBlks)-1] @@ -173,30 +175,20 @@ func TestBlockBackfill(t *testing.T) { // Backfill some blocks innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { - idx := -1 - for i, blk := range innerBlks { + for _, blk := range innerBlks { if bytes.Equal(b, blk.Bytes()) { - idx = i - break + return blk, nil } } - if idx == -1 { - return nil, database.ErrNotFound - } - return innerBlks[idx], nil + return nil, database.ErrNotFound } innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - idx := -1 - for i, blk := range innerBlks { + for _, blk := range innerBlks { if blkID == blk.ID() { - idx = i - break + return blk, nil } } - if idx == -1 { - return nil, database.ErrNotFound - } - return innerBlks[idx], nil + return nil, database.ErrNotFound } innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { lowestblk := innerBlks[0] @@ -216,6 +208,144 @@ func TestBlockBackfill(t *testing.T) { require.NoError(err) require.Equal(proBlks[0].Parent(), nextBlkID) require.Equal(proBlks[0].Height()-1, nextBlkHeight) + + // check proBlocks have been indexed + for _, blk := range proBlks { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } +} + +func TestBlockBackfillPartialSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(100) + blkCount = 10 + startBlkHeight = uint64(1492) + + // create a list of consecutive blocks and build state summary of top of them + proBlks, innerBlks = createTestBlocks(t, vm, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + innerSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: innerSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary := createTestStateSummary(t, vm, proTopBlk.ID(), forkHeight, innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + blkID, height, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(proTopBlk.ID(), blkID) + require.Equal(proTopBlk.Height(), height) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + + // simulate that lower half of backfilled blocks won't be accepted by innerVM + idx := len(innerBlks) / 2 + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID != blk.ID() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return nil, database.ErrNotFound + } + return blk, nil + } + return nil, database.ErrNotFound + } + + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + // assume lowest half blocks fails verification + return innerBlks[idx].ID(), innerBlks[idx].Height(), nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[idx].ID(), nextBlkID) + require.Equal(proBlks[idx].Height(), nextBlkHeight) + + // check only upper half of blocks have been indexed + for i, blk := range proBlks { + if i <= idx { + _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.ErrorIs(err, database.ErrNotFound) + + _, err = vm.GetBlock(ctx, blk.ID()) + require.ErrorIs(err, database.ErrNotFound) + } else { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } + } } func createTestBlocks( @@ -311,7 +441,14 @@ func createTestStateSummary( return summary } -func setupBlockBackfillingVM(t *testing.T, toEngineCh chan<- common.Message) (*fullVM, *VM, chan<- common.Message) { +func setupBlockBackfillingVM( + t *testing.T, + toEngineCh chan<- common.Message, +) ( + *fullVM, + *VM, + chan<- common.Message, +) { require := require.New(t) innerVM := &fullVM{ From d40a4347e129f425af655f086916ff196b7fe026 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Thu, 2 Nov 2023 14:59:14 +0100 Subject: [PATCH 12/25] fixed last accepted block data --- vms/proposervm/post_fork_block.go | 7 ++++++- vms/proposervm/post_fork_option_test.go | 2 ++ vms/proposervm/vm.go | 14 +++++++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/vms/proposervm/post_fork_block.go b/vms/proposervm/post_fork_block.go index 28d127d33f70..0e6f8f39600e 100644 --- a/vms/proposervm/post_fork_block.go +++ b/vms/proposervm/post_fork_block.go @@ -33,7 +33,12 @@ func (b *postForkBlock) Accept(ctx context.Context) error { func (b *postForkBlock) acceptOuterBlk() error { // Update in-memory references b.status = choices.Accepted - b.vm.lastAcceptedTime = b.Timestamp() + + // following backfill introduction we may store past + // blocks, hence safeguard lastAcceptedTime + if b.Timestamp().After(b.vm.lastAcceptedTime) { + b.vm.lastAcceptedTime = b.Timestamp() + } return b.vm.acceptPostForkBlock(b) } diff --git a/vms/proposervm/post_fork_option_test.go b/vms/proposervm/post_fork_option_test.go index d713faffb551..d801d9ff8678 100644 --- a/vms/proposervm/post_fork_option_test.go +++ b/vms/proposervm/post_fork_option_test.go @@ -576,6 +576,7 @@ func TestOptionTimestampValidity(t *testing.T) { }, BytesV: []byte{2}, ParentV: coreOracleBlkID, + HeightV: coreGenBlk.Height() + 2, TimestampV: coreGenBlk.Timestamp().Add(time.Second), }, &snowman.TestBlock{ @@ -585,6 +586,7 @@ func TestOptionTimestampValidity(t *testing.T) { }, BytesV: []byte{3}, ParentV: coreOracleBlkID, + HeightV: coreGenBlk.Height() + 2, TimestampV: coreGenBlk.Timestamp().Add(time.Second), }, }, diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index a8c17efb7cab..b56341a954e0 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -805,13 +805,17 @@ func (vm *VM) acceptPostForkBlock(blk PostForkBlock) error { height := blk.Height() blkID := blk.ID() - vm.lastAcceptedHeight = height - delete(vm.verifiedBlocks, blkID) + // following backfill introduction we may store past + // blocks, hence safeguard last accepted block data + if height >= vm.lastAcceptedHeight { + vm.lastAcceptedHeight = height + delete(vm.verifiedBlocks, blkID) - // Persist this block, its height index, and its status - if err := vm.State.SetLastAccepted(blkID); err != nil { - return err + if err := vm.State.SetLastAccepted(blkID); err != nil { + return err + } } + if err := vm.State.PutBlock(blk.getStatelessBlk(), choices.Accepted); err != nil { return err } From 4173d31ea4b411be9ece2c818a7925e1337455b0 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Thu, 2 Nov 2023 16:12:36 +0100 Subject: [PATCH 13/25] refactored block backfilling --- vms/proposervm/state_syncable_vm.go | 85 ++++++++++++++++++++--------- vms/proposervm/vm.go | 2 + 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index c90371031ab1..0367d1af56b8 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -177,45 +177,77 @@ func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, uint64, error) } func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, uint64, error) { - var ( - blks = make(map[uint64]Block) - innerBlksBytes = make([][]byte, 0, len(blksBytes)) - ) + blks := make(map[uint64]Block) + // 1. Parse for i, blkBytes := range blksBytes { blk, err := vm.parseBlock(ctx, blkBytes) if err != nil { return ids.Empty, 0, fmt.Errorf("failed parsing backfilled block, index %d, %w", i, err) } + blks[blk.Height()] = blk + } - // TODO: introduce validation + // 2. Validate outer blocks + blkHeights := maps.Keys(blks) + sort.Slice(blkHeights, func(i, j int) bool { + return blkHeights[i] < blkHeights[j] // sort in ascending order by heights + }) - blks[blk.Height()] = blk - innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) + topBlk, err := vm.getBlock(ctx, vm.latestBackfilledBlock) + if err != nil { + return ids.Empty, 0, fmt.Errorf("failed retrieving latest backfilled block, %s, %w", vm.latestBackfilledBlock, err) + } + for i := len(blkHeights) - 1; i >= 0; i-- { + blk := blks[blkHeights[i]] + if topBlk.Parent() != blk.ID() { + return ids.Empty, 0, fmt.Errorf("unexpected backfilled block %s, expected child' parent is %s", blk.ID(), topBlk.Parent()) + } + if err := blk.acceptOuterBlk(); err != nil { + return ids.Empty, 0, fmt.Errorf("failed indexing backfilled block, blkID %s, %w", blk.ID(), err) + } + topBlk = blk } + // 3. Backfill inner blocks to innerVM + innerBlksBytes := make([][]byte, 0, len(blksBytes)) + for _, blk := range blks { + innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) + } _, nextInnerBlkHeight, err := vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) - if err == nil { - // no error signals at least a single block has been accepted by the innerVM. (not necessarily all of them) - // Find out which blocks have been accepted and store them. - // Should the process err while looping there will be innerVM blocks not indexed by proposerVM. Repair will be - // carried out upon restart. - - // Make sure to sort blocks by height to accept them in the right order - blkHeights := maps.Keys(blks) - sort.Slice(blkHeights, func(i, j int) bool { - return blkHeights[i] < blkHeights[j] // sort in ascending order - }) - - for _, h := range blkHeights { - blk := blks[h] - _, err := vm.ChainVM.GetBlock(ctx, blk.getInnerBlk().ID()) - if err != nil { - continue + switch err { + case block.ErrStopBlockBackfilling: + return ids.Empty, 0, err // done backfilling + case nil: + // check alignment + default: + return ids.Empty, 0, fmt.Errorf("failed inner VM block backfilling, %w", err) + } + + // 4. Check alignment + blkReversalHappened := false + for _, blk := range blks { + innerBlkID := blk.getInnerBlk().ID() + switch _, err := vm.ChainVM.GetBlock(ctx, innerBlkID); err { + case nil: + continue + case database.ErrNotFound: + // revert block acceptance since innerVM block has not been accepted + if err := vm.State.DeleteBlock(blk.ID()); err != nil { + return ids.Empty, 0, fmt.Errorf("failed reverting backfilled VM block %s, %w", blk.ID(), err) } - if err := blk.acceptOuterBlk(); err != nil { - return ids.Empty, 0, fmt.Errorf("failed indexing backfilled block, blkID %s, %w", blk.ID(), err) + if err := vm.State.DeleteBlockIDAtHeight(blk.Height()); err != nil { + return ids.Empty, 0, fmt.Errorf("failed reverting backfilled VM block from height index %s, %w", blk.ID(), err) } + blkReversalHappened = true + default: + return ids.Empty, 0, fmt.Errorf("failed checking innerVM block %s, %w", innerBlkID, err) + } + } + + if blkReversalHappened { + if err := vm.db.Commit(); err != nil { + return ids.Empty, 0, fmt.Errorf("failed committing backfilled blocks reversal, %w", err) } } @@ -234,5 +266,6 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) return ids.Empty, 0, fmt.Errorf("failed retrieving proposer block %s: %w", childBlkID, err) } + vm.latestBackfilledBlock = childBlkID return childBlk.Parent(), innerBlkHeight, nil } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index b56341a954e0..f3d304e36897 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -135,6 +135,8 @@ type VM struct { lastAcceptedHeight uint64 stateSyncDone utils.Atomic[bool] + + latestBackfilledBlock ids.ID } // New performs best when [minBlkDelay] is whole seconds. This is because block From d937723300129f222f951f9a11f1c7f8dff400e7 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Thu, 2 Nov 2023 16:45:39 +0100 Subject: [PATCH 14/25] stored last backfilled block --- vms/proposervm/state/block_height_index.go | 17 +++++++++++-- vms/proposervm/state/mock_state.go | 29 ++++++++++++++++++++++ vms/proposervm/state_syncable_vm.go | 7 ++++++ vms/proposervm/vm.go | 5 ++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/vms/proposervm/state/block_height_index.go b/vms/proposervm/state/block_height_index.go index e16100bddd89..807047d7b0f2 100644 --- a/vms/proposervm/state/block_height_index.go +++ b/vms/proposervm/state/block_height_index.go @@ -19,8 +19,9 @@ var ( heightPrefix = []byte("height") metadataPrefix = []byte("metadata") - forkKey = []byte("fork") - checkpointKey = []byte("checkpoint") + forkKey = []byte("fork") + checkpointKey = []byte("checkpoint") + latestBackfilledKey = []byte("latestBackfilled") ) type HeightIndexGetter interface { @@ -32,12 +33,16 @@ type HeightIndexGetter interface { // Fork height is stored when the first post-fork block/option is accepted. // Before that, fork height won't be found. GetForkHeight() (uint64, error) + + GetLastBackfilledBlkID() (ids.ID, error) } type HeightIndexWriter interface { SetForkHeight(height uint64) error SetBlockIDAtHeight(height uint64, blkID ids.ID) error DeleteBlockIDAtHeight(height uint64) error + + SetLastBackfilledBlkID(blkID ids.ID) error } // A checkpoint is the blockID of the next block to be considered @@ -139,3 +144,11 @@ func (hi *heightIndex) SetCheckpoint(blkID ids.ID) error { func (hi *heightIndex) DeleteCheckpoint() error { return hi.metadataDB.Delete(checkpointKey) } + +func (hi *heightIndex) SetLastBackfilledBlkID(blkID ids.ID) error { + return database.PutID(hi.metadataDB, latestBackfilledKey, blkID) +} + +func (hi *heightIndex) GetLastBackfilledBlkID() (ids.ID, error) { + return database.GetID(hi.metadataDB, latestBackfilledKey) +} diff --git a/vms/proposervm/state/mock_state.go b/vms/proposervm/state/mock_state.go index 40ef830a1365..b6ceefa73521 100644 --- a/vms/proposervm/state/mock_state.go +++ b/vms/proposervm/state/mock_state.go @@ -185,6 +185,21 @@ func (mr *MockStateMockRecorder) GetLastAccepted() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastAccepted", reflect.TypeOf((*MockState)(nil).GetLastAccepted)) } +// GetLastBackfilledBlkID mocks base method. +func (m *MockState) GetLastBackfilledBlkID() (ids.ID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLastBackfilledBlkID") + ret0, _ := ret[0].(ids.ID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLastBackfilledBlkID indicates an expected call of GetLastBackfilledBlkID. +func (mr *MockStateMockRecorder) GetLastBackfilledBlkID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastBackfilledBlkID", reflect.TypeOf((*MockState)(nil).GetLastBackfilledBlkID)) +} + // GetMinimumHeight mocks base method. func (m *MockState) GetMinimumHeight() (uint64, error) { m.ctrl.T.Helper() @@ -269,3 +284,17 @@ func (mr *MockStateMockRecorder) SetLastAccepted(arg0 interface{}) *gomock.Call mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastAccepted", reflect.TypeOf((*MockState)(nil).SetLastAccepted), arg0) } + +// SetLastBackfilledBlkID mocks base method. +func (m *MockState) SetLastBackfilledBlkID(arg0 ids.ID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLastBackfilledBlkID", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLastBackfilledBlkID indicates an expected call of SetLastBackfilledBlkID. +func (mr *MockStateMockRecorder) SetLastBackfilledBlkID(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastBackfilledBlkID", reflect.TypeOf((*MockState)(nil).SetLastBackfilledBlkID), arg0) +} diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 0367d1af56b8..8276defa5129 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -267,5 +267,12 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) } vm.latestBackfilledBlock = childBlkID + if err := vm.State.SetLastBackfilledBlkID(childBlkID); err != nil { + return ids.Empty, 0, fmt.Errorf("failed storing last backfilled block ID, %w", err) + } + if err := vm.db.Commit(); err != nil { + return ids.Empty, 0, fmt.Errorf("failed committing backfilled blocks reversal, %w", err) + } + return childBlk.Parent(), innerBlkHeight, nil } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index f3d304e36897..8f4e3876020a 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -265,6 +265,11 @@ func (vm *VM) Initialize( return err } + vm.latestBackfilledBlock, err = vm.State.GetLastBackfilledBlkID() + if err != nil && err != database.ErrNotFound { + return fmt.Errorf("failed loading last backfilled block, %w", err) + } + forkHeight, err := vm.getForkHeight() switch err { case nil: From 13a73a2c5a97ecc1f568c50ef0d5454bbe471063 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Thu, 2 Nov 2023 17:21:09 +0100 Subject: [PATCH 15/25] wip: backfilled blocks repairing --- vms/proposervm/state_syncable_vm.go | 24 +++++++------- vms/proposervm/vm.go | 51 +++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 8276defa5129..bcec8ad5c64d 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -225,32 +225,20 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u } // 4. Check alignment - blkReversalHappened := false for _, blk := range blks { innerBlkID := blk.getInnerBlk().ID() switch _, err := vm.ChainVM.GetBlock(ctx, innerBlkID); err { case nil: continue case database.ErrNotFound: - // revert block acceptance since innerVM block has not been accepted - if err := vm.State.DeleteBlock(blk.ID()); err != nil { - return ids.Empty, 0, fmt.Errorf("failed reverting backfilled VM block %s, %w", blk.ID(), err) - } - if err := vm.State.DeleteBlockIDAtHeight(blk.Height()); err != nil { + if err := vm.revertBackfilledBlock(blk); err != nil { return ids.Empty, 0, fmt.Errorf("failed reverting backfilled VM block from height index %s, %w", blk.ID(), err) } - blkReversalHappened = true default: return ids.Empty, 0, fmt.Errorf("failed checking innerVM block %s, %w", innerBlkID, err) } } - if blkReversalHappened { - if err := vm.db.Commit(); err != nil { - return ids.Empty, 0, fmt.Errorf("failed committing backfilled blocks reversal, %w", err) - } - } - return vm.nextBlockBackfillData(ctx, nextInnerBlkHeight) } @@ -276,3 +264,13 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) return childBlk.Parent(), innerBlkHeight, nil } + +func (vm *VM) revertBackfilledBlock(blk Block) error { + if err := vm.State.DeleteBlock(blk.ID()); err != nil { + return fmt.Errorf("failed reverting backfilled VM block %s, %w", blk.ID(), err) + } + if err := vm.State.DeleteBlockIDAtHeight(blk.Height()); err != nil { + return fmt.Errorf("failed reverting backfilled VM block from height index %s, %w", blk.ID(), err) + } + return nil +} diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 8f4e3876020a..5735f71f8bd8 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -265,9 +265,8 @@ func (vm *VM) Initialize( return err } - vm.latestBackfilledBlock, err = vm.State.GetLastBackfilledBlkID() - if err != nil && err != database.ErrNotFound { - return fmt.Errorf("failed loading last backfilled block, %w", err) + if err := vm.repairBlockBackfilling(ctx); err != nil { + return err } forkHeight, err := vm.getForkHeight() @@ -288,6 +287,52 @@ func (vm *VM) Initialize( return nil } +func (vm *VM) repairBlockBackfilling(ctx context.Context) error { + bottomBlkID, err := vm.State.GetLastBackfilledBlkID() + switch { + case errors.Is(err, database.ErrNotFound): + // vm never backfilled blocks. + return nil + case err != nil: + return fmt.Errorf("failed loading last backfilled block ID %s, %w", bottomBlkID, err) + default: + // check alignment + } + + for { + blk, err := vm.getBlock(ctx, bottomBlkID) + if err != nil { + return fmt.Errorf("failed retrieving latest backfilled block, %s: %w", bottomBlkID, err) + } + + var ( + innerBlkID = blk.getInnerBlk().ID() + childBlkHeight = blk.Height() + 1 + ) + + _, err = vm.ChainVM.GetBlock(ctx, innerBlkID) + switch err { + case nil: + if err := vm.db.Commit(); err != nil { + return fmt.Errorf("failed committing backfilled blocks reversal, %w", err) + } + return nil // proposerVM and innerVM aligned + case database.ErrNotFound: + if err := vm.revertBackfilledBlock(blk); err != nil { + return err + } + childBlkID, err := vm.GetBlockIDAtHeight(ctx, childBlkHeight) + if err != nil { + return fmt.Errorf("failed retrieving blkID at height %d, while repairing backfilled blocks: %w", childBlkHeight, err) + } + bottomBlkID = childBlkID + default: + return fmt.Errorf("failed retrieving inner vm blk id %s, while repairing backfilled blocks: %w", innerBlkID, err) + } + vm.latestBackfilledBlock = bottomBlkID + } +} + // shutdown ops then propagate shutdown to innerVM func (vm *VM) Shutdown(ctx context.Context) error { vm.onShutdown() From 0ead0a5cb61abc74075a3f7cef3cf99e1ab888a2 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Thu, 2 Nov 2023 21:23:19 +0100 Subject: [PATCH 16/25] nit --- vms/proposervm/state_sync_block_backfilling_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index 65879c74e377..a8b3c125bebc 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -161,13 +161,13 @@ func TestBlockBackfillSuccess(t *testing.T) { return nil, database.ErrNotFound } } + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } fromInnerVMCh <- common.StateSyncDone <-toEngineCh - innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { - return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil - } blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) require.Equal(proTopBlk.ID(), blkID) @@ -275,13 +275,13 @@ func TestBlockBackfillPartialSuccess(t *testing.T) { return nil, database.ErrNotFound } } + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } fromInnerVMCh <- common.StateSyncDone <-toEngineCh - innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { - return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil - } blkID, height, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) require.Equal(proTopBlk.ID(), blkID) From 8ac3a71192b5a91da0ba1e29d2e70df8f1a510d5 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Fri, 3 Nov 2023 18:07:36 +0100 Subject: [PATCH 17/25] wip: added preFork UTs for block backfilling --- .../state_sync_block_backfilling_test.go | 148 ++++++++++++++---- vms/proposervm/state_syncable_vm.go | 28 ++-- 2 files changed, 139 insertions(+), 37 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index a8b3c125bebc..e4a955017e0e 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -24,7 +24,87 @@ import ( statelessblock "github.com/ava-labs/avalanchego/vms/proposervm/block" ) -func TestBlockBackfillEnabled(t *testing.T) { +func TestBlockBackfillEnabledPreFork(t *testing.T) { + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + // 1. Accept a State summary + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: 100, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + + // Block backfilling not enabled before state sync is accepted + ctx := context.Background() + _, _, err := vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + _, err = stateSummary.Accept(ctx) + require.NoError(err) + + // 2. Signal to the ProposerVM that state sync is done + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + // Block backfilling not enabled before innerVM declares state sync done + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + // 3. Finally check that block backfilling is enabled looking at innerVM + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + innerVM.BackfillBlocksEnabledF = func(_ context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(innerStateSyncedBlk.Parent(), blkID) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) +} + +func TestBlockBackfillEnabledPostFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) @@ -51,7 +131,7 @@ func TestBlockBackfillEnabled(t *testing.T) { HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary := createTestStateSummary(t, vm, proVMParentStateSummaryBlk, forkHeight, innerVM, innerSummary, innerStateSyncedBlk) + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) // Block backfilling not enabled before state sync is accepted ctx := context.Background() @@ -105,7 +185,7 @@ func TestBlockBackfillEnabled(t *testing.T) { require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) } -func TestBlockBackfillSuccess(t *testing.T) { +func TestBlockBackfillPostForkSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) @@ -120,7 +200,7 @@ func TestBlockBackfillSuccess(t *testing.T) { startBlkHeight = uint64(1492) // create a list of consecutive blocks and build state summary of top of them - proBlks, innerBlks = createTestBlocks(t, vm, blkCount, startBlkHeight) + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) innerTopBlk = innerBlks[len(innerBlks)-1] proTopBlk = proBlks[len(proBlks)-1] @@ -140,7 +220,7 @@ func TestBlockBackfillSuccess(t *testing.T) { HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary := createTestStateSummary(t, vm, proTopBlk.ID(), forkHeight, innerVM, innerSummary, innerStateSyncedBlk) + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil @@ -219,7 +299,7 @@ func TestBlockBackfillSuccess(t *testing.T) { } } -func TestBlockBackfillPartialSuccess(t *testing.T) { +func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) @@ -234,7 +314,7 @@ func TestBlockBackfillPartialSuccess(t *testing.T) { startBlkHeight = uint64(1492) // create a list of consecutive blocks and build state summary of top of them - proBlks, innerBlks = createTestBlocks(t, vm, blkCount, startBlkHeight) + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) innerTopBlk = innerBlks[len(innerBlks)-1] proTopBlk = proBlks[len(proBlks)-1] @@ -254,7 +334,7 @@ func TestBlockBackfillPartialSuccess(t *testing.T) { HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary := createTestStateSummary(t, vm, proTopBlk.ID(), forkHeight, innerVM, innerSummary, innerStateSyncedBlk) + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil @@ -350,6 +430,7 @@ func TestBlockBackfillPartialSuccess(t *testing.T) { func createTestBlocks( t *testing.T, vm *VM, + forkHeight uint64, blkCount int, startBlkHeight uint64, ) ( @@ -362,12 +443,14 @@ func createTestBlocks( latestProBlkID = ids.GenerateTestID() dummyBlkTime = time.Now() - dummyPChainHeight = uint64(1492) + dummyPChainHeight = startBlkHeight / 2 innerBlks = make([]snowman.Block, 0, blkCount) proBlks = make([]snowman.Block, 0, blkCount) ) for idx := 0; idx < blkCount; idx++ { + blkHeight := startBlkHeight + uint64(idx) + rndBytes := ids.GenerateTestID() innerBlkTop := &snowman.TestBlock{ TestDecidable: choices.TestDecidable{ @@ -381,41 +464,50 @@ func createTestBlocks( latestInnerBlkID = innerBlkTop.ID() innerBlks = append(innerBlks, innerBlkTop) - statelessChild, err := statelessblock.BuildUnsigned( - latestProBlkID, - dummyBlkTime, - dummyPChainHeight, - innerBlkTop.Bytes(), - ) - require.NoError(err) - proBlkTop := &postForkBlock{ - SignedBlock: statelessChild, - postForkCommonComponents: postForkCommonComponents{ - vm: vm, - innerBlk: innerBlkTop, - status: choices.Processing, - }, + if blkHeight < forkHeight { + proBlks = append(proBlks, &preForkBlock{ + vm: vm, + Block: innerBlkTop, + }) + } else { + statelessChild, err := statelessblock.BuildUnsigned( + latestProBlkID, + dummyBlkTime, + dummyPChainHeight, + innerBlkTop.Bytes(), + ) + require.NoError(err) + proBlkTop := &postForkBlock{ + SignedBlock: statelessChild, + postForkCommonComponents: postForkCommonComponents{ + vm: vm, + innerBlk: innerBlkTop, + status: choices.Processing, + }, + } + latestProBlkID = proBlkTop.ID() + proBlks = append(proBlks, proBlkTop) } - latestProBlkID = proBlkTop.ID() - proBlks = append(proBlks, proBlkTop) } return proBlks, innerBlks } -func createTestStateSummary( +func createPostForkStateSummary( t *testing.T, vm *VM, - proVMParentStateSummaryBlk ids.ID, forkHeight uint64, + proVMParentStateSummaryBlk ids.ID, innerVM *fullVM, innerSummary *block.TestStateSummary, innerBlk *snowman.TestBlock, ) block.StateSummary { require := require.New(t) + + pchainHeight := innerBlk.Height() / 2 slb, err := statelessblock.Build( proVMParentStateSummaryBlk, innerBlk.Timestamp(), - 100, // pChainHeight, + pchainHeight, vm.stakingCertLeaf, innerBlk.Bytes(), vm.ctx.ChainID, diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index bcec8ad5c64d..188a72759119 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -13,6 +13,7 @@ import ( "github.com/ava-labs/avalanchego/database" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow/consensus/snowman" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" "github.com/ava-labs/avalanchego/vms/proposervm/summary" ) @@ -249,17 +250,26 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) return ids.Empty, 0, fmt.Errorf("failed retrieving proposer block ID at height %d: %w", childBlkHeight, err) } - childBlk, err := vm.getBlock(ctx, childBlkID) - if err != nil { + var childBlk snowman.Block + childBlk, err = vm.getPostForkBlock(ctx, childBlkID) + switch err { + case nil: + vm.latestBackfilledBlock = childBlkID + if err := vm.State.SetLastBackfilledBlkID(childBlkID); err != nil { + return ids.Empty, 0, fmt.Errorf("failed storing last backfilled block ID, %w", err) + } + if err := vm.db.Commit(); err != nil { + return ids.Empty, 0, fmt.Errorf("failed committing backfilled blocks reversal, %w", err) + } + case database.ErrNotFound: + // proposerVM may not be active yet. + childBlk, err = vm.getPreForkBlock(ctx, childBlkID) + if err != nil { + return ids.Empty, 0, fmt.Errorf("failed retrieving innerVM block %s: %w", childBlkID, err) + } + default: return ids.Empty, 0, fmt.Errorf("failed retrieving proposer block %s: %w", childBlkID, err) - } - vm.latestBackfilledBlock = childBlkID - if err := vm.State.SetLastBackfilledBlkID(childBlkID); err != nil { - return ids.Empty, 0, fmt.Errorf("failed storing last backfilled block ID, %w", err) - } - if err := vm.db.Commit(); err != nil { - return ids.Empty, 0, fmt.Errorf("failed committing backfilled blocks reversal, %w", err) } return childBlk.Parent(), innerBlkHeight, nil From 4ac552b7d6a8d1cb2c5b4a2f0d12776441f619db Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Fri, 3 Nov 2023 20:41:19 +0100 Subject: [PATCH 18/25] wip: some more preFork UTs for block backfilling --- .../state_sync_block_backfilling_test.go | 275 ++++++++++++++++++ vms/proposervm/state_syncable_vm.go | 27 +- vms/proposervm/vm.go | 3 + 3 files changed, 299 insertions(+), 6 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index e4a955017e0e..63cb077e9cd7 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -24,6 +24,280 @@ import ( statelessblock "github.com/ava-labs/avalanchego/vms/proposervm/block" ) +// Pre Fork section +func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(2000) + blkCount = 10 + startBlkHeight = uint64(100) + + // create a list of consecutive blocks and build state summary of top of them + // proBlks should all be preForkBlocks + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + preForkTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(preForkTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + // simulate that lower half of backfilled blocks won't be accepted by innerVM + idx := len(innerBlks) / 2 + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID != blk.ID() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return nil, database.ErrNotFound + } + return blk, nil + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height != blk.Height() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return ids.Empty, database.ErrNotFound + } + return blk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + // assume lowest half blocks fails verification + return innerBlks[idx].ID(), innerBlks[idx].Height(), nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[idx].ID(), nextBlkID) + require.Equal(proBlks[idx].Height(), nextBlkHeight) + + // check only upper half of blocks have been indexed + for i, blk := range proBlks { + if i <= idx { + _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.ErrorIs(err, database.ErrNotFound) + + _, err = vm.GetBlock(ctx, blk.ID()) + require.ErrorIs(err, database.ErrNotFound) + } else { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } + } +} + +func TestBlockBackfillPreForkSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(2000) + blkCount = 10 + startBlkHeight = uint64(100) + + // create a list of consecutive blocks and build state summary of top of them + // proBlks should all be preForkBlocks + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + preForkTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(preForkTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID == blk.ID() { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height == blk.Height() { + return blk.ID(), nil + } + } + return ids.Empty, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + lowestblk := innerBlks[0] + for _, blk := range innerBlks { + if blk.Height() < lowestblk.Height() { + lowestblk = blk + } + } + return lowestblk.Parent(), lowestblk.Height() - 1, nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[0].Parent(), nextBlkID) + require.Equal(proBlks[0].Height()-1, nextBlkHeight) + + // check proBlocks have been indexed + for _, blk := range proBlks { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } +} + func TestBlockBackfillEnabledPreFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) @@ -104,6 +378,7 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) } +// Post Fork section func TestBlockBackfillEnabledPostFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 188a72759119..bcd651d2cbbf 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -189,17 +189,32 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u blks[blk.Height()] = blk } - // 2. Validate outer blocks + // 2. Validate blocks, checking that they are continguous blkHeights := maps.Keys(blks) sort.Slice(blkHeights, func(i, j int) bool { return blkHeights[i] < blkHeights[j] // sort in ascending order by heights }) - topBlk, err := vm.getBlock(ctx, vm.latestBackfilledBlock) - if err != nil { - return ids.Empty, 0, fmt.Errorf("failed retrieving latest backfilled block, %s, %w", vm.latestBackfilledBlock, err) + var ( + topBlk = blks[blkHeights[len(blkHeights)-1]] + topIdx = len(blkHeights) - 2 + ) + + // vm.latestBackfilledBlock is non nil only if proposerVM has forked + if vm.latestBackfilledBlock != ids.Empty { + latestBackfilledBlk, err := vm.getBlock(ctx, vm.latestBackfilledBlock) + if err != nil { + return ids.Empty, 0, fmt.Errorf("failed retrieving latest backfilled block, %s, %w", vm.latestBackfilledBlock, err) + } + if latestBackfilledBlk.Parent() != topBlk.ID() { + return ids.Empty, 0, fmt.Errorf("unexpected backfilled block %s, expected child' parent is %s", topBlk.ID(), latestBackfilledBlk.Parent()) + } + + topBlk = latestBackfilledBlk + topIdx = len(blkHeights) - 1 } - for i := len(blkHeights) - 1; i >= 0; i-- { + + for i := topIdx; i >= 0; i-- { blk := blks[blkHeights[i]] if topBlk.Parent() != blk.ID() { return ids.Empty, 0, fmt.Errorf("unexpected backfilled block %s, expected child' parent is %s", blk.ID(), topBlk.Parent()) @@ -272,7 +287,7 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) } - return childBlk.Parent(), innerBlkHeight, nil + return childBlk.Parent(), childBlk.Height() - 1, nil } func (vm *VM) revertBackfilledBlock(blk Block) error { diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index ed85d4514151..137263e1ee8d 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -135,6 +135,9 @@ type VM struct { stateSyncDone utils.Atomic[bool] + // latestBackfilledBlock track the latest post fork block + // indexed via block backfilling. Will be ids.Empty if proposerVM + // fork is not active latestBackfilledBlock ids.ID } From 380a318c0a04ba7b5aee3621786b06c5eab8f363 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Fri, 3 Nov 2023 21:03:37 +0100 Subject: [PATCH 19/25] nit --- .../state_sync_block_backfilling_test.go | 242 +++++++++--------- vms/proposervm/state_syncable_vm.go | 1 - 2 files changed, 121 insertions(+), 122 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index 63cb077e9cd7..64f4ef7699e2 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -25,8 +25,7 @@ import ( ) // Pre Fork section -func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { - // setup VM with backfill enabled +func TestBlockBackfillEnabledPreFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) @@ -34,41 +33,32 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { require.NoError(vm.Shutdown(context.Background())) }() - var ( - forkHeight = uint64(2000) - blkCount = 10 - startBlkHeight = uint64(100) - - // create a list of consecutive blocks and build state summary of top of them - // proBlks should all be preForkBlocks - proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) - - innerTopBlk = innerBlks[len(innerBlks)-1] - preForkTopBlk = proBlks[len(proBlks)-1] - stateSummaryHeight = innerTopBlk.Height() + 1 - ) - + // 1. Accept a State summary stateSummary := &block.TestStateSummary{ IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, - HeightV: stateSummaryHeight, + HeightV: 100, BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, } innerStateSyncedBlk := &snowman.TestBlock{ TestDecidable: choices.TestDecidable{ IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, }, - ParentV: innerTopBlk.ID(), HeightV: stateSummary.Height(), BytesV: []byte("inner state synced block"), } + + // Block backfilling not enabled before state sync is accepted + ctx := context.Background() + _, _, err := vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } - - ctx := context.Background() - _, err := stateSummary.Accept(ctx) + _, err = stateSummary.Accept(ctx) require.NoError(err) + // 2. Signal to the ProposerVM that state sync is done innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { return innerStateSyncedBlk.ID(), nil } @@ -81,10 +71,21 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { } } + // Block backfilling not enabled before innerVM declares state sync done + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + fromInnerVMCh <- common.StateSyncDone <-toEngineCh + // 3. Finally check that block backfilling is enabled looking at innerVM innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + innerVM.BackfillBlocksEnabledF = func(_ context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { @@ -93,81 +94,15 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { } return ids.Empty, database.ErrNotFound } - blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) - require.Equal(preForkTopBlk.ID(), blkID) - - // Backfill some blocks - innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { - for _, blk := range innerBlks { - if bytes.Equal(b, blk.Bytes()) { - return blk, nil - } - } - return nil, database.ErrNotFound - } - // simulate that lower half of backfilled blocks won't be accepted by innerVM - idx := len(innerBlks) / 2 - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - for _, blk := range innerBlks { - if blkID != blk.ID() { - continue - } - // if it's one of the lower half blocks, assume it's not stored - // since it was rejected - if blk.Height() <= innerBlks[idx].Height() { - return nil, database.ErrNotFound - } - return blk, nil - } - return nil, database.ErrNotFound - } - innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { - for _, blk := range innerBlks { - if height != blk.Height() { - continue - } - // if it's one of the lower half blocks, assume it's not stored - // since it was rejected - if blk.Height() <= innerBlks[idx].Height() { - return ids.Empty, database.ErrNotFound - } - return blk.ID(), nil - } - return ids.Empty, database.ErrNotFound - } - innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { - // assume lowest half blocks fails verification - return innerBlks[idx].ID(), innerBlks[idx].Height(), nil - } - - blkBytes := make([][]byte, 0, len(proBlks)) - for _, blk := range proBlks { - blkBytes = append(blkBytes, blk.Bytes()) - } - nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) - require.NoError(err) - require.Equal(proBlks[idx].ID(), nextBlkID) - require.Equal(proBlks[idx].Height(), nextBlkHeight) - - // check only upper half of blocks have been indexed - for i, blk := range proBlks { - if i <= idx { - _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) - require.ErrorIs(err, database.ErrNotFound) - - _, err = vm.GetBlock(ctx, blk.ID()) - require.ErrorIs(err, database.ErrNotFound) - } else { - blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) - require.NoError(err) - require.Equal(blk.ID(), blkID) + require.Equal(innerStateSyncedBlk.Parent(), blkID) - _, err = vm.GetBlock(ctx, blkID) - require.NoError(err) - } + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) } func TestBlockBackfillPreForkSuccess(t *testing.T) { @@ -181,7 +116,7 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { var ( forkHeight = uint64(2000) - blkCount = 10 + blkCount = 8 startBlkHeight = uint64(100) // create a list of consecutive blocks and build state summary of top of them @@ -298,7 +233,8 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { } } -func TestBlockBackfillEnabledPreFork(t *testing.T) { +func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { + // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) @@ -306,32 +242,41 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { require.NoError(vm.Shutdown(context.Background())) }() - // 1. Accept a State summary + var ( + forkHeight = uint64(2000) + blkCount = 10 + startBlkHeight = uint64(100) + + // create a list of consecutive blocks and build state summary of top of them + // proBlks should all be preForkBlocks + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + preForkTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + stateSummary := &block.TestStateSummary{ IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, - HeightV: 100, + HeightV: stateSummaryHeight, BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, } innerStateSyncedBlk := &snowman.TestBlock{ TestDecidable: choices.TestDecidable{ IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, }, + ParentV: innerTopBlk.ID(), HeightV: stateSummary.Height(), BytesV: []byte("inner state synced block"), } - - // Block backfilling not enabled before state sync is accepted - ctx := context.Background() - _, _, err := vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } - _, err = stateSummary.Accept(ctx) + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) require.NoError(err) - // 2. Signal to the ProposerVM that state sync is done innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { return innerStateSyncedBlk.ID(), nil } @@ -344,21 +289,10 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { } } - // Block backfilling not enabled before innerVM declares state sync done - _, _, err = vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - fromInnerVMCh <- common.StateSyncDone <-toEngineCh - // 3. Finally check that block backfilling is enabled looking at innerVM innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { - return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled - } - _, _, err = vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - - innerVM.BackfillBlocksEnabledF = func(_ context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { @@ -367,15 +301,81 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { } return ids.Empty, database.ErrNotFound } + blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) - require.Equal(innerStateSyncedBlk.Parent(), blkID) + require.Equal(preForkTopBlk.ID(), blkID) - innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { - return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + // simulate that lower half of backfilled blocks won't be accepted by innerVM + idx := len(innerBlks) / 2 + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID != blk.ID() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return nil, database.ErrNotFound + } + return blk, nil + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height != blk.Height() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return ids.Empty, database.ErrNotFound + } + return blk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + // assume lowest half blocks fails verification + return innerBlks[idx].ID(), innerBlks[idx].Height(), nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[idx].ID(), nextBlkID) + require.Equal(proBlks[idx].Height(), nextBlkHeight) + + // check only upper half of blocks have been indexed + for i, blk := range proBlks { + if i <= idx { + _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.ErrorIs(err, database.ErrNotFound) + + _, err = vm.GetBlock(ctx, blk.ID()) + require.ErrorIs(err, database.ErrNotFound) + } else { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } } - _, _, err = vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) } // Post Fork section @@ -471,7 +471,7 @@ func TestBlockBackfillPostForkSuccess(t *testing.T) { var ( forkHeight = uint64(100) - blkCount = 10 + blkCount = 12 startBlkHeight = uint64(1492) // create a list of consecutive blocks and build state summary of top of them diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index bcd651d2cbbf..22792d31cf3e 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -284,7 +284,6 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) } default: return ids.Empty, 0, fmt.Errorf("failed retrieving proposer block %s: %w", childBlkID, err) - } return childBlk.Parent(), childBlk.Height() - 1, nil From feecab0aa0ede46112293cd6482ed912ae60ff35 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Fri, 3 Nov 2023 21:22:02 +0100 Subject: [PATCH 20/25] wip: some more acrossFork UTs for block backfilling --- .../state_sync_block_backfilling_test.go | 529 +++++++++++++++--- vms/proposervm/state_syncable_vm.go | 3 - 2 files changed, 440 insertions(+), 92 deletions(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index 64f4ef7699e2..7377307474ca 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -24,8 +24,8 @@ import ( statelessblock "github.com/ava-labs/avalanchego/vms/proposervm/block" ) -// Pre Fork section -func TestBlockBackfillEnabledPreFork(t *testing.T) { +// Post Fork section +func TestBlockBackfillEnabledPostFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) @@ -34,25 +34,32 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { }() // 1. Accept a State summary - stateSummary := &block.TestStateSummary{ + var ( + forkHeight = uint64(100) + stateSummaryHeight = uint64(2023) + proVMParentStateSummaryBlk = ids.GenerateTestID() + ) + + innerSummary := &block.TestStateSummary{ IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, - HeightV: 100, + HeightV: stateSummaryHeight, BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, } innerStateSyncedBlk := &snowman.TestBlock{ TestDecidable: choices.TestDecidable{ IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, }, - HeightV: stateSummary.Height(), + HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) // Block backfilling not enabled before state sync is accepted ctx := context.Background() _, _, err := vm.BackfillBlocksEnabled(ctx) require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } _, err = stateSummary.Accept(ctx) @@ -85,18 +92,12 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { _, _, err = vm.BackfillBlocksEnabled(ctx) require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - innerVM.BackfillBlocksEnabledF = func(_ context.Context) (ids.ID, uint64, error) { + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } - innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { - if height == innerStateSyncedBlk.Height() { - return innerStateSyncedBlk.ID(), nil - } - return ids.Empty, database.ErrNotFound - } blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) - require.Equal(innerStateSyncedBlk.Parent(), blkID) + require.Equal(proVMParentStateSummaryBlk, blkID) innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled @@ -105,7 +106,7 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) } -func TestBlockBackfillPreForkSuccess(t *testing.T) { +func TestBlockBackfillPostForkSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) @@ -115,20 +116,19 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { }() var ( - forkHeight = uint64(2000) - blkCount = 8 - startBlkHeight = uint64(100) + forkHeight = uint64(100) + blkCount = 12 + startBlkHeight = uint64(1492) // create a list of consecutive blocks and build state summary of top of them - // proBlks should all be preForkBlocks proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) innerTopBlk = innerBlks[len(innerBlks)-1] - preForkTopBlk = proBlks[len(proBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] stateSummaryHeight = innerTopBlk.Height() + 1 ) - stateSummary := &block.TestStateSummary{ + innerSummary := &block.TestStateSummary{ IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, HeightV: stateSummaryHeight, BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, @@ -138,10 +138,12 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, }, ParentV: innerTopBlk.ID(), - HeightV: stateSummary.Height(), + HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } @@ -160,23 +162,16 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { return nil, database.ErrNotFound } } - - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } - innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { - if height == innerStateSyncedBlk.Height() { - return innerStateSyncedBlk.ID(), nil - } - return ids.Empty, database.ErrNotFound - } + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) - require.Equal(preForkTopBlk.ID(), blkID) + require.Equal(proTopBlk.ID(), blkID) // Backfill some blocks innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { @@ -195,14 +190,6 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { } return nil, database.ErrNotFound } - innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { - for _, blk := range innerBlks { - if height == blk.Height() { - return blk.ID(), nil - } - } - return ids.Empty, database.ErrNotFound - } innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { lowestblk := innerBlks[0] for _, blk := range innerBlks { @@ -233,7 +220,7 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { } } -func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { +func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) @@ -243,20 +230,19 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { }() var ( - forkHeight = uint64(2000) + forkHeight = uint64(100) blkCount = 10 - startBlkHeight = uint64(100) + startBlkHeight = uint64(1492) // create a list of consecutive blocks and build state summary of top of them - // proBlks should all be preForkBlocks proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) innerTopBlk = innerBlks[len(innerBlks)-1] - preForkTopBlk = proBlks[len(proBlks)-1] + proTopBlk = proBlks[len(proBlks)-1] stateSummaryHeight = innerTopBlk.Height() + 1 ) - stateSummary := &block.TestStateSummary{ + innerSummary := &block.TestStateSummary{ IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, HeightV: stateSummaryHeight, BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, @@ -266,10 +252,12 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, }, ParentV: innerTopBlk.ID(), - HeightV: stateSummary.Height(), + HeightV: innerSummary.Height(), BytesV: []byte("inner state synced block"), } - stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + stateSummary := createPostForkStateSummary(t, vm, forkHeight, proTopBlk.ID(), innerVM, innerSummary, innerStateSyncedBlk) + + innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } @@ -288,23 +276,17 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { return nil, database.ErrNotFound } } - - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } - innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { - if height == innerStateSyncedBlk.Height() { - return innerStateSyncedBlk.ID(), nil - } - return ids.Empty, database.ErrNotFound - } - blkID, _, err := vm.BackfillBlocksEnabled(ctx) + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + blkID, height, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) - require.Equal(preForkTopBlk.ID(), blkID) + require.Equal(proTopBlk.ID(), blkID) + require.Equal(proTopBlk.Height(), height) // Backfill some blocks innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { @@ -315,6 +297,7 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { } return nil, database.ErrNotFound } + // simulate that lower half of backfilled blocks won't be accepted by innerVM idx := len(innerBlks) / 2 innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { @@ -331,20 +314,7 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { } return nil, database.ErrNotFound } - innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { - for _, blk := range innerBlks { - if height != blk.Height() { - continue - } - // if it's one of the lower half blocks, assume it's not stored - // since it was rejected - if blk.Height() <= innerBlks[idx].Height() { - return ids.Empty, database.ErrNotFound - } - return blk.ID(), nil - } - return ids.Empty, database.ErrNotFound - } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { // assume lowest half blocks fails verification return innerBlks[idx].ID(), innerBlks[idx].Height(), nil @@ -378,8 +348,8 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { } } -// Post Fork section -func TestBlockBackfillEnabledPostFork(t *testing.T) { +// Across Fork section +func TestBlockBackfillEnabledAcrossFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) @@ -389,8 +359,8 @@ func TestBlockBackfillEnabledPostFork(t *testing.T) { // 1. Accept a State summary var ( - forkHeight = uint64(100) - stateSummaryHeight = uint64(2023) + forkHeight = uint64(50) + stateSummaryHeight = forkHeight + 1 proVMParentStateSummaryBlk = ids.GenerateTestID() ) @@ -460,7 +430,7 @@ func TestBlockBackfillEnabledPostFork(t *testing.T) { require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) } -func TestBlockBackfillPostForkSuccess(t *testing.T) { +func TestBlockBackfillAcrossForkSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) @@ -471,8 +441,8 @@ func TestBlockBackfillPostForkSuccess(t *testing.T) { var ( forkHeight = uint64(100) - blkCount = 12 - startBlkHeight = uint64(1492) + blkCount = 4 + startBlkHeight = forkHeight - uint64(blkCount)/2 // create a list of consecutive blocks and build state summary of top of them proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) @@ -553,6 +523,14 @@ func TestBlockBackfillPostForkSuccess(t *testing.T) { } return lowestblk.Parent(), lowestblk.Height() - 1, nil } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height == blk.Height() { + return blk.ID(), nil + } + } + return ids.Empty, database.ErrNotFound + } blkBytes := make([][]byte, 0, len(proBlks)) for _, blk := range proBlks { @@ -574,7 +552,7 @@ func TestBlockBackfillPostForkSuccess(t *testing.T) { } } -func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { +func TestBlockBackfillAcrossForkPartialSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) @@ -585,8 +563,12 @@ func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { var ( forkHeight = uint64(100) - blkCount = 10 - startBlkHeight = uint64(1492) + blkCount = 8 + startBlkHeight = forkHeight - uint64(blkCount)/2 + + // simulate that the bottom [idxFailure] blocks will fail + // being pushed in innerVM + idxFailure = 3 // create a list of consecutive blocks and build state summary of top of them proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) @@ -652,6 +634,360 @@ func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { return nil, database.ErrNotFound } + // simulate that lower half of backfilled blocks won't be accepted by innerVM + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID != blk.ID() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idxFailure].Height() { + return nil, database.ErrNotFound + } + return blk, nil + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height != blk.Height() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idxFailure].Height() { + return ids.Empty, database.ErrNotFound + } + return blk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + // assume lowest half blocks fails verification + return innerBlks[idxFailure].ID(), innerBlks[idxFailure].Height(), nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[idxFailure].ID(), nextBlkID) + require.Equal(proBlks[idxFailure].Height(), nextBlkHeight) + + // check only upper half of blocks have been indexed + for i, blk := range proBlks { + if i <= idxFailure { + _, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.ErrorIs(err, database.ErrNotFound) + + _, err = vm.GetBlock(ctx, blk.ID()) + require.ErrorIs(err, database.ErrNotFound) + } else { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } + } +} + +// Pre Fork section +func TestBlockBackfillEnabledPreFork(t *testing.T) { + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + // 1. Accept a State summary + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: 100, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + + // Block backfilling not enabled before state sync is accepted + ctx := context.Background() + _, _, err := vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + _, err = stateSummary.Accept(ctx) + require.NoError(err) + + // 2. Signal to the ProposerVM that state sync is done + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + // Block backfilling not enabled before innerVM declares state sync done + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + // 3. Finally check that block backfilling is enabled looking at innerVM + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) + + innerVM.BackfillBlocksEnabledF = func(_ context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(innerStateSyncedBlk.Parent(), blkID) + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled + } + _, _, err = vm.BackfillBlocksEnabled(ctx) + require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) +} + +func TestBlockBackfillPreForkSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(2000) + blkCount = 8 + startBlkHeight = uint64(100) + + // create a list of consecutive blocks and build state summary of top of them + // proBlks should all be preForkBlocks + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + preForkTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(preForkTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + for _, blk := range innerBlks { + if blkID == blk.ID() { + return blk, nil + } + } + return nil, database.ErrNotFound + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height == blk.Height() { + return blk.ID(), nil + } + } + return ids.Empty, database.ErrNotFound + } + innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { + lowestblk := innerBlks[0] + for _, blk := range innerBlks { + if blk.Height() < lowestblk.Height() { + lowestblk = blk + } + } + return lowestblk.Parent(), lowestblk.Height() - 1, nil + } + + blkBytes := make([][]byte, 0, len(proBlks)) + for _, blk := range proBlks { + blkBytes = append(blkBytes, blk.Bytes()) + } + nextBlkID, nextBlkHeight, err := vm.BackfillBlocks(ctx, blkBytes) + require.NoError(err) + require.Equal(proBlks[0].Parent(), nextBlkID) + require.Equal(proBlks[0].Height()-1, nextBlkHeight) + + // check proBlocks have been indexed + for _, blk := range proBlks { + blkID, err := vm.GetBlockIDAtHeight(ctx, blk.Height()) + require.NoError(err) + require.Equal(blk.ID(), blkID) + + _, err = vm.GetBlock(ctx, blkID) + require.NoError(err) + } +} + +func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { + // setup VM with backfill enabled + require := require.New(t) + toEngineCh := make(chan common.Message) + innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + defer func() { + require.NoError(vm.Shutdown(context.Background())) + }() + + var ( + forkHeight = uint64(2000) + blkCount = 10 + startBlkHeight = uint64(100) + + // create a list of consecutive blocks and build state summary of top of them + // proBlks should all be preForkBlocks + proBlks, innerBlks = createTestBlocks(t, vm, forkHeight, blkCount, startBlkHeight) + + innerTopBlk = innerBlks[len(innerBlks)-1] + preForkTopBlk = proBlks[len(proBlks)-1] + stateSummaryHeight = innerTopBlk.Height() + 1 + ) + + stateSummary := &block.TestStateSummary{ + IDV: ids.ID{'s', 'u', 'm', 'm', 'a', 'r', 'y', 'I', 'D'}, + HeightV: stateSummaryHeight, + BytesV: []byte{'i', 'n', 'n', 'e', 'r'}, + } + innerStateSyncedBlk := &snowman.TestBlock{ + TestDecidable: choices.TestDecidable{ + IDV: ids.ID{'i', 'n', 'n', 'e', 'r', 'S', 'y', 'n', 'c', 'e', 'd'}, + }, + ParentV: innerTopBlk.ID(), + HeightV: stateSummary.Height(), + BytesV: []byte("inner state synced block"), + } + stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { + return block.StateSyncStatic, nil + } + + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) + + innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { + return innerStateSyncedBlk.ID(), nil + } + innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { + switch blkID { + case innerStateSyncedBlk.ID(): + return innerStateSyncedBlk, nil + default: + return nil, database.ErrNotFound + } + } + + fromInnerVMCh <- common.StateSyncDone + <-toEngineCh + + innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { + return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil + } + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + if height == innerStateSyncedBlk.Height() { + return innerStateSyncedBlk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } + + blkID, _, err := vm.BackfillBlocksEnabled(ctx) + require.NoError(err) + require.Equal(preForkTopBlk.ID(), blkID) + + // Backfill some blocks + innerVM.ParseBlockF = func(_ context.Context, b []byte) (snowman.Block, error) { + for _, blk := range innerBlks { + if bytes.Equal(b, blk.Bytes()) { + return blk, nil + } + } + return nil, database.ErrNotFound + } // simulate that lower half of backfilled blocks won't be accepted by innerVM idx := len(innerBlks) / 2 innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { @@ -668,7 +1004,20 @@ func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { } return nil, database.ErrNotFound } - + innerVM.GetBlockIDAtHeightF = func(ctx context.Context, height uint64) (ids.ID, error) { + for _, blk := range innerBlks { + if height != blk.Height() { + continue + } + // if it's one of the lower half blocks, assume it's not stored + // since it was rejected + if blk.Height() <= innerBlks[idx].Height() { + return ids.Empty, database.ErrNotFound + } + return blk.ID(), nil + } + return ids.Empty, database.ErrNotFound + } innerVM.BackfillBlocksF = func(_ context.Context, b [][]byte) (ids.ID, uint64, error) { // assume lowest half blocks fails verification return innerBlks[idx].ID(), innerBlks[idx].Height(), nil @@ -715,7 +1064,6 @@ func createTestBlocks( require := require.New(t) var ( latestInnerBlkID = ids.GenerateTestID() - latestProBlkID = ids.GenerateTestID() dummyBlkTime = time.Now() dummyPChainHeight = startBlkHeight / 2 @@ -745,6 +1093,10 @@ func createTestBlocks( Block: innerBlkTop, }) } else { + latestProBlkID := ids.GenerateTestID() + if len(proBlks) != 0 { + latestProBlkID = proBlks[len(proBlks)-1].ID() + } statelessChild, err := statelessblock.BuildUnsigned( latestProBlkID, dummyBlkTime, @@ -760,7 +1112,6 @@ func createTestBlocks( status: choices.Processing, }, } - latestProBlkID = proBlkTop.ID() proBlks = append(proBlks, proBlkTop) } } diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 22792d31cf3e..809549dd6dd2 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -206,9 +206,6 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u if err != nil { return ids.Empty, 0, fmt.Errorf("failed retrieving latest backfilled block, %s, %w", vm.latestBackfilledBlock, err) } - if latestBackfilledBlk.Parent() != topBlk.ID() { - return ids.Empty, 0, fmt.Errorf("unexpected backfilled block %s, expected child' parent is %s", topBlk.ID(), latestBackfilledBlk.Parent()) - } topBlk = latestBackfilledBlk topIdx = len(blkHeights) - 1 From 374c38580229b9d152c8f7a5d2088da88e719481 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Sat, 4 Nov 2023 00:00:11 +0100 Subject: [PATCH 21/25] dropped state sync done check --- vms/proposervm/scheduler/scheduler.go | 13 +- vms/proposervm/scheduler/scheduler_test.go | 36 ++-- .../state_sync_block_backfilling_test.go | 174 ++---------------- vms/proposervm/state_syncable_vm.go | 2 +- vms/proposervm/vm.go | 4 +- 5 files changed, 38 insertions(+), 191 deletions(-) diff --git a/vms/proposervm/scheduler/scheduler.go b/vms/proposervm/scheduler/scheduler.go index 91ef6d09e546..5946a67b9b77 100644 --- a/vms/proposervm/scheduler/scheduler.go +++ b/vms/proposervm/scheduler/scheduler.go @@ -9,7 +9,6 @@ import ( "go.uber.org/zap" "github.com/ava-labs/avalanchego/snow/engine/common" - "github.com/ava-labs/avalanchego/utils" "github.com/ava-labs/avalanchego/utils/logging" ) @@ -38,18 +37,15 @@ type scheduler struct { // from telling the engine to call its VM's BuildBlock method until the // given time newBuildBlockTime chan time.Time - - stateSyncDone *utils.Atomic[bool] } -func New(log logging.Logger, toEngine chan<- common.Message, stateSyncDone *utils.Atomic[bool]) (Scheduler, chan<- common.Message) { +func New(log logging.Logger, toEngine chan<- common.Message) (Scheduler, chan<- common.Message) { vmToEngine := make(chan common.Message, cap(toEngine)) return &scheduler{ log: log, fromVM: vmToEngine, toEngine: toEngine, newBuildBlockTime: make(chan time.Time), - stateSyncDone: stateSyncDone, }, vmToEngine } @@ -78,10 +74,9 @@ waitloop: for { select { - case msg := <-s.fromVM: // Pass the message up to the engine - if msg == common.StateSyncDone { - s.stateSyncDone.Set(true) - } + case msg := <-s.fromVM: + // Give the engine the message from the VM asking the engine to + // build a block select { case s.toEngine <- msg: default: diff --git a/vms/proposervm/scheduler/scheduler_test.go b/vms/proposervm/scheduler/scheduler_test.go index e83cec258ac9..821a36883e90 100644 --- a/vms/proposervm/scheduler/scheduler_test.go +++ b/vms/proposervm/scheduler/scheduler_test.go @@ -10,18 +10,14 @@ import ( "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/snow/engine/common" - "github.com/ava-labs/avalanchego/utils" "github.com/ava-labs/avalanchego/utils/logging" ) func TestDelayFromNew(t *testing.T) { - var ( - toEngine = make(chan common.Message, 10) - startTime = time.Now().Add(50 * time.Millisecond) - dummyStateSyncDone = utils.Atomic[bool]{} - ) + toEngine := make(chan common.Message, 10) + startTime := time.Now().Add(50 * time.Millisecond) - s, fromVM := New(logging.NoLog{}, toEngine, &dummyStateSyncDone) + s, fromVM := New(logging.NoLog{}, toEngine) defer s.Close() go s.Dispatch(startTime) @@ -32,14 +28,11 @@ func TestDelayFromNew(t *testing.T) { } func TestDelayFromSetTime(t *testing.T) { - var ( - toEngine = make(chan common.Message, 10) - now = time.Now() - startTime = now.Add(50 * time.Millisecond) - dummyStateSyncDone = utils.Atomic[bool]{} - ) - - s, fromVM := New(logging.NoLog{}, toEngine, &dummyStateSyncDone) + toEngine := make(chan common.Message, 10) + now := time.Now() + startTime := now.Add(50 * time.Millisecond) + + s, fromVM := New(logging.NoLog{}, toEngine) defer s.Close() go s.Dispatch(now) @@ -52,14 +45,11 @@ func TestDelayFromSetTime(t *testing.T) { } func TestReceipt(*testing.T) { - var ( - toEngine = make(chan common.Message, 10) - now = time.Now() - startTime = now.Add(50 * time.Millisecond) - dummyStateSyncDone = utils.Atomic[bool]{} - ) - - s, fromVM := New(logging.NoLog{}, toEngine, &dummyStateSyncDone) + toEngine := make(chan common.Message, 10) + now := time.Now() + startTime := now.Add(50 * time.Millisecond) + + s, fromVM := New(logging.NoLog{}, toEngine) defer s.Close() go s.Dispatch(now) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index 7377307474ca..5d755e8b1004 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -28,7 +28,7 @@ import ( func TestBlockBackfillEnabledPostFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -54,38 +54,14 @@ func TestBlockBackfillEnabledPostFork(t *testing.T) { } stateSummary := createPostForkStateSummary(t, vm, forkHeight, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) - // Block backfilling not enabled before state sync is accepted ctx := context.Background() - _, _, err := vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } - _, err = stateSummary.Accept(ctx) + _, err := stateSummary.Accept(ctx) require.NoError(err) - // 2. Signal to the ProposerVM that state sync is done - innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { - return innerStateSyncedBlk.ID(), nil - } - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - switch blkID { - case innerStateSyncedBlk.ID(): - return innerStateSyncedBlk, nil - default: - return nil, database.ErrNotFound - } - } - - // Block backfilling not enabled before innerVM declares state sync done - _, _, err = vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - - // 3. Finally check that block backfilling is enabled looking at innerVM + // 2. Check that block backfilling is enabled looking at innerVM innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled } @@ -110,7 +86,7 @@ func TestBlockBackfillPostForkSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -151,24 +127,10 @@ func TestBlockBackfillPostForkSuccess(t *testing.T) { _, err := stateSummary.Accept(ctx) require.NoError(err) - innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { - return innerStateSyncedBlk.ID(), nil - } - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - switch blkID { - case innerStateSyncedBlk.ID(): - return innerStateSyncedBlk, nil - default: - return nil, database.ErrNotFound - } - } innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) require.Equal(proTopBlk.ID(), blkID) @@ -224,7 +186,7 @@ func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -265,24 +227,10 @@ func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { _, err := stateSummary.Accept(ctx) require.NoError(err) - innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { - return innerStateSyncedBlk.ID(), nil - } - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - switch blkID { - case innerStateSyncedBlk.ID(): - return innerStateSyncedBlk, nil - default: - return nil, database.ErrNotFound - } - } innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - blkID, height, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) require.Equal(proTopBlk.ID(), blkID) @@ -352,7 +300,7 @@ func TestBlockBackfillPostForkPartialSuccess(t *testing.T) { func TestBlockBackfillEnabledAcrossFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -378,38 +326,14 @@ func TestBlockBackfillEnabledAcrossFork(t *testing.T) { } stateSummary := createPostForkStateSummary(t, vm, forkHeight, proVMParentStateSummaryBlk, innerVM, innerSummary, innerStateSyncedBlk) - // Block backfilling not enabled before state sync is accepted - ctx := context.Background() - _, _, err := vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - innerSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } - _, err = stateSummary.Accept(ctx) + ctx := context.Background() + _, err := stateSummary.Accept(ctx) require.NoError(err) - // 2. Signal to the ProposerVM that state sync is done - innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { - return innerStateSyncedBlk.ID(), nil - } - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - switch blkID { - case innerStateSyncedBlk.ID(): - return innerStateSyncedBlk, nil - default: - return nil, database.ErrNotFound - } - } - - // Block backfilling not enabled before innerVM declares state sync done - _, _, err = vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - - // 3. Finally check that block backfilling is enabled looking at innerVM + // 2. Check that block backfilling is enabled looking at innerVM innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled } @@ -434,7 +358,7 @@ func TestBlockBackfillAcrossForkSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -475,24 +399,10 @@ func TestBlockBackfillAcrossForkSuccess(t *testing.T) { _, err := stateSummary.Accept(ctx) require.NoError(err) - innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { - return innerStateSyncedBlk.ID(), nil - } - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - switch blkID { - case innerStateSyncedBlk.ID(): - return innerStateSyncedBlk, nil - default: - return nil, database.ErrNotFound - } - } innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - blkID, _, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) require.Equal(proTopBlk.ID(), blkID) @@ -556,7 +466,7 @@ func TestBlockBackfillAcrossForkPartialSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -601,24 +511,10 @@ func TestBlockBackfillAcrossForkPartialSuccess(t *testing.T) { _, err := stateSummary.Accept(ctx) require.NoError(err) - innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { - return innerStateSyncedBlk.ID(), nil - } - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - switch blkID { - case innerStateSyncedBlk.ID(): - return innerStateSyncedBlk, nil - default: - return nil, database.ErrNotFound - } - } innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - blkID, height, err := vm.BackfillBlocksEnabled(ctx) require.NoError(err) require.Equal(proTopBlk.ID(), blkID) @@ -701,7 +597,7 @@ func TestBlockBackfillAcrossForkPartialSuccess(t *testing.T) { func TestBlockBackfillEnabledPreFork(t *testing.T) { require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -720,38 +616,15 @@ func TestBlockBackfillEnabledPreFork(t *testing.T) { BytesV: []byte("inner state synced block"), } - // Block backfilling not enabled before state sync is accepted - ctx := context.Background() - _, _, err := vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - stateSummary.AcceptF = func(ctx context.Context) (block.StateSyncMode, error) { return block.StateSyncStatic, nil } - _, err = stateSummary.Accept(ctx) - require.NoError(err) - - // 2. Signal to the ProposerVM that state sync is done - innerVM.LastAcceptedF = func(context.Context) (ids.ID, error) { - return innerStateSyncedBlk.ID(), nil - } - innerVM.GetBlockF = func(_ context.Context, blkID ids.ID) (snowman.Block, error) { - switch blkID { - case innerStateSyncedBlk.ID(): - return innerStateSyncedBlk, nil - default: - return nil, database.ErrNotFound - } - } - - // Block backfilling not enabled before innerVM declares state sync done - _, _, err = vm.BackfillBlocksEnabled(ctx) - require.ErrorIs(err, block.ErrBlockBackfillingNotEnabled) - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh + ctx := context.Background() + _, err := stateSummary.Accept(ctx) + require.NoError(err) - // 3. Finally check that block backfilling is enabled looking at innerVM + // 2. Check that block backfilling is enabled looking at innerVM innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled } @@ -782,7 +655,7 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -834,9 +707,6 @@ func TestBlockBackfillPreForkSuccess(t *testing.T) { } } - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } @@ -910,7 +780,7 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { // setup VM with backfill enabled require := require.New(t) toEngineCh := make(chan common.Message) - innerVM, vm, fromInnerVMCh := setupBlockBackfillingVM(t, toEngineCh) + innerVM, vm := setupBlockBackfillingVM(t, toEngineCh) defer func() { require.NoError(vm.Shutdown(context.Background())) }() @@ -962,9 +832,6 @@ func TestBlockBackfillPreForkPartialSuccess(t *testing.T) { } } - fromInnerVMCh <- common.StateSyncDone - <-toEngineCh - innerVM.BackfillBlocksEnabledF = func(ctx context.Context) (ids.ID, uint64, error) { return innerStateSyncedBlk.ID(), innerStateSyncedBlk.Height() - 1, nil } @@ -1164,7 +1031,6 @@ func setupBlockBackfillingVM( ) ( *fullVM, *VM, - chan<- common.Message, ) { require := require.New(t) @@ -1193,12 +1059,10 @@ func setupBlockBackfillingVM( BytesV: []byte("genesis state"), } - toProVMChannel := make(chan<- common.Message) innerVM.InitializeF = func(_ context.Context, _ *snow.Context, _ database.Database, _ []byte, _ []byte, _ []byte, ch chan<- common.Message, _ []*common.Fx, _ common.AppSender, ) error { - toProVMChannel = ch return nil } innerVM.VerifyHeightIndexF = func(context.Context) error { @@ -1237,5 +1101,5 @@ func setupBlockBackfillingVM( nil, )) - return innerVM, vm, toProVMChannel + return innerVM, vm } diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 809549dd6dd2..c92a4cc7e451 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -165,7 +165,7 @@ func (vm *VM) buildStateSummary(ctx context.Context, innerSummary block.StateSum } func (vm *VM) BackfillBlocksEnabled(ctx context.Context) (ids.ID, uint64, error) { - if vm.ssVM == nil || !vm.stateSyncDone.Get() { + if vm.ssVM == nil { return ids.Empty, 0, block.ErrBlockBackfillingNotEnabled } diff --git a/vms/proposervm/vm.go b/vms/proposervm/vm.go index 137263e1ee8d..c7c8439ba319 100644 --- a/vms/proposervm/vm.go +++ b/vms/proposervm/vm.go @@ -133,8 +133,6 @@ type VM struct { // lastAcceptedHeight is set to the last accepted PostForkBlock's height. lastAcceptedHeight uint64 - stateSyncDone utils.Atomic[bool] - // latestBackfilledBlock track the latest post fork block // indexed via block backfilling. Will be ids.Empty if proposerVM // fork is not active @@ -224,7 +222,7 @@ func (vm *VM) Initialize( indexerState := state.New(indexerDB) vm.hIndexer = indexer.NewHeightIndexer(vm, vm.ctx.Log, indexerState) - scheduler, vmToEngine := scheduler.New(vm.ctx.Log, toEngine, &vm.stateSyncDone) + scheduler, vmToEngine := scheduler.New(vm.ctx.Log, toEngine) vm.Scheduler = scheduler vm.toScheduler = vmToEngine From f60c21ead9e79d435807553e129996e6dbccb20d Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Sat, 4 Nov 2023 17:10:26 +0100 Subject: [PATCH 22/25] introduced backfilling internal error --- vms/proposervm/state_syncable_vm.go | 74 ++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index c92a4cc7e451..fbc0fda6a834 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -204,7 +204,12 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u if vm.latestBackfilledBlock != ids.Empty { latestBackfilledBlk, err := vm.getBlock(ctx, vm.latestBackfilledBlock) if err != nil { - return ids.Empty, 0, fmt.Errorf("failed retrieving latest backfilled block, %s, %w", vm.latestBackfilledBlock, err) + return ids.Empty, 0, fmt.Errorf( + "failed retrieving latest backfilled block, %s, %w, %w", + vm.latestBackfilledBlock, + err, + block.ErrInternalBlockBackfilling, + ) } topBlk = latestBackfilledBlk @@ -217,7 +222,12 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u return ids.Empty, 0, fmt.Errorf("unexpected backfilled block %s, expected child' parent is %s", blk.ID(), topBlk.Parent()) } if err := blk.acceptOuterBlk(); err != nil { - return ids.Empty, 0, fmt.Errorf("failed indexing backfilled block, blkID %s, %w", blk.ID(), err) + return ids.Empty, 0, fmt.Errorf( + "failed indexing backfilled block, blkID %s, %w, %w", + blk.ID(), + err, + block.ErrInternalBlockBackfilling, + ) } topBlk = blk } @@ -231,10 +241,12 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u switch err { case block.ErrStopBlockBackfilling: return ids.Empty, 0, err // done backfilling + case block.ErrInternalBlockBackfilling: + return ids.Empty, 0, err case nil: - // check alignment + // check proposerVM and innerVM alignment default: - return ids.Empty, 0, fmt.Errorf("failed inner VM block backfilling, %w", err) + // non-internal error in innerVM, check proposerVM and innerVM alignment } // 4. Check alignment @@ -248,7 +260,12 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u return ids.Empty, 0, fmt.Errorf("failed reverting backfilled VM block from height index %s, %w", blk.ID(), err) } default: - return ids.Empty, 0, fmt.Errorf("failed checking innerVM block %s, %w", innerBlkID, err) + return ids.Empty, 0, fmt.Errorf( + "failed checking innerVM block %s, %w, %w", + innerBlkID, + err, + block.ErrInternalBlockBackfilling, + ) } } @@ -259,7 +276,12 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) childBlkHeight := innerBlkHeight + 1 childBlkID, err := vm.GetBlockIDAtHeight(ctx, childBlkHeight) if err != nil { - return ids.Empty, 0, fmt.Errorf("failed retrieving proposer block ID at height %d: %w", childBlkHeight, err) + return ids.Empty, 0, fmt.Errorf( + "failed retrieving proposer block ID at height %d: %w, %w", + childBlkHeight, + err, + block.ErrInternalBlockBackfilling, + ) } var childBlk snowman.Block @@ -268,19 +290,38 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) case nil: vm.latestBackfilledBlock = childBlkID if err := vm.State.SetLastBackfilledBlkID(childBlkID); err != nil { - return ids.Empty, 0, fmt.Errorf("failed storing last backfilled block ID, %w", err) + return ids.Empty, 0, fmt.Errorf( + "failed storing last backfilled block ID: %w, %w", + err, + block.ErrInternalBlockBackfilling, + ) } if err := vm.db.Commit(); err != nil { - return ids.Empty, 0, fmt.Errorf("failed committing backfilled blocks reversal, %w", err) + return ids.Empty, 0, fmt.Errorf( + "failed committing backfilled blocks reversal: %w, %w", + err, + block.ErrInternalBlockBackfilling, + ) } case database.ErrNotFound: // proposerVM may not be active yet. childBlk, err = vm.getPreForkBlock(ctx, childBlkID) if err != nil { - return ids.Empty, 0, fmt.Errorf("failed retrieving innerVM block %s: %w", childBlkID, err) + return ids.Empty, + 0, + fmt.Errorf("failed retrieving innerVM block %s: %w, %w", + childBlkID, + err, + block.ErrInternalBlockBackfilling, + ) } default: - return ids.Empty, 0, fmt.Errorf("failed retrieving proposer block %s: %w", childBlkID, err) + return ids.Empty, 0, fmt.Errorf( + "failed retrieving proposer block %s: %w, %w", + childBlkID, + err, + block.ErrInternalBlockBackfilling, + ) } return childBlk.Parent(), childBlk.Height() - 1, nil @@ -288,10 +329,19 @@ func (vm *VM) nextBlockBackfillData(ctx context.Context, innerBlkHeight uint64) func (vm *VM) revertBackfilledBlock(blk Block) error { if err := vm.State.DeleteBlock(blk.ID()); err != nil { - return fmt.Errorf("failed reverting backfilled VM block %s, %w", blk.ID(), err) + return fmt.Errorf( + "failed reverting backfilled VM block %s: %w, %w", + blk.ID(), + err, + block.ErrInternalBlockBackfilling) } if err := vm.State.DeleteBlockIDAtHeight(blk.Height()); err != nil { - return fmt.Errorf("failed reverting backfilled VM block from height index %s, %w", blk.ID(), err) + return fmt.Errorf( + "failed reverting backfilled VM block from height index %s: %w, %w", + blk.ID(), + err, + block.ErrInternalBlockBackfilling, + ) } return nil } From 968ddbdb73caed455fcdcffa94ebfd9c12fa1acb Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Sun, 5 Nov 2023 19:03:15 +0100 Subject: [PATCH 23/25] fixed error checking --- vms/proposervm/state_syncable_vm.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index fbc0fda6a834..68a35b1e66bc 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -5,6 +5,7 @@ package proposervm import ( "context" + "errors" "fmt" "sort" @@ -238,12 +239,12 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u innerBlksBytes = append(innerBlksBytes, blk.getInnerBlk().Bytes()) } _, nextInnerBlkHeight, err := vm.ssVM.BackfillBlocks(ctx, innerBlksBytes) - switch err { - case block.ErrStopBlockBackfilling: + switch { + case errors.Is(err, block.ErrStopBlockBackfilling): return ids.Empty, 0, err // done backfilling - case block.ErrInternalBlockBackfilling: + case errors.Is(err, block.ErrInternalBlockBackfilling): return ids.Empty, 0, err - case nil: + case err == nil: // check proposerVM and innerVM alignment default: // non-internal error in innerVM, check proposerVM and innerVM alignment From 181d827cf1a17604d2a19f1e046c722b85f924e6 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Mon, 27 Nov 2023 13:57:40 +0100 Subject: [PATCH 24/25] nit --- vms/proposervm/state_syncable_vm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vms/proposervm/state_syncable_vm.go b/vms/proposervm/state_syncable_vm.go index 68a35b1e66bc..d5d0e3212e04 100644 --- a/vms/proposervm/state_syncable_vm.go +++ b/vms/proposervm/state_syncable_vm.go @@ -193,7 +193,7 @@ func (vm *VM) BackfillBlocks(ctx context.Context, blksBytes [][]byte) (ids.ID, u // 2. Validate blocks, checking that they are continguous blkHeights := maps.Keys(blks) sort.Slice(blkHeights, func(i, j int) bool { - return blkHeights[i] < blkHeights[j] // sort in ascending order by heights + return blkHeights[i] < blkHeights[j] }) var ( From 29c5a5a228f0dc3743364c2e7fc45545149dff38 Mon Sep 17 00:00:00 2001 From: Alberto Benegiamo Date: Wed, 27 Dec 2023 18:12:46 +0100 Subject: [PATCH 25/25] fixed merge --- vms/proposervm/state_sync_block_backfilling_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vms/proposervm/state_sync_block_backfilling_test.go b/vms/proposervm/state_sync_block_backfilling_test.go index 78904d70cc23..73c3fb8d74f5 100644 --- a/vms/proposervm/state_sync_block_backfilling_test.go +++ b/vms/proposervm/state_sync_block_backfilling_test.go @@ -19,6 +19,7 @@ import ( "github.com/ava-labs/avalanchego/snow/consensus/snowman" "github.com/ava-labs/avalanchego/snow/engine/common" "github.com/ava-labs/avalanchego/snow/engine/snowman/block" + "github.com/ava-labs/avalanchego/snow/snowtest" "github.com/ava-labs/avalanchego/vms/proposervm/summary" statelessblock "github.com/ava-labs/avalanchego/vms/proposervm/block" @@ -1088,7 +1089,7 @@ func setupBlockBackfillingVM( }, ) - ctx := snow.DefaultContextTest() + ctx := snowtest.Context(t, snowtest.CChainID) ctx.NodeID = ids.NodeIDFromCert(pTestCert) require.NoError(vm.Initialize(