Defer consensus verification

proto/vm/vm.proto

@@ -81,6 +81,7 @@ service VM {
   rpc GetStateSummary(GetStateSummaryRequest) returns (GetStateSummaryResponse);
 
   // Block
+  rpc BlockVerifyProposer(BlockVerifyProposerRequest) returns (google.protobuf.Empty);
   rpc BlockVerify(BlockVerifyRequest) returns (BlockVerifyResponse);
   rpc BlockAccept(BlockAcceptRequest) returns (google.protobuf.Empty);
   rpc BlockReject(BlockRejectRequest) returns (google.protobuf.Empty);
@@ -215,6 +216,10 @@ message SetPreferenceRequest {
   bytes id = 1;
 }
 
+message BlockVerifyProposerRequest {
+  bytes bytes = 1;
+}
+
 message BlockVerifyRequest {
   bytes bytes = 1;
 

snow/consensus/snowman/block.go

@@ -27,6 +27,8 @@ type Block interface {
 	// Parent returns the ID of this block's parent.
 	Parent() ids.ID
 
+	VerifyProposer(context.Context) error
+
 	// Verify that the state transition this block would make if accepted is
 	// valid. If the state transition is invalid, a non-nil error should be
 	// returned.

snow/consensus/snowman/metrics.go

@@ -41,7 +41,9 @@ type metrics struct {
 	// numProcessing keeps track of the number of processing blocks
 	numProcessing prometheus.Gauge
 
-	blockSizeAcceptedSum prometheus.Gauge
+	blockSizeAcceptedSum       prometheus.Gauge
+	timesVerifiedAcceptedSum   prometheus.Gauge
+	verifiedOnceBlocksAccepted prometheus.Counter
 	// pollsAccepted tracks the number of polls that a block was in processing
 	// for before being accepted
 	pollsAccepted metric.Averager
@@ -50,7 +52,9 @@ type metrics struct {
 	latAccepted          metric.Averager
 	buildLatencyAccepted prometheus.Gauge
 
-	blockSizeRejectedSum prometheus.Gauge
+	blockSizeRejectedSum     prometheus.Gauge
+	timesVerifiedRejectedSum prometheus.Gauge
+	unverifiedBlocksRejected prometheus.Counter
 	// pollsRejected tracks the number of polls that a block was in processing
 	// for before being rejected
 	pollsRejected metric.Averager
@@ -107,6 +111,21 @@ func newMetrics(
 			Name:      "blks_accepted_container_size_sum",
 			Help:      "cumulative size of all accepted blocks",
 		}),
+		buildLatencyAccepted: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "blks_build_accept_latency",
+			Help:      "time (in ns) from the timestamp of a block to the time it was accepted",
+		}),
+		timesVerifiedAcceptedSum: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "blks_times_verified_accepted_sum",
+			Help:      "cumulative number of times all accepted blocks were verified",
+		}),
+		verifiedOnceBlocksAccepted: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "verified_once_blks_rejected",
+			Help:      "number of blocks accepted that were only verified once",
+		}),
 		pollsAccepted: metric.NewAveragerWithErrs(
 			namespace,
 			"blks_polls_accepted",
@@ -127,17 +146,22 @@ func newMetrics(
 			reg,
 			&errs,
 		),
-		buildLatencyAccepted: prometheus.NewGauge(prometheus.GaugeOpts{
-			Namespace: namespace,
-			Name:      "blks_build_accept_latency",
-			Help:      "time (in ns) from the timestamp of a block to the time it was accepted",
-		}),
 
 		blockSizeRejectedSum: prometheus.NewGauge(prometheus.GaugeOpts{
 			Namespace: namespace,
 			Name:      "blks_rejected_container_size_sum",
 			Help:      "cumulative size of all rejected blocks",
 		}),
+		timesVerifiedRejectedSum: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "blks_times_verified_rejected_sum",
+			Help:      "cumulative number of times all rejected blocks were verified",
+		}),
+		unverifiedBlocksRejected: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "unverified_blks_rejected",
+			Help:      "number of blocks rejected that were never verified",
+		}),
 		pollsRejected: metric.NewAveragerWithErrs(
 			namespace,
 			"blks_polls_rejected",
@@ -183,7 +207,11 @@ func newMetrics(
 		reg.Register(m.numProcessing),
 		reg.Register(m.blockSizeAcceptedSum),
 		reg.Register(m.buildLatencyAccepted),
+		reg.Register(m.timesVerifiedAcceptedSum),
+		reg.Register(m.verifiedOnceBlocksAccepted),
 		reg.Register(m.blockSizeRejectedSum),
+		reg.Register(m.timesVerifiedRejectedSum),
+		reg.Register(m.unverifiedBlocksRejected),
 		reg.Register(m.numSuccessfulPolls),
 		reg.Register(m.numFailedPolls),
 	)
@@ -207,6 +235,7 @@ func (m *metrics) Accepted(
 	blkID ids.ID,
 	height uint64,
 	timestamp time.Time,
+	timesVerified int,
 	pollNumber uint64,
 	blockSize int,
 ) {
@@ -224,6 +253,11 @@ func (m *metrics) Accepted(
 	m.numProcessing.Dec()
 
 	m.blockSizeAcceptedSum.Add(float64(blockSize))
+	m.timesVerifiedAcceptedSum.Add(float64(timesVerified))
+
+	if timesVerified == 1 {
+		m.verifiedOnceBlocksAccepted.Inc()
+	}
 
 	m.pollsAccepted.Observe(float64(pollNumber - start.pollNumber))
 
@@ -235,7 +269,12 @@ func (m *metrics) Accepted(
 	m.buildLatencyAccepted.Add(float64(builtDuration))
 }
 
-func (m *metrics) Rejected(blkID ids.ID, pollNumber uint64, blockSize int) {
+func (m *metrics) Rejected(
+	blkID ids.ID,
+	timesVerified int,
+	pollNumber uint64,
+	blockSize int,
+) {
 	start, ok := m.processingBlocks.Get(blkID)
 	if !ok {
 		m.log.Error("unable to measure latency",
@@ -249,6 +288,13 @@ func (m *metrics) Rejected(blkID ids.ID, pollNumber uint64, blockSize int) {
 
 	m.blockSizeRejectedSum.Add(float64(blockSize))
 
+	m.blockSizeRejectedSum.Add(float64(blockSize))
+	m.timesVerifiedRejectedSum.Add(float64(timesVerified))
+
+	if timesVerified == 0 {
+		m.unverifiedBlocksRejected.Inc()
+	}
+
 	m.pollsRejected.Observe(float64(pollNumber - start.pollNumber))
 
 	duration := time.Since(start.time)

snow/consensus/snowman/oracle_block.go

@@ -18,6 +18,5 @@ var ErrNotOracle = errors.New("block isn't an oracle")
 type OracleBlock interface {
 	// Options returns the possible children of this block in the order this
 	// validator prefers the blocks.
-	// Options is guaranteed to only be called on a verified block.
 	Options(context.Context) ([2]Block, error)
 }

snow/consensus/snowman/snowman_block.go

@@ -4,11 +4,21 @@
 package snowman
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/snow/choices"
 	"github.com/ava-labs/avalanchego/snow/consensus/snowball"
+	"github.com/ava-labs/avalanchego/utils/set"
 )
 
+const minReverificationDelay = 250 * time.Millisecond
+
+var errRecentlyFailedVerification = errors.New("recently failed verification")
+
 // Tracks the state of a snowman block
 type snowmanBlock struct {
 	// parameters to initialize the snowball instance with
@@ -17,6 +27,13 @@ type snowmanBlock struct {
 	// block that this node contains. For the genesis, this value will be nil
 	blk Block
 
+	// verified is set to true when it is safe to mark this block as preferred.
+	// If consensus implies that this block should be preferred and it hasn't
+	// been verified, the block should be verified.
+	verified         bool
+	lastTimeVerified time.Time
+	numTimesVerified int
+
 	// shouldFalter is set to true if this node, and all its descendants received
 	// less than Alpha votes
 	shouldFalter bool
@@ -27,9 +44,8 @@ type snowmanBlock struct {
 	sb snowball.Consensus
 
 	// children is the set of blocks that have been issued that name this block
-	// as their parent. If this node has not had a child issued under it, this value
-	// will be nil
-	children map[ids.ID]Block
+	// as their parent.
+	children set.Set[ids.ID]
 }
 
 func (n *snowmanBlock) AddChild(child Block) {
@@ -39,12 +55,29 @@ func (n *snowmanBlock) AddChild(child Block) {
 	// should be initialized.
 	if n.sb == nil {
 		n.sb = snowball.NewTree(n.params, childID)
-		n.children = make(map[ids.ID]Block)
 	} else {
 		n.sb.Add(childID)
 	}
 
-	n.children[childID] = child
+	n.children.Add(childID)
+}
+
+func (n *snowmanBlock) Verify(ctx context.Context) error {
+	if n.verified {
+		return nil
+	}
+
+	now := time.Now()
+	if timeSinceLastVerification := now.Sub(n.lastTimeVerified); timeSinceLastVerification < minReverificationDelay {
+		return fmt.Errorf("%w: %s", errRecentlyFailedVerification, timeSinceLastVerification)
+	}
+
+	n.lastTimeVerified = now
+	n.numTimesVerified++
+
+	err := n.blk.Verify(ctx)
+	n.verified = err == nil
+	return err
 }
 
 func (n *snowmanBlock) Accepted() bool {

snow/consensus/snowman/test_block.go

@@ -21,11 +21,12 @@ var (
 type TestBlock struct {
 	choices.TestDecidable
 
-	ParentV    ids.ID
-	HeightV    uint64
-	TimestampV time.Time
-	VerifyV    error
-	BytesV     []byte
+	ParentV         ids.ID
+	HeightV         uint64
+	TimestampV      time.Time
+	VerifyProposerV error
+	VerifyV         error
+	BytesV          []byte
 }
 
 func (b *TestBlock) Parent() ids.ID {
@@ -40,6 +41,10 @@ func (b *TestBlock) Timestamp() time.Time {
 	return b.TimestampV
 }
 
+func (b *TestBlock) VerifyProposer(context.Context) error {
+	return b.VerifyProposerV
+}
+
 func (b *TestBlock) Verify(context.Context) error {
 	return b.VerifyV
 }