From b827781c1334217cd54dcee1b811d6eb4653e5b1 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 20 Jul 2022 16:22:39 -0400 Subject: [PATCH 01/50] plumbing for spec block assembly --- data/pools/transactionPool.go | 78 ++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index cb547a0cdd..1bbc1676f9 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -49,8 +49,9 @@ import ( // TransactionPool.AssembleBlock constructs a valid block for // proposal given a deadline. type TransactionPool struct { - // feePerByte is stored at the beginning of this struct to ensure it has a 64 bit aligned address. This is needed as it's being used - // with atomic operations which require 64 bit alignment on arm. + // feePerByte is stored at the beginning of this struct to ensure it has a + // 64 bit aligned address. This is needed as it's being used with atomic + // operations which require 64 bit alignment on arm. feePerByte uint64 // const @@ -92,6 +93,14 @@ type TransactionPool struct { // proposalAssemblyTime is the ProposalAssemblyTime configured for this node. proposalAssemblyTime time.Duration + + cfg config.Local + speculatedBlockChan chan speculatedBlock +} + +type speculatedBlock struct { + blk *ledgercore.ValidatedBlock + branch bookkeeping.BlockHash } // BlockEvaluator defines the block evaluator interface exposed by the ledger package. @@ -122,6 +131,7 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo txPoolMaxSize: cfg.TxPoolSize, proposalAssemblyTime: cfg.ProposalAssemblyTime, log: log, + cfg: cfg, } pool.cond.L = &pool.mu pool.assemblyCond.L = &pool.assemblyMu @@ -129,8 +139,49 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -// poolAsmResults is used to syncronize the state of the block assembly process. The structure reading/writing is syncronized -// via the pool.assemblyMu lock. +func (pool *TransactionPool) copyTransactionPoolOverNewLedger(ledger *ledger.Ledger) *TransactionPool { + pool.mu.Lock() + defer pool.mu.Unlock() + + pool.speculatedBlockChan = make(chan speculatedBlock, 1) + + copy := TransactionPool{ + pendingTxids: make(map[transactions.Txid]transactions.SignedTxn), + rememberedTxids: make(map[transactions.Txid]transactions.SignedTxn), + expiredTxCount: make(map[basics.Round]int), + ledger: ledger, + statusCache: makeStatusCache(pool.cfg.TxPoolSize), + logProcessBlockStats: pool.cfg.EnableProcessBlockStats, + logAssembleStats: pool.cfg.EnableAssembleStats, + expFeeFactor: pool.cfg.TxPoolExponentialIncreaseFactor, + txPoolMaxSize: pool.cfg.TxPoolSize, + proposalAssemblyTime: pool.cfg.ProposalAssemblyTime, + log: pool.log, //TODO(yossi) change the logger's copy to add a prefix indicating this is the pool's copy/speculation. So failures will be indicative + cfg: pool.cfg, + speculatedBlockChan: pool.speculatedBlockChan, + } + copy.cond.L = ©.mu + copy.assemblyCond.L = ©.assemblyMu + copy.recomputeBlockEvaluator(nil, 0) + + // deep copy txns + for txid, txn := range pool.pendingTxids { + copy.pendingTxids[txid] = txn + } + + // TODO(yossi) I think it's okay to not do the following deep copying. Is it? + for txid, txn := range pool.rememberedTxids { + copy.rememberedTxids[txid] = txn + } + for round, i := range pool.expiredTxCount { + copy.expiredTxCount[round] = i + } + + return © +} + +// poolAsmResults is used to syncronize the state of the block assembly process. +// The structure reading/writing is syncronized via the pool.assemblyMu lock. type poolAsmResults struct { // the ok variable indicates whether the assembly for the block roundStartedEvaluating was complete ( i.e. ok == true ) or // whether it's still in-progress. @@ -326,7 +377,7 @@ func (pool *TransactionPool) computeFeePerByte() uint64 { return feePerByte } -// checkSufficientFee take a set of signed transactions and verifies that each transaction has +// checkSufficientFee takes a set of signed transactions and verifies that each transaction has // sufficient fee to get into the transaction pool func (pool *TransactionPool) checkSufficientFee(txgroup []transactions.SignedTxn) error { // Special case: the compact cert transaction, if issued from the @@ -481,6 +532,23 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } +func (pool *TransactionPool) OnNewSpeculatedBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { + speculatedLedger := nil + speculatedPool := pool.copyTransactionPoolOverNewLedger(speculatedLedger) + speculatedPool.OnNewBlock(block, delta) + + if speculatedPool.assemblyResults.err != nil { + return + } + + speculatedPool.speculatedBlockChan <- speculatedBlock{blk: speculatedPool.assemblyResults.blk, branch: block.Hash()} + select { + case speculatedPool.speculatedBlockChan <- speculatedBlock{blk: speculatedPool.assemblyResults.blk, branch: block.Hash()}: + default: + speculatedPool.log.Warnf("failed writing speculated block to channel, channel already has a block") + } +} + // OnNewBlock excises transactions from the pool that are included in the specified Block or if they've expired func (pool *TransactionPool) OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { var stats telemetryspec.ProcessBlockMetrics From f44a5e8fce66be0465b1943555baf8df88764bb2 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 21 Jul 2022 12:47:59 -0400 Subject: [PATCH 02/50] with spec ledger code --- ledger/speculative.go | 766 +++++++++++++++++++++++++++++++++++++ ledger/speculative_test.go | 137 +++++++ 2 files changed, 903 insertions(+) create mode 100644 ledger/speculative.go create mode 100644 ledger/speculative_test.go diff --git a/ledger/speculative.go b/ledger/speculative.go new file mode 100644 index 0000000000..84a0579f63 --- /dev/null +++ b/ledger/speculative.go @@ -0,0 +1,766 @@ +// Copyright (C) 2019-2021 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package ledger + +import ( + "context" + "database/sql" + "fmt" + "sync" + "time" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/committee" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/data/transactions/verify" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" + "github.com/algorand/go-algorand/util/execpool" +) + +// validatedBlockAsLFE presents a ledgerForEvaluator interface on top of +// a ValidatedBlock. This makes it possible to construct a BlockEvaluator +// on top, which in turn allows speculatively constructing a subsequent +// block, before the ValidatedBlock is committed to the ledger. +// +// This is what the state looks like: +// +// previousLFE <--------- roundCowBase +// ^ l ^ +// | | lookupParent +// | | +// | roundCowState +// | ^ +// | | state +// | | +// | ValidatedBlock -------> Block +// | ^ blk +// | | vb +// | l | +// \---------- validatedBlockAsLFE +// +// where previousLFE might be the full ledger, or might be another +// validatedBlockAsLFE. +type validatedBlockAsLFE struct { + // l points to the underlying ledger; it might be another instance + // of validatedBlockAsLFE if we are speculating on a chain of many + // blocks. + l ledgerForEvaluator + + // vb points to the ValidatedBlock that logically extends the + // state of the ledger. + vb *ValidatedBlock +} + +// makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a +// ValidatedBlock. +func makeValidatedBlockAsLFE(vb *ValidatedBlock) (*validatedBlockAsLFE, error) { + if vb.state.commitParent != nil { + return nil, fmt.Errorf("makeValidatedBlockAsLFE(): vb not committed") + } + + base, ok := vb.state.lookupParent.(*roundCowBase) + if !ok { + return nil, fmt.Errorf("makeValidatedBlockAsLFE(): missing roundCowBase") + } + + return &validatedBlockAsLFE{ + l: base.l, + vb: vb, + }, nil +} + +// resetParent() changes the parent ledger of this validated block to l. +// This is used when the parent block is committed, so that we can garbage +// collect the parent's validatedBlockAsLFE and instead start using the +// committed ledger. +func (v *validatedBlockAsLFE) resetParent(l ledgerForEvaluator) error { + base, ok := v.vb.state.lookupParent.(*roundCowBase) + if !ok { + return fmt.Errorf("validatedBlockAsLFE: lookupParent no longer roundCowBase") + } + + v.l = l + base.l = l + return nil +} + +// BlockHdr implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) BlockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { + if r == v.vb.blk.Round() { + return v.vb.blk.BlockHeader, nil + } + + return v.l.BlockHdr(r) +} + +// CheckDup implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) CheckDup(currentProto config.ConsensusParams, current basics.Round, firstValid basics.Round, lastValid basics.Round, txid transactions.Txid, txl TxLease) error { + if current == v.vb.blk.Round() { + return v.vb.state.checkDup(firstValid, lastValid, txid, txl.Txlease) + } + + return v.l.CheckDup(currentProto, current, firstValid, lastValid, txid, txl) +} + +// CompactCertVoters implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) CompactCertVoters(r basics.Round) (*VotersForRound, error) { + if r >= v.vb.blk.Round() { + // We do not support computing the compact cert voters for rounds + // that have not been committed to the ledger yet. This should not + // be a problem as long as the agreement pipeline depth does not + // exceed CompactCertVotersLookback. + err := fmt.Errorf("validatedBlockAsLFE.CompactCertVoters(%d): validated block is for round %d, voters not available", r, v.vb.blk.Round()) + logging.Base().Warn(err.Error()) + return nil, err + } + + return v.l.CompactCertVoters(r) +} + +// GenesisHash implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) GenesisHash() crypto.Digest { + return v.l.GenesisHash() +} + +// GetCreatorForRound implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) GetCreatorForRound(r basics.Round, cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) { + if r == v.vb.blk.Round() { + return v.vb.state.getCreator(cidx, ctype) + } + + return v.l.GetCreatorForRound(r, cidx, ctype) +} + +// LookupWithoutRewards implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) LookupWithoutRewards(r basics.Round, a basics.Address) (basics.AccountData, basics.Round, error) { + if r == v.vb.blk.Round() { + data, err := v.vb.state.lookup(a) + return data, r, err + } + + return v.l.LookupWithoutRewards(r, a) +} + +// Totals implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) Totals(r basics.Round) (ledgercore.AccountTotals, error) { + if r == v.vb.blk.Round() { + return v.vb.state.totals() + } + return v.l.Totals(r) +} + +// A speculativeBlock is a block that we have validated, but have not agreed +// upon (no continuous chain of certificates). There may be a certificate +// associated with a speculativeBlock, if we get a certificate for a speculative +// block out-of-order with its parent. In this case, we hang on to the certificate +// until all of the parents are committed themselves. +type speculativeBlock struct { + lfe *validatedBlockAsLFE + cert *agreement.Certificate +} + +// ledgerForSpeculation defines the part of the Ledger interface that is +// needed by the speculative ledger code. +type ledgerForSpeculation interface { + ledgerForEvaluator + + Latest() basics.Round + AddBlock(blk bookkeeping.Block, c agreement.Certificate) error + AddValidatedBlock(vb ValidatedBlock, c agreement.Certificate) error + EnsureValidatedBlock(vb *ValidatedBlock, c agreement.Certificate) + Wait(basics.Round) chan struct{} + + TrackerDB() db.Pair + TrackerLog() logging.Logger + VerifiedTransactionCache() verify.VerifiedTransactionCache + RegisterBlockListeners([]BlockListener) +} + +// The SpeculativeLedger tracks speculative blocks that have been proposed +// over the network but that have not yet been agreed upon (no certificate). +type SpeculativeLedger struct { + mu sync.Mutex + + // blocks contains the set of blocks that we have received in a + // proposal but for whose round we have not yet reached consensus. + blocks map[bookkeeping.BlockHash]speculativeBlock + + // l is the committed ledger. + l ledgerForSpeculation + + // dbs is l.TrackerDB() + dbs db.Pair + + // log for logging + log logging.Logger +} + +// speculativeBlocksSchema describes the on-disk state format for storing +// speculative blocks. We make the round number explicit so that we can +// more easily sort and delete by round number. +var speculativeBlockSchema = []string{ + `CREATE TABLE IF NOT EXISTS speculative ( + rnd integer, + blkhash blob, + blkdata blob, + certdata blob, + PRIMARY KEY (rnd, blkhash))`, +} + +// addSpeculativeBlock records a new speculative block. +func (sl *SpeculativeLedger) addSpeculativeBlock(vblk ValidatedBlock, cert *agreement.Certificate) error { + err := sl.addSpeculativeBlockInMem(vblk, cert) + if err != nil { + return err + } + + var certbuf []byte + if cert != nil { + certbuf = protocol.Encode(cert) + } + + blkhash := vblk.blk.Hash() + err = sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("INSERT INTO speculative (rnd, blkhash, blkdata, certdata) VALUES (?, ?, ?, ?)", + vblk.blk.Round(), blkhash[:], protocol.Encode(&vblk.blk), certbuf) + return err + }) + + return nil +} + +// addSpeculativeBlockInMem updates the in-memory state with a new speculative +// block, but does not store the block on-disk. +func (sl *SpeculativeLedger) addSpeculativeBlockInMem(vblk ValidatedBlock, cert *agreement.Certificate) error { + // The parent of this block must be committed or present in the + // speculation tracker. + latest := sl.l.Latest() + if vblk.blk.Round() > latest+1 { + prevhash := vblk.blk.Branch + _, ok := sl.blocks[prevhash] + if !ok { + return fmt.Errorf("addSpeculativeBlockInMem(%d): latest is %d, missing parent %s", vblk.blk.Round(), latest, prevhash) + } + } + + lfe, err := makeValidatedBlockAsLFE(&vblk) + if err != nil { + return err + } + + h := vblk.blk.Hash() + sl.blocks[h] = speculativeBlock{ + lfe: lfe, + cert: cert, + } + return nil +} + +type blkcert struct { + blk bookkeeping.Block + cert *agreement.Certificate +} + +func (sl *SpeculativeLedger) loadFromDisk() error { + sl.mu.Lock() + defer sl.mu.Unlock() + + var blocks []blkcert + + err := sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + for _, tableCreate := range speculativeBlockSchema { + _, err := tx.Exec(tableCreate) + if err != nil { + return fmt.Errorf("SpeculativeLedger could not create: %v", err) + } + } + + rows, err := tx.Query("SELECT blkdata, certdata FROM speculative ORDER BY rnd ASC") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var blkbuf []byte + var certbuf []byte + err = rows.Scan(&blkbuf, &certbuf) + if err != nil { + return err + } + + var entry blkcert + err = protocol.Decode(blkbuf, &entry.blk) + if err != nil { + return err + } + + if len(certbuf) > 0 { + entry.cert = new(agreement.Certificate) + err = protocol.Decode(certbuf, entry.cert) + if err != nil { + return err + } + } + + blocks = append(blocks, entry) + } + + return nil + }) + if err != nil { + return err + } + + for _, entry := range blocks { + blk := &entry.blk + var parentLedger ledgerForEvaluator + if blk.Round() == sl.l.Latest()+1 { + parentLedger = sl.l + } else { + parentHash := blk.Branch + parent, ok := sl.blocks[parentHash] + if !ok { + sl.log.Warnf("SpeculativeLedger.loadFromDisk: cannot find parent %v for block %v round %d, latest %d", parentHash, blk.Hash(), blk.Round(), sl.l.Latest()) + continue + } + parentLedger = parent.lfe + } + + state, err := eval(context.Background(), parentLedger, *blk, false, sl.l.VerifiedTransactionCache(), nil) + if err != nil { + sl.log.Warnf("SpeculativeLedger.loadFromDisk: block %d round %d: %v", blk.Hash(), blk.Round(), err) + continue + } + + vblk := ValidatedBlock{ + blk: *blk, + state: state, + } + err = sl.addSpeculativeBlockInMem(vblk, entry.cert) + if err != nil { + sl.log.Warnf("SpeculativeLedger.loadFromDisk: block %d round %d: addSpeculativeBlockInMem: %v", blk.Hash(), blk.Round(), err) + } + } + + return nil +} + +func (sl *SpeculativeLedger) OnNewBlock(blk bookkeeping.Block, delta ledgercore.StateDelta) { + sl.mu.Lock() + defer sl.mu.Unlock() + + for h, specblk := range sl.blocks { + if specblk.lfe.vb.blk.Round() < blk.Round() { + // Older blocks hanging around for whatever reason. + // Shouldn't happen, but clean them up just in case. + delete(sl.blocks, h) + } + + if specblk.lfe.vb.blk.Round() == blk.Round() { + // Block for the same round. Clear it out. + delete(sl.blocks, h) + + if h == blk.Hash() { + // Same block; we speculated correctly. + } else { + // Different block for the same round. + // Now we know this is an incorrect speculation; + // clear out its children too. + sl.invalidateChildren(h) + } + } + + if specblk.lfe.vb.blk.Round() == blk.Round()+1 && + specblk.lfe.vb.blk.Branch == blk.Hash() { + + // If this is a child of the now-committed block, + // update its parent ledger pointer to avoid chains + // of validatedBlockAsLFE's. + specblk.lfe.resetParent(sl.l) + + // If this child has a certificate associated with it, + // add the block to the ledger. This will in turn cause + // the ledger to call our newBlock() again, which will + // commit any subsequent blocks that already have certs. + if specblk.cert != nil { + sl.l.EnsureValidatedBlock(specblk.lfe.vb, *specblk.cert) + } + } + } + + err := sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("DELETE FROM speculative WHERE rnd<=?", blk.Round()) + return err + }) + if err != nil { + sl.log.Warnf("SpeculativeLedger.OnNewBlock: cannot delete blocks up to %d: %v", blk.Round(), err) + } +} + +func (sl *SpeculativeLedger) invalidateChildren(branch bookkeeping.BlockHash) { + for h, specblk := range sl.blocks { + if specblk.lfe.vb.blk.Branch == branch { + delete(sl.blocks, h) + sl.invalidateChildren(h) + } + } +} + +// MakeSpeculativeLedger constructs a SpeculativeLedger around a Ledger. +func MakeSpeculativeLedger(l ledgerForSpeculation) (*SpeculativeLedger, error) { + sl := &SpeculativeLedger{ + l: l, + dbs: l.TrackerDB(), + log: l.TrackerLog(), + blocks: make(map[bookkeeping.BlockHash]speculativeBlock), + } + + err := sl.loadFromDisk() + if err != nil { + return nil, err + } + + l.RegisterBlockListeners([]BlockListener{sl}) + return sl, nil +} + +// leafNotFound is an error indicating that the leaf hash was not present +// in the speculative ledger. +type leafNotFound struct { + h bookkeeping.BlockHash +} + +// Error implements the error interface. +func (lnf leafNotFound) Error() string { + return fmt.Sprintf("SpeculativeLedger.blockHdr: leaf %v not found", lnf.h) +} + +// LFE returns a ledgerForEvaluator for round r from the speculative ledger. +func (sl *SpeculativeLedger) LFE(r basics.Round, leaf bookkeeping.BlockHash) (ledgerForEvaluator, error) { + sl.mu.Lock() + defer sl.mu.Unlock() + + entry, ok := sl.blocks[leaf] + if ok { + return entry.lfe, nil + } + + if r <= sl.l.Latest() { + return sl.l, nil + } + + return nil, leafNotFound{h: leaf} +} + +// BlockHdr returns the block header for round r from the speculative ledger. +func (sl *SpeculativeLedger) BlockHdr(r basics.Round, leaf bookkeeping.BlockHash) (bookkeeping.BlockHeader, error) { + lfe, err := sl.LFE(r, leaf) + if err != nil { + return bookkeeping.BlockHeader{}, err + } + + return lfe.BlockHdr(r) +} + +// NextRound returns the next round for which no block has been committed. +func (sl *SpeculativeLedger) NextRound() basics.Round { + return sl.l.Latest() + 1 +} + +// Wait returns a channel that closes once a given round is stored +// durably. If the block was added speculatively, Wait indicates +// when the block is durably stored as a speculative block. If the +// block was added non-speculatively, Wait indicates when the block +// is durably stored as a non-speculative block. +// +// Wait properly supports waiting for a round r that has already been +// added (but perhaps not durably stored yet) for both speculative and +// non-speculative blocks. +// +// Wait supports waiting for a round r that has not been added yet, but +// only for non-speculative blocks. It is not well-defined which speculative +// block for round r, to be added later, we might want to wait for. +func (sl *SpeculativeLedger) Wait(r basics.Round, leaf bookkeeping.BlockHash) chan struct{} { + // Check for a pending non-speculative block. This might exist + // even if we already have a speculative block too. + if r <= sl.l.Latest() { + return sl.l.Wait(r) + } + + // Check if we have a speculative block for this round. Speculative + // blocks are currently written to durable storage synchronously, so + // no waiting is needed. + sl.mu.Lock() + entry, ok := sl.blocks[leaf] + sl.mu.Unlock() + if ok && r <= entry.lfe.vb.blk.Round() { + closed := make(chan struct{}) + close(closed) + return closed + } + + // No speculative block present, and not pending in the blockQ. + // Wait for the block to be inserted by someone else (e.g., catchup). + return sl.l.Wait(r) +} + +// Seed returns the VRF seed that in a given round's block header. +func (sl *SpeculativeLedger) Seed(r basics.Round, leaf bookkeeping.BlockHash) (committee.Seed, error) { + blockhdr, err := sl.BlockHdr(r, leaf) + if err != nil { + return committee.Seed{}, err + } + return blockhdr.Seed, nil +} + +// Lookup returns the AccountData associated with some Address at the +// conclusion of a given round. +func (sl *SpeculativeLedger) Lookup(r basics.Round, leaf bookkeeping.BlockHash, addr basics.Address) (basics.AccountData, error) { + lfe, err := sl.LFE(r, leaf) + if err != nil { + return basics.AccountData{}, err + } + + data, _, err := lfe.LookupWithoutRewards(r, addr) + if err != nil { + return basics.AccountData{}, err + } + + blockhdr, err := lfe.BlockHdr(r) + if err != nil { + return basics.AccountData{}, err + } + + rewardsProto := config.Consensus[blockhdr.CurrentProtocol] + rewardsLevel := blockhdr.RewardsLevel + return data.WithUpdatedRewards(rewardsProto, rewardsLevel), nil +} + +// Circulation returns the total amount of money in online accounts at the +// conclusion of a given round. +func (sl *SpeculativeLedger) Circulation(r basics.Round, leaf bookkeeping.BlockHash) (basics.MicroAlgos, error) { + lfe, err := sl.LFE(r, leaf) + if err != nil { + return basics.MicroAlgos{}, err + } + + totals, err := lfe.Totals(r) + if err != nil { + return basics.MicroAlgos{}, err + } + + return totals.Online.Money, nil +} + +// LookupDigest returns the Digest of the block that was agreed on in a given round. +func (sl *SpeculativeLedger) LookupDigest(r basics.Round, leaf bookkeeping.BlockHash) (crypto.Digest, error) { + blockhdr, err := sl.BlockHdr(r, leaf) + if err != nil { + return crypto.Digest{}, err + } + return crypto.Digest(blockhdr.Hash()), nil +} + +// ConsensusParams returns the consensus parameters for a given round. +func (sl *SpeculativeLedger) ConsensusParams(r basics.Round, leaf bookkeeping.BlockHash) (config.ConsensusParams, error) { + blockhdr, err := sl.BlockHdr(r, leaf) + if err != nil { + return config.ConsensusParams{}, err + } + return config.Consensus[blockhdr.CurrentProtocol], nil +} + +// ConsensusVersion returns the consensus version for a given round. +func (sl *SpeculativeLedger) ConsensusVersion(r basics.Round, leaf bookkeeping.BlockHash) (protocol.ConsensusVersion, error) { + blockhdr, err := sl.BlockHdr(r, leaf) + if err != nil { + return "", err + } + return blockhdr.CurrentProtocol, nil +} + +// AddSpeculativeBlock records a new speculative block. +func (sl *SpeculativeLedger) AddSpeculativeBlock(vblk ValidatedBlock) error { + sl.mu.Lock() + defer sl.mu.Unlock() + return sl.addSpeculativeBlock(vblk, nil) +} + +// AddBlock adds a certificate together with a block to the ledger. +func (sl *SpeculativeLedger) AddBlock(blk bookkeeping.Block, cert agreement.Certificate) error { + vb, err := sl.Validate(context.Background(), blk.Branch, blk, nil) + if err != nil { + return err + } + + return sl.AddValidatedBlock(*vb, cert) +} + +// AddValidatedBlock adds a certificate together with a block to the ledger. +func (sl *SpeculativeLedger) AddValidatedBlock(vb ValidatedBlock, cert agreement.Certificate) error { + sl.mu.Lock() + + // If this block is for the next round expected by the ledger, + // add it directly to the underlying ledger. That avoids writing + // the cert to disk twice. The tracker lock does not guard against + // concurrent insertion of blocks, but the latest round returned + // by the ledger is monotonically increasing, so if it increments + // concurrently with us, the block insertion should error out anyway. + if vb.blk.Round() <= sl.l.Latest()+1 { + sl.mu.Unlock() + return sl.l.AddValidatedBlock(vb, cert) + } + + // This is a speculative block. We are holding the sl.mu lock, which + // will serialize us with respect to calls to OnNewBlock(). We rely + // on that so that OnNewBlock() will insert this certificate into the + // ledger, if/when this block stops being speculative. + defer sl.mu.Unlock() + + // This block might be not even in our set of known speculative blocks + // yet, so add it there if need be. addSpeculativeBlock() takes a cert, + // so nothing left for us to do if we invoke it. + entry, ok := sl.blocks[vb.blk.Hash()] + if !ok { + return sl.addSpeculativeBlock(vb, &cert) + } + + entry.cert = &cert + blkhash := entry.lfe.vb.blk.Hash() + return sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("UPDATE speculative SET certdata=? WHERE rnd=? AND blkhash=?", + protocol.Encode(&cert), entry.lfe.vb.blk.Round(), blkhash[:]) + return err + }) +} + +// Validate validates whether a block is valid, on a particular leaf branch. +func (sl *SpeculativeLedger) Validate(ctx context.Context, leaf bookkeeping.BlockHash, blk bookkeeping.Block, pool execpool.BacklogPool) (*ValidatedBlock, error) { + state, err := sl.eval(ctx, leaf, blk, true, pool) + if err != nil { + return nil, err + } + + return &ValidatedBlock{ + blk: blk, + state: state, + }, nil +} + +// eval evaluates a block on a particular leaf branch. +func (sl *SpeculativeLedger) eval(ctx context.Context, leaf bookkeeping.BlockHash, blk bookkeeping.Block, validate bool, executionPool execpool.BacklogPool) (*roundCowState, error) { + lfe, err := sl.LFE(blk.Round().SubSaturate(1), leaf) + if err != nil { + return nil, err + } + + return eval(ctx, lfe, blk, validate, sl.l.VerifiedTransactionCache(), executionPool) +} + +// StartEvaluator starts a block evaluator with a particular block header. +// The block header's Branch value determines which speculative branch is used. +// This is intended to be used by the transaction pool assembly code. +func (sl *SpeculativeLedger) StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint int) (*BlockEvaluator, error) { + lfe, err := sl.LFE(hdr.Round.SubSaturate(1), hdr.Branch) + if err != nil { + return nil, err + } + + return startEvaluator(lfe, hdr, paysetHint, true, true) +} + +// EnsureBlock ensures that the block, and associated certificate c, are +// written to the speculative ledger, or that some other block for the +// same round is written to the ledger. +// This function can be called concurrently. +func (sl *SpeculativeLedger) EnsureBlock(block *bookkeeping.Block, c agreement.Certificate) { + round := block.Round() + protocolErrorLogged := false + + // As a fallback, bail out if the base (non-speculative) ledger has a + // block for the same round number. + for sl.l.Latest() < round { + err := sl.l.AddBlock(*block, c) + if err == nil { + break + } + + switch err.(type) { + case protocol.Error: + if !protocolErrorLogged { + logging.Base().Errorf("unrecoverable protocol error detected at block %d: %v", round, err) + protocolErrorLogged = true + } + case ledgercore.BlockInLedgerError: + logging.Base().Debugf("could not write block %d to the ledger: %v", round, err) + return // this error implies that l.Latest() >= round + default: + logging.Base().Errorf("could not write block %d to the speculative ledger: %v", round, err) + } + + // If there was an error add a short delay before the next attempt. + time.Sleep(100 * time.Millisecond) + } +} + +// EnsureValidatedBlock ensures that the block, and associated certificate c, are +// written to the speculative ledger, or that some other block for the same round is +// written to the ledger. +func (sl *SpeculativeLedger) EnsureValidatedBlock(vb *ValidatedBlock, c agreement.Certificate) { + round := vb.Block().Round() + + // As a fallback, bail out if the base (non-speculative) ledger has a + // block for the same round number. + for sl.l.Latest() < round { + err := sl.AddValidatedBlock(*vb, c) + if err == nil { + break + } + + logfn := logging.Base().Errorf + + switch err.(type) { + case ledgercore.BlockInLedgerError: + logfn = logging.Base().Debugf + } + + logfn("could not write block %d to the speculative ledger: %v", round, err) + } +} + +// Latest proxies Ledger.Latest() +func (sl *SpeculativeLedger) Latest() basics.Round { + return sl.l.Latest() +} + +// VerifiedTransactionCache proxies Ledger.VerifiedTransactionCache() +func (sl *SpeculativeLedger) VerifiedTransactionCache() verify.VerifiedTransactionCache { + return sl.l.VerifiedTransactionCache() +} + +// GenesisHash proxies Ledger.GenesisHash() +func (sl *SpeculativeLedger) GenesisHash() crypto.Digest { + return sl.l.GenesisHash() +} diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go new file mode 100644 index 0000000000..cd131e7712 --- /dev/null +++ b/ledger/speculative_test.go @@ -0,0 +1,137 @@ +// Copyright (C) 2019-2021 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package ledger + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" +) + +func TestSpeculative(t *testing.T) { + genesisInitState, _ := testGenerateInitState(t, protocol.ConsensusCurrentVersion, 1000) + const inMem = true + cfg := config.GetDefaultLocal() + log := logging.TestingLog(t) + l, err := OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) + require.NoError(t, err, "could not open ledger") + defer l.Close() + + blk0, err := l.BlockHdr(l.Latest()) + require.NoError(t, err) + + sl, err := MakeSpeculativeLedger(l) + require.NoError(t, err) + + var blk1 bookkeeping.Block + blk1.CurrentProtocol = protocol.ConsensusCurrentVersion + blk1.Branch = blk0.Hash() + blk1.RewardsPool = testPoolAddr + blk1.FeeSink = testSinkAddr + blk1.BlockHeader.GenesisHash = genesisInitState.GenesisHash + blk1.BlockHeader.Round = l.Latest() + 1 + + state, err := sl.eval(context.Background(), bookkeeping.BlockHash{}, blk1, false, nil) + require.NoError(t, err) + + vblk1 := ValidatedBlock{ + blk: blk1, + state: state, + } + + err = sl.AddSpeculativeBlock(vblk1) + require.NoError(t, err) + + _, err = sl.ConsensusVersion(blk1.BlockHeader.Round, bookkeeping.BlockHash{}) + require.Error(t, err) + + var randomLeaf bookkeeping.BlockHash + crypto.RandBytes(randomLeaf[:]) + _, err = sl.ConsensusVersion(blk1.BlockHeader.Round, randomLeaf) + require.Error(t, err) + + cv1, err := sl.ConsensusVersion(blk1.BlockHeader.Round, blk1.Hash()) + require.NoError(t, err) + require.Equal(t, cv1, blk1.CurrentProtocol) + + blk2 := blk1 + blk2.BlockHeader.Round++ + blk2.Branch = blk1.Hash() + + // Pick some accounts at random + var addr1, addr2 basics.Address + for a := range genesisInitState.Accounts { + if addr1 == (basics.Address{}) { + addr1 = a + } else if addr2 == (basics.Address{}) { + addr2 = a + } else { + break + } + } + + var tx21 transactions.Transaction + tx21.Type = protocol.PaymentTx + tx21.Sender = addr1 + tx21.Receiver = addr2 + tx21.FirstValid = blk2.BlockHeader.Round + tx21.LastValid = blk2.BlockHeader.Round + tx21.Amount.Raw = 1000000 + blk2.Payset = append(blk2.Payset, transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: tx21, + }, + }, + HasGenesisID: true, + }) + + state, err = sl.eval(context.Background(), bookkeeping.BlockHash{}, blk2, false, nil) + require.Error(t, err) + + state, err = sl.eval(context.Background(), blk1.Hash(), blk2, false, nil) + require.NoError(t, err) + + vblk2 := ValidatedBlock{ + blk: blk2, + state: state, + } + + err = sl.AddSpeculativeBlock(vblk2) + require.NoError(t, err) + + ad11, err := sl.Lookup(blk1.Round(), blk1.Hash(), addr1) + require.NoError(t, err) + + ad22, err := sl.Lookup(blk2.Round(), blk2.Hash(), addr1) + require.NoError(t, err) + + ad12, err := sl.Lookup(blk1.Round(), blk2.Hash(), addr1) + require.NoError(t, err) + + require.Equal(t, ad12.MicroAlgos.Raw, ad11.MicroAlgos.Raw) + require.Equal(t, ad22.MicroAlgos.Raw, ad11.MicroAlgos.Raw-1000000) +} From 1e1c3ad5cc3bd5b4df7fe5e75fb58e7070ddeff9 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 22 Jul 2022 10:53:29 -0400 Subject: [PATCH 03/50] block as LFE code --- data/pools/transactionPool.go | 17 +- ledger/ledgercore/validatedBlock.go | 18 + ledger/speculative.go | 706 ++-------------------------- 3 files changed, 57 insertions(+), 684 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 1bbc1676f9..2d711e1e68 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -532,20 +532,21 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } -func (pool *TransactionPool) OnNewSpeculatedBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { - speculatedLedger := nil - speculatedPool := pool.copyTransactionPoolOverNewLedger(speculatedLedger) - speculatedPool.OnNewBlock(block, delta) +func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { + speculativeLedger := nil + speculativePool := pool.copyTransactionPoolOverNewLedger(speculatedLedger) + // check block's prev == ledger latest + speculativePool.OnNewBlock(block, delta) - if speculatedPool.assemblyResults.err != nil { + if speculativePool.assemblyResults.err != nil { return } - speculatedPool.speculatedBlockChan <- speculatedBlock{blk: speculatedPool.assemblyResults.blk, branch: block.Hash()} + speculativePool.speculatedBlockChan <- speculatedBlock{blk: speculativePool.assemblyResults.blk, branch: block.Hash()} select { - case speculatedPool.speculatedBlockChan <- speculatedBlock{blk: speculatedPool.assemblyResults.blk, branch: block.Hash()}: + case speculativePool.speculatedBlockChan <- speculatedBlock{blk: speculativePool.assemblyResults.blk, branch: block.Hash()}: default: - speculatedPool.log.Warnf("failed writing speculated block to channel, channel already has a block") + speculativePool.log.Warnf("failed writing speculated block to channel, channel already has a block") } } diff --git a/ledger/ledgercore/validatedBlock.go b/ledger/ledgercore/validatedBlock.go index 9b4aa30f10..299830dd97 100644 --- a/ledger/ledgercore/validatedBlock.go +++ b/ledger/ledgercore/validatedBlock.go @@ -17,8 +17,11 @@ package ledgercore import ( + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" + "github.com/algorand/go-algorand/data/transactions" ) // ValidatedBlock represents the result of a block validation. It can @@ -50,6 +53,21 @@ func (vb ValidatedBlock) WithSeed(s committee.Seed) ValidatedBlock { } } +func (vb ValidatedBlock) CheckDup(currentProto config.ConsensusParams, firstValid, lastValid basics.Round, txid transactions.Txid, txl Txlease) error { + _, present := vb.delta.Txids[txid] + if present { + return &TransactionInLedgerError{Txid: txid} + } + + if currentProto.SupportTransactionLeases && (txl.Lease != [32]byte{}) { + expires, ok := vb.delta.Txleases[txl] + if ok && vb.blk.Round() <= expires { + return MakeLeaseInLedgerError(txid, txl) + } + } + return nil +} + // MakeValidatedBlock creates a validated block. func MakeValidatedBlock(blk bookkeeping.Block, delta StateDelta) ValidatedBlock { return ValidatedBlock{ diff --git a/ledger/speculative.go b/ledger/speculative.go index 84a0579f63..267cb23712 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -17,28 +17,28 @@ package ledger import ( - "context" - "database/sql" - "fmt" - "sync" - "time" - - "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/data/committee" "github.com/algorand/go-algorand/data/transactions" - "github.com/algorand/go-algorand/data/transactions/verify" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/logging" - "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/db" - "github.com/algorand/go-algorand/util/execpool" ) -// validatedBlockAsLFE presents a ledgerForEvaluator interface on top of +type LedgerForEvaluator interface { + // Needed for cow.go + BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) + CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error + LookupWithoutRewards(basics.Round, basics.Address) (basics.AccountData, basics.Round, error) + GetCreatorForRound(basics.Round, basics.CreatableIndex, basics.CreatableType) (basics.Address, bool, error) + + // Needed for the evaluator + GenesisHash() crypto.Digest + Totals(basics.Round) (ledgercore.AccountTotals, error) + CompactCertVoters(basics.Round) (*ledgercore.VotersForRound, error) +} + +// validatedBlockAsLFE presents a LedgerForEvaluator interface on top of // a ValidatedBlock. This makes it possible to construct a BlockEvaluator // on top, which in turn allows speculatively constructing a subsequent // block, before the ValidatedBlock is committed to the ledger. @@ -65,79 +65,40 @@ type validatedBlockAsLFE struct { // l points to the underlying ledger; it might be another instance // of validatedBlockAsLFE if we are speculating on a chain of many // blocks. - l ledgerForEvaluator + l LedgerForEvaluator // vb points to the ValidatedBlock that logically extends the // state of the ledger. - vb *ValidatedBlock + vb *ledgercore.ValidatedBlock } // makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a // ValidatedBlock. -func makeValidatedBlockAsLFE(vb *ValidatedBlock) (*validatedBlockAsLFE, error) { - if vb.state.commitParent != nil { - return nil, fmt.Errorf("makeValidatedBlockAsLFE(): vb not committed") - } - - base, ok := vb.state.lookupParent.(*roundCowBase) - if !ok { - return nil, fmt.Errorf("makeValidatedBlockAsLFE(): missing roundCowBase") - } - +func makeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { return &validatedBlockAsLFE{ - l: base.l, + l: l, vb: vb, }, nil } -// resetParent() changes the parent ledger of this validated block to l. -// This is used when the parent block is committed, so that we can garbage -// collect the parent's validatedBlockAsLFE and instead start using the -// committed ledger. -func (v *validatedBlockAsLFE) resetParent(l ledgerForEvaluator) error { - base, ok := v.vb.state.lookupParent.(*roundCowBase) - if !ok { - return fmt.Errorf("validatedBlockAsLFE: lookupParent no longer roundCowBase") - } - - v.l = l - base.l = l - return nil -} - // BlockHdr implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) BlockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { - if r == v.vb.blk.Round() { - return v.vb.blk.BlockHeader, nil + if r == v.vb.Block().Round() { + return v.vb.Block().BlockHeader, nil } return v.l.BlockHdr(r) } // CheckDup implements the ledgerForEvaluator interface. -func (v *validatedBlockAsLFE) CheckDup(currentProto config.ConsensusParams, current basics.Round, firstValid basics.Round, lastValid basics.Round, txid transactions.Txid, txl TxLease) error { - if current == v.vb.blk.Round() { - return v.vb.state.checkDup(firstValid, lastValid, txid, txl.Txlease) +func (v *validatedBlockAsLFE) CheckDup(currentProto config.ConsensusParams, current basics.Round, firstValid basics.Round, lastValid basics.Round, txid transactions.Txid, txl ledgercore.Txlease) error { + if current == v.vb.Block().Round() { + return v.vb.CheckDup(currentProto, firstValid, lastValid, txid, txl) } return v.l.CheckDup(currentProto, current, firstValid, lastValid, txid, txl) } -// CompactCertVoters implements the ledgerForEvaluator interface. -func (v *validatedBlockAsLFE) CompactCertVoters(r basics.Round) (*VotersForRound, error) { - if r >= v.vb.blk.Round() { - // We do not support computing the compact cert voters for rounds - // that have not been committed to the ledger yet. This should not - // be a problem as long as the agreement pipeline depth does not - // exceed CompactCertVotersLookback. - err := fmt.Errorf("validatedBlockAsLFE.CompactCertVoters(%d): validated block is for round %d, voters not available", r, v.vb.blk.Round()) - logging.Base().Warn(err.Error()) - return nil, err - } - - return v.l.CompactCertVoters(r) -} - // GenesisHash implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) GenesisHash() crypto.Digest { return v.l.GenesisHash() @@ -145,622 +106,15 @@ func (v *validatedBlockAsLFE) GenesisHash() crypto.Digest { // GetCreatorForRound implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) GetCreatorForRound(r basics.Round, cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) { - if r == v.vb.blk.Round() { - return v.vb.state.getCreator(cidx, ctype) - } - - return v.l.GetCreatorForRound(r, cidx, ctype) -} - -// LookupWithoutRewards implements the ledgerForEvaluator interface. -func (v *validatedBlockAsLFE) LookupWithoutRewards(r basics.Round, a basics.Address) (basics.AccountData, basics.Round, error) { - if r == v.vb.blk.Round() { - data, err := v.vb.state.lookup(a) - return data, r, err - } - - return v.l.LookupWithoutRewards(r, a) -} - -// Totals implements the ledgerForEvaluator interface. -func (v *validatedBlockAsLFE) Totals(r basics.Round) (ledgercore.AccountTotals, error) { - if r == v.vb.blk.Round() { - return v.vb.state.totals() - } - return v.l.Totals(r) -} - -// A speculativeBlock is a block that we have validated, but have not agreed -// upon (no continuous chain of certificates). There may be a certificate -// associated with a speculativeBlock, if we get a certificate for a speculative -// block out-of-order with its parent. In this case, we hang on to the certificate -// until all of the parents are committed themselves. -type speculativeBlock struct { - lfe *validatedBlockAsLFE - cert *agreement.Certificate -} - -// ledgerForSpeculation defines the part of the Ledger interface that is -// needed by the speculative ledger code. -type ledgerForSpeculation interface { - ledgerForEvaluator - - Latest() basics.Round - AddBlock(blk bookkeeping.Block, c agreement.Certificate) error - AddValidatedBlock(vb ValidatedBlock, c agreement.Certificate) error - EnsureValidatedBlock(vb *ValidatedBlock, c agreement.Certificate) - Wait(basics.Round) chan struct{} - - TrackerDB() db.Pair - TrackerLog() logging.Logger - VerifiedTransactionCache() verify.VerifiedTransactionCache - RegisterBlockListeners([]BlockListener) -} - -// The SpeculativeLedger tracks speculative blocks that have been proposed -// over the network but that have not yet been agreed upon (no certificate). -type SpeculativeLedger struct { - mu sync.Mutex - - // blocks contains the set of blocks that we have received in a - // proposal but for whose round we have not yet reached consensus. - blocks map[bookkeeping.BlockHash]speculativeBlock - - // l is the committed ledger. - l ledgerForSpeculation - - // dbs is l.TrackerDB() - dbs db.Pair - - // log for logging - log logging.Logger -} - -// speculativeBlocksSchema describes the on-disk state format for storing -// speculative blocks. We make the round number explicit so that we can -// more easily sort and delete by round number. -var speculativeBlockSchema = []string{ - `CREATE TABLE IF NOT EXISTS speculative ( - rnd integer, - blkhash blob, - blkdata blob, - certdata blob, - PRIMARY KEY (rnd, blkhash))`, -} - -// addSpeculativeBlock records a new speculative block. -func (sl *SpeculativeLedger) addSpeculativeBlock(vblk ValidatedBlock, cert *agreement.Certificate) error { - err := sl.addSpeculativeBlockInMem(vblk, cert) - if err != nil { - return err - } - - var certbuf []byte - if cert != nil { - certbuf = protocol.Encode(cert) - } - - blkhash := vblk.blk.Hash() - err = sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { - _, err := tx.Exec("INSERT INTO speculative (rnd, blkhash, blkdata, certdata) VALUES (?, ?, ?, ?)", - vblk.blk.Round(), blkhash[:], protocol.Encode(&vblk.blk), certbuf) - return err - }) - - return nil -} - -// addSpeculativeBlockInMem updates the in-memory state with a new speculative -// block, but does not store the block on-disk. -func (sl *SpeculativeLedger) addSpeculativeBlockInMem(vblk ValidatedBlock, cert *agreement.Certificate) error { - // The parent of this block must be committed or present in the - // speculation tracker. - latest := sl.l.Latest() - if vblk.blk.Round() > latest+1 { - prevhash := vblk.blk.Branch - _, ok := sl.blocks[prevhash] - if !ok { - return fmt.Errorf("addSpeculativeBlockInMem(%d): latest is %d, missing parent %s", vblk.blk.Round(), latest, prevhash) - } - } - - lfe, err := makeValidatedBlockAsLFE(&vblk) - if err != nil { - return err - } - - h := vblk.blk.Hash() - sl.blocks[h] = speculativeBlock{ - lfe: lfe, - cert: cert, - } - return nil -} - -type blkcert struct { - blk bookkeeping.Block - cert *agreement.Certificate -} - -func (sl *SpeculativeLedger) loadFromDisk() error { - sl.mu.Lock() - defer sl.mu.Unlock() - - var blocks []blkcert - - err := sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { - for _, tableCreate := range speculativeBlockSchema { - _, err := tx.Exec(tableCreate) - if err != nil { - return fmt.Errorf("SpeculativeLedger could not create: %v", err) - } - } - - rows, err := tx.Query("SELECT blkdata, certdata FROM speculative ORDER BY rnd ASC") - if err != nil { - return err - } - defer rows.Close() - - for rows.Next() { - var blkbuf []byte - var certbuf []byte - err = rows.Scan(&blkbuf, &certbuf) - if err != nil { - return err - } - - var entry blkcert - err = protocol.Decode(blkbuf, &entry.blk) - if err != nil { - return err - } - - if len(certbuf) > 0 { - entry.cert = new(agreement.Certificate) - err = protocol.Decode(certbuf, entry.cert) - if err != nil { - return err - } - } - - blocks = append(blocks, entry) - } - - return nil - }) - if err != nil { - return err - } - - for _, entry := range blocks { - blk := &entry.blk - var parentLedger ledgerForEvaluator - if blk.Round() == sl.l.Latest()+1 { - parentLedger = sl.l - } else { - parentHash := blk.Branch - parent, ok := sl.blocks[parentHash] - if !ok { - sl.log.Warnf("SpeculativeLedger.loadFromDisk: cannot find parent %v for block %v round %d, latest %d", parentHash, blk.Hash(), blk.Round(), sl.l.Latest()) - continue - } - parentLedger = parent.lfe - } - - state, err := eval(context.Background(), parentLedger, *blk, false, sl.l.VerifiedTransactionCache(), nil) - if err != nil { - sl.log.Warnf("SpeculativeLedger.loadFromDisk: block %d round %d: %v", blk.Hash(), blk.Round(), err) - continue - } - - vblk := ValidatedBlock{ - blk: *blk, - state: state, - } - err = sl.addSpeculativeBlockInMem(vblk, entry.cert) - if err != nil { - sl.log.Warnf("SpeculativeLedger.loadFromDisk: block %d round %d: addSpeculativeBlockInMem: %v", blk.Hash(), blk.Round(), err) - } - } - - return nil -} - -func (sl *SpeculativeLedger) OnNewBlock(blk bookkeeping.Block, delta ledgercore.StateDelta) { - sl.mu.Lock() - defer sl.mu.Unlock() - - for h, specblk := range sl.blocks { - if specblk.lfe.vb.blk.Round() < blk.Round() { - // Older blocks hanging around for whatever reason. - // Shouldn't happen, but clean them up just in case. - delete(sl.blocks, h) - } - - if specblk.lfe.vb.blk.Round() == blk.Round() { - // Block for the same round. Clear it out. - delete(sl.blocks, h) - - if h == blk.Hash() { - // Same block; we speculated correctly. - } else { - // Different block for the same round. - // Now we know this is an incorrect speculation; - // clear out its children too. - sl.invalidateChildren(h) - } - } - - if specblk.lfe.vb.blk.Round() == blk.Round()+1 && - specblk.lfe.vb.blk.Branch == blk.Hash() { - - // If this is a child of the now-committed block, - // update its parent ledger pointer to avoid chains - // of validatedBlockAsLFE's. - specblk.lfe.resetParent(sl.l) - - // If this child has a certificate associated with it, - // add the block to the ledger. This will in turn cause - // the ledger to call our newBlock() again, which will - // commit any subsequent blocks that already have certs. - if specblk.cert != nil { - sl.l.EnsureValidatedBlock(specblk.lfe.vb, *specblk.cert) - } - } - } - - err := sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { - _, err := tx.Exec("DELETE FROM speculative WHERE rnd<=?", blk.Round()) - return err - }) - if err != nil { - sl.log.Warnf("SpeculativeLedger.OnNewBlock: cannot delete blocks up to %d: %v", blk.Round(), err) - } -} - -func (sl *SpeculativeLedger) invalidateChildren(branch bookkeeping.BlockHash) { - for h, specblk := range sl.blocks { - if specblk.lfe.vb.blk.Branch == branch { - delete(sl.blocks, h) - sl.invalidateChildren(h) - } - } -} - -// MakeSpeculativeLedger constructs a SpeculativeLedger around a Ledger. -func MakeSpeculativeLedger(l ledgerForSpeculation) (*SpeculativeLedger, error) { - sl := &SpeculativeLedger{ - l: l, - dbs: l.TrackerDB(), - log: l.TrackerLog(), - blocks: make(map[bookkeeping.BlockHash]speculativeBlock), - } - - err := sl.loadFromDisk() - if err != nil { - return nil, err - } - - l.RegisterBlockListeners([]BlockListener{sl}) - return sl, nil -} - -// leafNotFound is an error indicating that the leaf hash was not present -// in the speculative ledger. -type leafNotFound struct { - h bookkeeping.BlockHash -} - -// Error implements the error interface. -func (lnf leafNotFound) Error() string { - return fmt.Sprintf("SpeculativeLedger.blockHdr: leaf %v not found", lnf.h) -} - -// LFE returns a ledgerForEvaluator for round r from the speculative ledger. -func (sl *SpeculativeLedger) LFE(r basics.Round, leaf bookkeeping.BlockHash) (ledgerForEvaluator, error) { - sl.mu.Lock() - defer sl.mu.Unlock() - - entry, ok := sl.blocks[leaf] - if ok { - return entry.lfe, nil - } - - if r <= sl.l.Latest() { - return sl.l, nil - } - - return nil, leafNotFound{h: leaf} -} - -// BlockHdr returns the block header for round r from the speculative ledger. -func (sl *SpeculativeLedger) BlockHdr(r basics.Round, leaf bookkeeping.BlockHash) (bookkeeping.BlockHeader, error) { - lfe, err := sl.LFE(r, leaf) - if err != nil { - return bookkeeping.BlockHeader{}, err - } - - return lfe.BlockHdr(r) -} - -// NextRound returns the next round for which no block has been committed. -func (sl *SpeculativeLedger) NextRound() basics.Round { - return sl.l.Latest() + 1 -} - -// Wait returns a channel that closes once a given round is stored -// durably. If the block was added speculatively, Wait indicates -// when the block is durably stored as a speculative block. If the -// block was added non-speculatively, Wait indicates when the block -// is durably stored as a non-speculative block. -// -// Wait properly supports waiting for a round r that has already been -// added (but perhaps not durably stored yet) for both speculative and -// non-speculative blocks. -// -// Wait supports waiting for a round r that has not been added yet, but -// only for non-speculative blocks. It is not well-defined which speculative -// block for round r, to be added later, we might want to wait for. -func (sl *SpeculativeLedger) Wait(r basics.Round, leaf bookkeeping.BlockHash) chan struct{} { - // Check for a pending non-speculative block. This might exist - // even if we already have a speculative block too. - if r <= sl.l.Latest() { - return sl.l.Wait(r) - } - - // Check if we have a speculative block for this round. Speculative - // blocks are currently written to durable storage synchronously, so - // no waiting is needed. - sl.mu.Lock() - entry, ok := sl.blocks[leaf] - sl.mu.Unlock() - if ok && r <= entry.lfe.vb.blk.Round() { - closed := make(chan struct{}) - close(closed) - return closed - } - - // No speculative block present, and not pending in the blockQ. - // Wait for the block to be inserted by someone else (e.g., catchup). - return sl.l.Wait(r) -} - -// Seed returns the VRF seed that in a given round's block header. -func (sl *SpeculativeLedger) Seed(r basics.Round, leaf bookkeeping.BlockHash) (committee.Seed, error) { - blockhdr, err := sl.BlockHdr(r, leaf) - if err != nil { - return committee.Seed{}, err - } - return blockhdr.Seed, nil -} - -// Lookup returns the AccountData associated with some Address at the -// conclusion of a given round. -func (sl *SpeculativeLedger) Lookup(r basics.Round, leaf bookkeeping.BlockHash, addr basics.Address) (basics.AccountData, error) { - lfe, err := sl.LFE(r, leaf) - if err != nil { - return basics.AccountData{}, err - } - - data, _, err := lfe.LookupWithoutRewards(r, addr) - if err != nil { - return basics.AccountData{}, err - } - - blockhdr, err := lfe.BlockHdr(r) - if err != nil { - return basics.AccountData{}, err - } - - rewardsProto := config.Consensus[blockhdr.CurrentProtocol] - rewardsLevel := blockhdr.RewardsLevel - return data.WithUpdatedRewards(rewardsProto, rewardsLevel), nil -} - -// Circulation returns the total amount of money in online accounts at the -// conclusion of a given round. -func (sl *SpeculativeLedger) Circulation(r basics.Round, leaf bookkeeping.BlockHash) (basics.MicroAlgos, error) { - lfe, err := sl.LFE(r, leaf) - if err != nil { - return basics.MicroAlgos{}, err - } - - totals, err := lfe.Totals(r) - if err != nil { - return basics.MicroAlgos{}, err - } - - return totals.Online.Money, nil -} - -// LookupDigest returns the Digest of the block that was agreed on in a given round. -func (sl *SpeculativeLedger) LookupDigest(r basics.Round, leaf bookkeeping.BlockHash) (crypto.Digest, error) { - blockhdr, err := sl.BlockHdr(r, leaf) - if err != nil { - return crypto.Digest{}, err - } - return crypto.Digest(blockhdr.Hash()), nil -} - -// ConsensusParams returns the consensus parameters for a given round. -func (sl *SpeculativeLedger) ConsensusParams(r basics.Round, leaf bookkeeping.BlockHash) (config.ConsensusParams, error) { - blockhdr, err := sl.BlockHdr(r, leaf) - if err != nil { - return config.ConsensusParams{}, err - } - return config.Consensus[blockhdr.CurrentProtocol], nil -} - -// ConsensusVersion returns the consensus version for a given round. -func (sl *SpeculativeLedger) ConsensusVersion(r basics.Round, leaf bookkeeping.BlockHash) (protocol.ConsensusVersion, error) { - blockhdr, err := sl.BlockHdr(r, leaf) - if err != nil { - return "", err - } - return blockhdr.CurrentProtocol, nil -} - -// AddSpeculativeBlock records a new speculative block. -func (sl *SpeculativeLedger) AddSpeculativeBlock(vblk ValidatedBlock) error { - sl.mu.Lock() - defer sl.mu.Unlock() - return sl.addSpeculativeBlock(vblk, nil) -} - -// AddBlock adds a certificate together with a block to the ledger. -func (sl *SpeculativeLedger) AddBlock(blk bookkeeping.Block, cert agreement.Certificate) error { - vb, err := sl.Validate(context.Background(), blk.Branch, blk, nil) - if err != nil { - return err - } - - return sl.AddValidatedBlock(*vb, cert) -} - -// AddValidatedBlock adds a certificate together with a block to the ledger. -func (sl *SpeculativeLedger) AddValidatedBlock(vb ValidatedBlock, cert agreement.Certificate) error { - sl.mu.Lock() - - // If this block is for the next round expected by the ledger, - // add it directly to the underlying ledger. That avoids writing - // the cert to disk twice. The tracker lock does not guard against - // concurrent insertion of blocks, but the latest round returned - // by the ledger is monotonically increasing, so if it increments - // concurrently with us, the block insertion should error out anyway. - if vb.blk.Round() <= sl.l.Latest()+1 { - sl.mu.Unlock() - return sl.l.AddValidatedBlock(vb, cert) - } - - // This is a speculative block. We are holding the sl.mu lock, which - // will serialize us with respect to calls to OnNewBlock(). We rely - // on that so that OnNewBlock() will insert this certificate into the - // ledger, if/when this block stops being speculative. - defer sl.mu.Unlock() - - // This block might be not even in our set of known speculative blocks - // yet, so add it there if need be. addSpeculativeBlock() takes a cert, - // so nothing left for us to do if we invoke it. - entry, ok := sl.blocks[vb.blk.Hash()] - if !ok { - return sl.addSpeculativeBlock(vb, &cert) - } - - entry.cert = &cert - blkhash := entry.lfe.vb.blk.Hash() - return sl.dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { - _, err := tx.Exec("UPDATE speculative SET certdata=? WHERE rnd=? AND blkhash=?", - protocol.Encode(&cert), entry.lfe.vb.blk.Round(), blkhash[:]) - return err - }) -} - -// Validate validates whether a block is valid, on a particular leaf branch. -func (sl *SpeculativeLedger) Validate(ctx context.Context, leaf bookkeeping.BlockHash, blk bookkeeping.Block, pool execpool.BacklogPool) (*ValidatedBlock, error) { - state, err := sl.eval(ctx, leaf, blk, true, pool) - if err != nil { - return nil, err - } - - return &ValidatedBlock{ - blk: blk, - state: state, - }, nil -} - -// eval evaluates a block on a particular leaf branch. -func (sl *SpeculativeLedger) eval(ctx context.Context, leaf bookkeeping.BlockHash, blk bookkeeping.Block, validate bool, executionPool execpool.BacklogPool) (*roundCowState, error) { - lfe, err := sl.LFE(blk.Round().SubSaturate(1), leaf) - if err != nil { - return nil, err - } - - return eval(ctx, lfe, blk, validate, sl.l.VerifiedTransactionCache(), executionPool) -} - -// StartEvaluator starts a block evaluator with a particular block header. -// The block header's Branch value determines which speculative branch is used. -// This is intended to be used by the transaction pool assembly code. -func (sl *SpeculativeLedger) StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint int) (*BlockEvaluator, error) { - lfe, err := sl.LFE(hdr.Round.SubSaturate(1), hdr.Branch) - if err != nil { - return nil, err - } - - return startEvaluator(lfe, hdr, paysetHint, true, true) -} - -// EnsureBlock ensures that the block, and associated certificate c, are -// written to the speculative ledger, or that some other block for the -// same round is written to the ledger. -// This function can be called concurrently. -func (sl *SpeculativeLedger) EnsureBlock(block *bookkeeping.Block, c agreement.Certificate) { - round := block.Round() - protocolErrorLogged := false - - // As a fallback, bail out if the base (non-speculative) ledger has a - // block for the same round number. - for sl.l.Latest() < round { - err := sl.l.AddBlock(*block, c) - if err == nil { - break - } - - switch err.(type) { - case protocol.Error: - if !protocolErrorLogged { - logging.Base().Errorf("unrecoverable protocol error detected at block %d: %v", round, err) - protocolErrorLogged = true + if r == v.vb.Block().Round() { + delta, ok := v.vb.Delta().Creatables[cidx] + if ok { + if delta.Created && delta.Ctype == ctype { + return delta.Creator, true, nil } - case ledgercore.BlockInLedgerError: - logging.Base().Debugf("could not write block %d to the ledger: %v", round, err) - return // this error implies that l.Latest() >= round - default: - logging.Base().Errorf("could not write block %d to the speculative ledger: %v", round, err) - } - - // If there was an error add a short delay before the next attempt. - time.Sleep(100 * time.Millisecond) - } -} - -// EnsureValidatedBlock ensures that the block, and associated certificate c, are -// written to the speculative ledger, or that some other block for the same round is -// written to the ledger. -func (sl *SpeculativeLedger) EnsureValidatedBlock(vb *ValidatedBlock, c agreement.Certificate) { - round := vb.Block().Round() - - // As a fallback, bail out if the base (non-speculative) ledger has a - // block for the same round number. - for sl.l.Latest() < round { - err := sl.AddValidatedBlock(*vb, c) - if err == nil { - break + return basics.Address{}, false, nil } - - logfn := logging.Base().Errorf - - switch err.(type) { - case ledgercore.BlockInLedgerError: - logfn = logging.Base().Debugf - } - - logfn("could not write block %d to the speculative ledger: %v", round, err) } -} - -// Latest proxies Ledger.Latest() -func (sl *SpeculativeLedger) Latest() basics.Round { - return sl.l.Latest() -} -// VerifiedTransactionCache proxies Ledger.VerifiedTransactionCache() -func (sl *SpeculativeLedger) VerifiedTransactionCache() verify.VerifiedTransactionCache { - return sl.l.VerifiedTransactionCache() -} - -// GenesisHash proxies Ledger.GenesisHash() -func (sl *SpeculativeLedger) GenesisHash() crypto.Digest { - return sl.l.GenesisHash() + return v.l.GetCreatorForRound(r, cidx, ctype) } From cf9eb605a924e6469b77fa598341d38d1a7bb4dd Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 22 Jul 2022 11:48:11 -0400 Subject: [PATCH 04/50] build verified block + spec ledger --- data/pools/transactionPool.go | 17 ++++++++++++----- ledger/ledgercore/validatedBlock.go | 4 ++++ ledger/speculative.go | 6 +++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 2d711e1e68..5fdb375164 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -59,7 +59,7 @@ type TransactionPool struct { logAssembleStats bool expFeeFactor uint64 txPoolMaxSize int - ledger *ledger.Ledger + ledger ledger.LedgerForEvaluator mu deadlock.Mutex cond sync.Cond @@ -139,17 +139,22 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverNewLedger(ledger *ledger.Ledger) *TransactionPool { +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) *TransactionPool { pool.mu.Lock() defer pool.mu.Unlock() + specLedger, err := ledger.MakeValidatedBlockAsLFE(block, pool.ledger) + if err != nil { + return nil + } + pool.speculatedBlockChan = make(chan speculatedBlock, 1) copy := TransactionPool{ pendingTxids: make(map[transactions.Txid]transactions.SignedTxn), rememberedTxids: make(map[transactions.Txid]transactions.SignedTxn), expiredTxCount: make(map[basics.Round]int), - ledger: ledger, + ledger: specLedger, statusCache: makeStatusCache(pool.cfg.TxPoolSize), logProcessBlockStats: pool.cfg.EnableProcessBlockStats, logAssembleStats: pool.cfg.EnableAssembleStats, @@ -533,8 +538,10 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig } func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { - speculativeLedger := nil - speculativePool := pool.copyTransactionPoolOverNewLedger(speculatedLedger) + + vb := ledgercore.MakeValidatedBlock(block, delta) + + speculativePool := pool.copyTransactionPoolOverSpecLedger(&vb) // check block's prev == ledger latest speculativePool.OnNewBlock(block, delta) diff --git a/ledger/ledgercore/validatedBlock.go b/ledger/ledgercore/validatedBlock.go index 299830dd97..400fa3ea15 100644 --- a/ledger/ledgercore/validatedBlock.go +++ b/ledger/ledgercore/validatedBlock.go @@ -68,6 +68,10 @@ func (vb ValidatedBlock) CheckDup(currentProto config.ConsensusParams, firstVali return nil } +func (vb ValidatedBlock) Hash() bookkeeping.BlockHash { + return vb.blk.Hash() +} + // MakeValidatedBlock creates a validated block. func MakeValidatedBlock(blk bookkeeping.Block, delta StateDelta) ValidatedBlock { return ValidatedBlock{ diff --git a/ledger/speculative.go b/ledger/speculative.go index 267cb23712..04e8810574 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -29,12 +29,12 @@ type LedgerForEvaluator interface { // Needed for cow.go BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error - LookupWithoutRewards(basics.Round, basics.Address) (basics.AccountData, basics.Round, error) + LookupWithoutRewards(basics.Round, basics.Address) (ledgercore.AccountData, basics.Round, error) GetCreatorForRound(basics.Round, basics.CreatableIndex, basics.CreatableType) (basics.Address, bool, error) // Needed for the evaluator GenesisHash() crypto.Digest - Totals(basics.Round) (ledgercore.AccountTotals, error) + //Totals(basics.Round) (ledgercore.AccountTotals, error) CompactCertVoters(basics.Round) (*ledgercore.VotersForRound, error) } @@ -74,7 +74,7 @@ type validatedBlockAsLFE struct { // makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a // ValidatedBlock. -func makeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { +func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { return &validatedBlockAsLFE{ l: l, vb: vb, From 3021f03ad94a6c0d98bc95bb64f6e886bf018e3c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 22 Jul 2022 13:20:47 -0400 Subject: [PATCH 05/50] checkpoint: making startevaluator compile --- ledger/speculative.go | 58 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/ledger/speculative.go b/ledger/speculative.go index 04e8810574..bc5e01d98a 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -17,12 +17,16 @@ package ledger import ( + "fmt" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/internal" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" ) type LedgerForEvaluator interface { @@ -34,8 +38,9 @@ type LedgerForEvaluator interface { // Needed for the evaluator GenesisHash() crypto.Digest - //Totals(basics.Round) (ledgercore.AccountTotals, error) + Latest() basics.Round CompactCertVoters(basics.Round) (*ledgercore.VotersForRound, error) + GenesisProto() config.ConsensusParams } // validatedBlockAsLFE presents a LedgerForEvaluator interface on top of @@ -75,6 +80,9 @@ type validatedBlockAsLFE struct { // makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a // ValidatedBlock. func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { + if vb.Block().Round().SubSaturate(1) != l.Latest() { + return nil, fmt.Errorf("MakeValidatedBlockAsLFE: Ledger round %d mismatches next block round %d", l.Latest(), vb.Block().Round()) + } return &validatedBlockAsLFE{ l: l, vb: vb, @@ -104,6 +112,26 @@ func (v *validatedBlockAsLFE) GenesisHash() crypto.Digest { return v.l.GenesisHash() } +// Latest implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) Latest() basics.Round { + return v.vb.Block().Round() +} + +// CompactCertVoters implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) CompactCertVoters(r basics.Round) (*ledgercore.VotersForRound, error) { + if r >= v.vb.Block().Round() { + // We do not support computing the compact cert voters for rounds + // that have not been committed to the ledger yet. This should not + // be a problem as long as the agreement pipeline depth does not + // exceed CompactCertVotersLookback. + err := fmt.Errorf("validatedBlockAsLFE.CompactCertVoters(%d): validated block is for round %d, voters not available", r, v.vb.Block().Round()) + logging.Base().Warn(err.Error()) + return nil, err + } + + return v.l.CompactCertVoters(r) +} + // GetCreatorForRound implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) GetCreatorForRound(r basics.Round, cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) { if r == v.vb.Block().Round() { @@ -118,3 +146,31 @@ func (v *validatedBlockAsLFE) GetCreatorForRound(r basics.Round, cidx basics.Cre return v.l.GetCreatorForRound(r, cidx, ctype) } + +// Totals implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) LatestTotals() (basics.Round, ledgercore.AccountTotals, error) { + return v.vb.Block().Round(), v.vb.Delta().Totals, nil +} + +// GenesisProto returns the initial protocol for this ledger. +func (v *validatedBlockAsLFE) GenesisProto() config.ConsensusParams { + return v.l.GenesisProto() +} + +// LookupApplication loads an application resource that matches the request parameters from the ledger. +func (v *validatedBlockAsLFE) LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) { + r, err := l.lookupResource(rnd, addr, basics.CreatableIndex(aidx), basics.AppCreatable) + return ledgercore.AppResource{AppParams: r.AppParams, AppLocalState: r.AppLocalState}, err +} + +// StartEvaluator starts a block evaluator with a particular block header. +// The block header's Branch value determines which speculative branch is used. +// This is intended to be used by the transaction pool assembly code. +func (v *validatedBlockAsLFE) StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint int) (*internal.BlockEvaluator, error) { + if hdr.Round.SubSaturate(1) != v.Latest() { + return nil, fmt.Errorf("StartEvaluator: LFE round %d mismatches next block round %d", v.Latest(), hdr.Round) + } + + evalopts := internal.EvaluatorOptions{PaysetHint: paysetHint, Validate: true, Generate: true} + return internal.StartEvaluator(v, hdr, evalopts) +} From 7c96473f0f34369d1d8d7acdebb7807c4a66d55d Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 25 Jul 2022 15:21:46 -0400 Subject: [PATCH 06/50] add new LFE LedgerForEvaluator functionality --- ledger/speculative.go | 50 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/ledger/speculative.go b/ledger/speculative.go index bc5e01d98a..e86e549beb 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -35,6 +35,9 @@ type LedgerForEvaluator interface { CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error LookupWithoutRewards(basics.Round, basics.Address) (ledgercore.AccountData, basics.Round, error) GetCreatorForRound(basics.Round, basics.CreatableIndex, basics.CreatableType) (basics.Address, bool, error) + StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint, maxTxnBytesPerBlock int) (*internal.BlockEvaluator, error) + LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) + LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) // Needed for the evaluator GenesisHash() crypto.Digest @@ -159,18 +162,55 @@ func (v *validatedBlockAsLFE) GenesisProto() config.ConsensusParams { // LookupApplication loads an application resource that matches the request parameters from the ledger. func (v *validatedBlockAsLFE) LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) { - r, err := l.lookupResource(rnd, addr, basics.CreatableIndex(aidx), basics.AppCreatable) - return ledgercore.AppResource{AppParams: r.AppParams, AppLocalState: r.AppLocalState}, err + + if rnd == v.vb.Block().Round() { + // Intentionally apply (pending) rewards up to rnd. + res, ok := v.vb.Delta().Accts.GetResource(addr, basics.CreatableIndex(aidx), basics.AppCreatable) + if ok { + return ledgercore.AppResource{AppParams: res.AppParams, AppLocalState: res.AppLocalState}, nil + } + } + return v.l.LookupApplication(rnd, addr, aidx) +} + +// LookupAsset loads an asset resource that matches the request parameters from the ledger. +func (v *validatedBlockAsLFE) LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) { + + if rnd == v.vb.Block().Round() { + // Intentionally apply (pending) rewards up to rnd. + res, ok := v.vb.Delta().Accts.GetResource(addr, basics.CreatableIndex(aidx), basics.AppCreatable) + if ok { + return ledgercore.AssetResource{AssetParams: res.AssetParams, AssetHolding: res.AssetHolding}, nil + } + } + return v.l.LookupAsset(rnd, addr, aidx) +} + +// LookupWithoutRewards implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) LookupWithoutRewards(r basics.Round, a basics.Address) (ledgercore.AccountData, basics.Round, error) { + if r == v.vb.Block().Round() { + data, ok := v.vb.Delta().Accts.GetData(a) + if ok { + return data, r, nil + } + } + + return v.l.LookupWithoutRewards(r, a) } // StartEvaluator starts a block evaluator with a particular block header. // The block header's Branch value determines which speculative branch is used. // This is intended to be used by the transaction pool assembly code. -func (v *validatedBlockAsLFE) StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint int) (*internal.BlockEvaluator, error) { +func (v *validatedBlockAsLFE) StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint, maxTxnBytesPerBlock int) (*internal.BlockEvaluator, error) { if hdr.Round.SubSaturate(1) != v.Latest() { return nil, fmt.Errorf("StartEvaluator: LFE round %d mismatches next block round %d", v.Latest(), hdr.Round) } - evalopts := internal.EvaluatorOptions{PaysetHint: paysetHint, Validate: true, Generate: true} - return internal.StartEvaluator(v, hdr, evalopts) + return internal.StartEvaluator(v, hdr, + internal.EvaluatorOptions{ + PaysetHint: paysetHint, + Generate: true, + Validate: true, + MaxTxnBytesPerBlock: maxTxnBytesPerBlock, + }) } From c54fa48b0bedc55d361ef1fc454e195693e48eab Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 25 Jul 2022 15:29:34 -0400 Subject: [PATCH 07/50] forward verified txns cache --- ledger/speculative.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ledger/speculative.go b/ledger/speculative.go index e86e549beb..b80961e60a 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -24,6 +24,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/data/transactions/verify" "github.com/algorand/go-algorand/ledger/internal" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" @@ -38,6 +39,7 @@ type LedgerForEvaluator interface { StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint, maxTxnBytesPerBlock int) (*internal.BlockEvaluator, error) LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) + VerifiedTransactionCache() verify.VerifiedTransactionCache // Needed for the evaluator GenesisHash() crypto.Digest @@ -198,9 +200,12 @@ func (v *validatedBlockAsLFE) LookupWithoutRewards(r basics.Round, a basics.Addr return v.l.LookupWithoutRewards(r, a) } -// StartEvaluator starts a block evaluator with a particular block header. -// The block header's Branch value determines which speculative branch is used. -// This is intended to be used by the transaction pool assembly code. +// VerifiedTransactionCache implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) VerifiedTransactionCache() verify.VerifiedTransactionCache { + return v.l.VerifiedTransactionCache() +} + +// StartEvaluator implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint, maxTxnBytesPerBlock int) (*internal.BlockEvaluator, error) { if hdr.Round.SubSaturate(1) != v.Latest() { return nil, fmt.Errorf("StartEvaluator: LFE round %d mismatches next block round %d", v.Latest(), hdr.Round) From 2484101fac3c2f4094f968ee09521f04c11623fc Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 25 Jul 2022 16:09:30 -0400 Subject: [PATCH 08/50] checkpoint: use spec pool --- data/pools/transactionPool.go | 44 +++++++++++++---------------------- ledger/speculative.go | 1 + 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 5fdb375164..572c130d59 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -93,12 +93,12 @@ type TransactionPool struct { // proposalAssemblyTime is the ProposalAssemblyTime configured for this node. proposalAssemblyTime time.Duration + cancel <-chan bool - cfg config.Local - speculatedBlockChan chan speculatedBlock + cfg config.Local } -type speculatedBlock struct { +type SpeculativeBlock struct { blk *ledgercore.ValidatedBlock branch bookkeeping.BlockHash } @@ -139,7 +139,7 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) *TransactionPool { +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock, cancelComputations <-chan bool) *TransactionPool { pool.mu.Lock() defer pool.mu.Unlock() @@ -148,10 +148,9 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore return nil } - pool.speculatedBlockChan = make(chan speculatedBlock, 1) - copy := TransactionPool{ - pendingTxids: make(map[transactions.Txid]transactions.SignedTxn), + // TODO(yossi) verify this assumption + pendingTxids: pool.pendingTxids, // it is safe to shallow copy pendingTxids since this map is only (atomically) swapped and never directly changed rememberedTxids: make(map[transactions.Txid]transactions.SignedTxn), expiredTxCount: make(map[basics.Round]int), ledger: specLedger, @@ -163,25 +162,10 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore proposalAssemblyTime: pool.cfg.ProposalAssemblyTime, log: pool.log, //TODO(yossi) change the logger's copy to add a prefix indicating this is the pool's copy/speculation. So failures will be indicative cfg: pool.cfg, - speculatedBlockChan: pool.speculatedBlockChan, + cancel: cancelComputations, } copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu - copy.recomputeBlockEvaluator(nil, 0) - - // deep copy txns - for txid, txn := range pool.pendingTxids { - copy.pendingTxids[txid] = txn - } - - // TODO(yossi) I think it's okay to not do the following deep copying. Is it? - for txid, txn := range pool.rememberedTxids { - copy.rememberedTxids[txid] = txn - } - for round, i := range pool.expiredTxCount { - copy.expiredTxCount[round] = i - } - return © } @@ -537,21 +521,19 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } -func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { +func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta, specBlock chan<- SpeculativeBlock, cancel <-chan bool) { vb := ledgercore.MakeValidatedBlock(block, delta) + speculativePool := pool.copyTransactionPoolOverSpecLedger(&vb, cancel) - speculativePool := pool.copyTransactionPoolOverSpecLedger(&vb) - // check block's prev == ledger latest speculativePool.OnNewBlock(block, delta) if speculativePool.assemblyResults.err != nil { return } - speculativePool.speculatedBlockChan <- speculatedBlock{blk: speculativePool.assemblyResults.blk, branch: block.Hash()} select { - case speculativePool.speculatedBlockChan <- speculatedBlock{blk: speculativePool.assemblyResults.blk, branch: block.Hash()}: + case specBlock <- SpeculativeBlock{blk: speculativePool.assemblyResults.blk, branch: block.Hash()}: default: speculativePool.log.Warnf("failed writing speculated block to channel, channel already has a block") } @@ -785,6 +767,12 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact // Feed the transactions in order for _, txgroup := range txgroups { + switch { + case <-pool.cancel: + return + default: // continue processing transactions + } + if len(txgroup) == 0 { asmStats.InvalidCount++ continue diff --git a/ledger/speculative.go b/ledger/speculative.go index b80961e60a..38dfcb5932 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -201,6 +201,7 @@ func (v *validatedBlockAsLFE) LookupWithoutRewards(r basics.Round, a basics.Addr } // VerifiedTransactionCache implements the ledgerForEvaluator interface. +// TODO(yossi) should we forward this cache? func (v *validatedBlockAsLFE) VerifiedTransactionCache() verify.VerifiedTransactionCache { return v.l.VerifiedTransactionCache() } From 251c02152607b2d9ba947183bb03e31b5bd42a8f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 25 Jul 2022 16:20:32 -0400 Subject: [PATCH 09/50] fixing tests --- ledger/speculative_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go index cd131e7712..5c0ff09eba 100644 --- a/ledger/speculative_test.go +++ b/ledger/speculative_test.go @@ -27,12 +27,13 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" ) func TestSpeculative(t *testing.T) { - genesisInitState, _ := testGenerateInitState(t, protocol.ConsensusCurrentVersion, 1000) + genesisInitState, _ := ledgertesting.GenerateInitState(t, protocol.ConsensusCurrentVersion, 1000) const inMem = true cfg := config.GetDefaultLocal() log := logging.TestingLog(t) @@ -43,9 +44,6 @@ func TestSpeculative(t *testing.T) { blk0, err := l.BlockHdr(l.Latest()) require.NoError(t, err) - sl, err := MakeSpeculativeLedger(l) - require.NoError(t, err) - var blk1 bookkeeping.Block blk1.CurrentProtocol = protocol.ConsensusCurrentVersion blk1.Branch = blk0.Hash() @@ -54,6 +52,9 @@ func TestSpeculative(t *testing.T) { blk1.BlockHeader.GenesisHash = genesisInitState.GenesisHash blk1.BlockHeader.Round = l.Latest() + 1 + sl, err := MakeSpeculativeLedger(l) + require.NoError(t, err) + state, err := sl.eval(context.Background(), bookkeeping.BlockHash{}, blk1, false, nil) require.NoError(t, err) From c3d4caa68b0cabf43735be24103cbb7e0413ec47 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 26 Jul 2022 13:31:36 -0400 Subject: [PATCH 10/50] speculative ledger unittest updates --- data/pools/transactionPool.go | 1 + ledger/speculative.go | 21 ++++++++++++-- ledger/speculative_test.go | 52 +++++++++++------------------------ 3 files changed, 35 insertions(+), 39 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 572c130d59..576ea7c11c 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -526,6 +526,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt vb := ledgercore.MakeValidatedBlock(block, delta) speculativePool := pool.copyTransactionPoolOverSpecLedger(&vb, cancel) + // TODO(yossi): this would process _all_ pending transactions. We could, however, finish after we have a block. speculativePool.OnNewBlock(block, delta) if speculativePool.assemblyResults.err != nil { diff --git a/ledger/speculative.go b/ledger/speculative.go index 38dfcb5932..b08ca6220d 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -85,8 +85,16 @@ type validatedBlockAsLFE struct { // makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a // ValidatedBlock. func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { - if vb.Block().Round().SubSaturate(1) != l.Latest() { - return nil, fmt.Errorf("MakeValidatedBlockAsLFE: Ledger round %d mismatches next block round %d", l.Latest(), vb.Block().Round()) + latestRound := l.Latest() + if vb.Block().Round().SubSaturate(1) != latestRound { + return nil, fmt.Errorf("MakeValidatedBlockAsLFE: Ledger round %d mismatches next block round %d", latestRound, vb.Block().Round()) + } + hdr, err := l.BlockHdr(latestRound) + if err != nil { + return nil, err + } + if vb.Block().Branch != hdr.Hash() { + return nil, fmt.Errorf("MakeValidatedBlockAsLFE: Ledger latest block hash %x mismatches block's prev hash %x", hdr.Hash(), vb.Block().Branch) } return &validatedBlockAsLFE{ l: l, @@ -172,12 +180,14 @@ func (v *validatedBlockAsLFE) LookupApplication(rnd basics.Round, addr basics.Ad return ledgercore.AppResource{AppParams: res.AppParams, AppLocalState: res.AppLocalState}, nil } } + + // app didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger + rnd = rnd.SubSaturate(1) return v.l.LookupApplication(rnd, addr, aidx) } // LookupAsset loads an asset resource that matches the request parameters from the ledger. func (v *validatedBlockAsLFE) LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) { - if rnd == v.vb.Block().Round() { // Intentionally apply (pending) rewards up to rnd. res, ok := v.vb.Delta().Accts.GetResource(addr, basics.CreatableIndex(aidx), basics.AppCreatable) @@ -185,6 +195,9 @@ func (v *validatedBlockAsLFE) LookupAsset(rnd basics.Round, addr basics.Address, return ledgercore.AssetResource{AssetParams: res.AssetParams, AssetHolding: res.AssetHolding}, nil } } + + // asset didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger + rnd = rnd.SubSaturate(1) return v.l.LookupAsset(rnd, addr, aidx) } @@ -197,6 +210,8 @@ func (v *validatedBlockAsLFE) LookupWithoutRewards(r basics.Round, a basics.Addr } } + // account didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger + r = r.SubSaturate(1) return v.l.LookupWithoutRewards(r, a) } diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go index 5c0ff09eba..c1f539b6d0 100644 --- a/ledger/speculative_test.go +++ b/ledger/speculative_test.go @@ -23,10 +23,11 @@ import ( "github.com/stretchr/testify/require" "github.com/algorand/go-algorand/config" - "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/internal" + "github.com/algorand/go-algorand/ledger/ledgercore" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -52,31 +53,17 @@ func TestSpeculative(t *testing.T) { blk1.BlockHeader.GenesisHash = genesisInitState.GenesisHash blk1.BlockHeader.Round = l.Latest() + 1 - sl, err := MakeSpeculativeLedger(l) - require.NoError(t, err) + //sl, err := MakeSpeculativeLedger(l) + //require.NoError(t, err) - state, err := sl.eval(context.Background(), bookkeeping.BlockHash{}, blk1, false, nil) - require.NoError(t, err) + state, err := internal.Eval(context.Background(), l, blk1, false, l.VerifiedTransactionCache(), nil) - vblk1 := ValidatedBlock{ - blk: blk1, - state: state, - } - - err = sl.AddSpeculativeBlock(vblk1) require.NoError(t, err) - _, err = sl.ConsensusVersion(blk1.BlockHeader.Round, bookkeeping.BlockHash{}) - require.Error(t, err) - - var randomLeaf bookkeeping.BlockHash - crypto.RandBytes(randomLeaf[:]) - _, err = sl.ConsensusVersion(blk1.BlockHeader.Round, randomLeaf) - require.Error(t, err) + vblk1 := ledgercore.MakeValidatedBlock(blk1, state) - cv1, err := sl.ConsensusVersion(blk1.BlockHeader.Round, blk1.Hash()) + blk1aslfe, err := MakeValidatedBlockAsLFE(&vblk1, l) require.NoError(t, err) - require.Equal(t, cv1, blk1.CurrentProtocol) blk2 := blk1 blk2.BlockHeader.Round++ @@ -110,29 +97,22 @@ func TestSpeculative(t *testing.T) { HasGenesisID: true, }) - state, err = sl.eval(context.Background(), bookkeeping.BlockHash{}, blk2, false, nil) - require.Error(t, err) - - state, err = sl.eval(context.Background(), blk1.Hash(), blk2, false, nil) - require.NoError(t, err) - - vblk2 := ValidatedBlock{ - blk: blk2, - state: state, - } - - err = sl.AddSpeculativeBlock(vblk2) + state, err = internal.Eval(context.Background(), blk1aslfe, blk2, false, l.VerifiedTransactionCache(), nil) require.NoError(t, err) - ad11, err := sl.Lookup(blk1.Round(), blk1.Hash(), addr1) + vblk2 := ledgercore.MakeValidatedBlock(blk2, state) + blk2aslfe, err := MakeValidatedBlockAsLFE(&vblk2, blk1aslfe) require.NoError(t, err) - ad22, err := sl.Lookup(blk2.Round(), blk2.Hash(), addr1) + ad11, rnd, err := blk2aslfe.LookupWithoutRewards(blk1.Round(), addr1) require.NoError(t, err) + // account was never changed + require.Equal(t, rnd, basics.Round(0)) - ad12, err := sl.Lookup(blk1.Round(), blk2.Hash(), addr1) + ad22, rnd, err := blk2aslfe.LookupWithoutRewards(blk2.Round(), addr1) require.NoError(t, err) + // account changed at blk2 + require.Equal(t, rnd, blk2.Round()) - require.Equal(t, ad12.MicroAlgos.Raw, ad11.MicroAlgos.Raw) require.Equal(t, ad22.MicroAlgos.Raw, ad11.MicroAlgos.Raw-1000000) } From 928e818dcf32b694d4ba112518041405318b1a68 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 26 Jul 2022 13:41:05 -0400 Subject: [PATCH 11/50] bug fix: subtract round only as fallback --- ledger/speculative.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ledger/speculative.go b/ledger/speculative.go index b08ca6220d..838414abe2 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -179,10 +179,11 @@ func (v *validatedBlockAsLFE) LookupApplication(rnd basics.Round, addr basics.Ad if ok { return ledgercore.AppResource{AppParams: res.AppParams, AppLocalState: res.AppLocalState}, nil } + + // fall back to looking up asset in ledger, until previous block + rnd = v.vb.Block().Round() - 1 } - // app didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger - rnd = rnd.SubSaturate(1) return v.l.LookupApplication(rnd, addr, aidx) } @@ -194,25 +195,26 @@ func (v *validatedBlockAsLFE) LookupAsset(rnd basics.Round, addr basics.Address, if ok { return ledgercore.AssetResource{AssetParams: res.AssetParams, AssetHolding: res.AssetHolding}, nil } + // fall back to looking up asset in ledger, until previous block + rnd = v.vb.Block().Round() - 1 } - // asset didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger - rnd = rnd.SubSaturate(1) return v.l.LookupAsset(rnd, addr, aidx) } // LookupWithoutRewards implements the ledgerForEvaluator interface. -func (v *validatedBlockAsLFE) LookupWithoutRewards(r basics.Round, a basics.Address) (ledgercore.AccountData, basics.Round, error) { - if r == v.vb.Block().Round() { +func (v *validatedBlockAsLFE) LookupWithoutRewards(rnd basics.Round, a basics.Address) (ledgercore.AccountData, basics.Round, error) { + if rnd == v.vb.Block().Round() { data, ok := v.vb.Delta().Accts.GetData(a) if ok { - return data, r, nil + return data, rnd, nil } + // fall back to looking up account in ledger, until previous block + rnd = v.vb.Block().Round() - 1 } // account didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger - r = r.SubSaturate(1) - return v.l.LookupWithoutRewards(r, a) + return v.l.LookupWithoutRewards(rnd, a) } // VerifiedTransactionCache implements the ledgerForEvaluator interface. From d9403a833ce8a04c35d4f7c2e9c383bc2249a621 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 26 Jul 2022 14:30:35 -0400 Subject: [PATCH 12/50] add funcs to read create/read spec blocks --- data/pools/transactionPool.go | 65 ++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 576ea7c11c..524e4d7af1 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -17,6 +17,7 @@ package pools import ( + "context" "errors" "fmt" "sync" @@ -93,14 +94,12 @@ type TransactionPool struct { // proposalAssemblyTime is the ProposalAssemblyTime configured for this node. proposalAssemblyTime time.Duration - cancel <-chan bool - cfg config.Local -} + ctx context.Context + cancelSpec context.CancelFunc + specBlock chan *ledgercore.ValidatedBlock -type SpeculativeBlock struct { - blk *ledgercore.ValidatedBlock - branch bookkeeping.BlockHash + cfg config.Local } // BlockEvaluator defines the block evaluator interface exposed by the ledger package. @@ -132,6 +131,7 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo proposalAssemblyTime: cfg.ProposalAssemblyTime, log: log, cfg: cfg, + ctx: context.Background(), } pool.cond.L = &pool.mu pool.assemblyCond.L = &pool.assemblyMu @@ -139,15 +139,17 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock, cancelComputations <-chan bool) *TransactionPool { +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, error) { pool.mu.Lock() defer pool.mu.Unlock() specLedger, err := ledger.MakeValidatedBlockAsLFE(block, pool.ledger) if err != nil { - return nil + return nil, nil, err } + ctx, cancel := context.WithCancel(pool.ctx) + copy := TransactionPool{ // TODO(yossi) verify this assumption pendingTxids: pool.pendingTxids, // it is safe to shallow copy pendingTxids since this map is only (atomically) swapped and never directly changed @@ -162,11 +164,19 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore proposalAssemblyTime: pool.cfg.ProposalAssemblyTime, log: pool.log, //TODO(yossi) change the logger's copy to add a prefix indicating this is the pool's copy/speculation. So failures will be indicative cfg: pool.cfg, - cancel: cancelComputations, + ctx: ctx, } copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu - return © + + // cancel any pending speculative assembly + if pool.cancelSpec != nil { + pool.cancelSpec() + } + pool.cancelSpec = cancel + pool.specBlock = make(chan *ledgercore.ValidatedBlock, 1) + + return ©, pool.specBlock, nil } // poolAsmResults is used to syncronize the state of the block assembly process. @@ -521,10 +531,13 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } -func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta, specBlock chan<- SpeculativeBlock, cancel <-chan bool) { - +func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { vb := ledgercore.MakeValidatedBlock(block, delta) - speculativePool := pool.copyTransactionPoolOverSpecLedger(&vb, cancel) + speculativePool, outchan, err := pool.copyTransactionPoolOverSpecLedger(&vb) + if err != nil { + pool.log.Warnf("OnNewSpeculativeBlock: %v", err) + return + } // TODO(yossi): this would process _all_ pending transactions. We could, however, finish after we have a block. speculativePool.OnNewBlock(block, delta) @@ -534,9 +547,27 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt } select { - case specBlock <- SpeculativeBlock{blk: speculativePool.assemblyResults.blk, branch: block.Hash()}: + case outchan <- speculativePool.assemblyResults.blk: + default: + speculativePool.log.Errorf("failed writing speculated block to channel, channel already has a block") + } +} + +func (pool *TransactionPool) TryReadSpeculativeBlock(branch bookkeeping.BlockHash) (*ledgercore.ValidatedBlock, error) { + pool.mu.Lock() + defer pool.mu.Unlock() + + select { + case vb := <-pool.specBlock: + if vb == nil { + return nil, fmt.Errorf("speculation block is nil") + } + if vb.Block().Branch != branch { + return nil, fmt.Errorf("misspeculated block: branch = %x vs. latest hash = %x", vb.Block().Branch, branch) + } + return vb, nil default: - speculativePool.log.Warnf("failed writing speculated block to channel, channel already has a block") + return nil, fmt.Errorf("speculation block not ready") } } @@ -768,8 +799,8 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact // Feed the transactions in order for _, txgroup := range txgroups { - switch { - case <-pool.cancel: + select { + case <-pool.ctx.Done(): return default: // continue processing transactions } From 847e521ba0a31a861069eb2cdd7b0b7df7fc80ac Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 26 Jul 2022 14:53:00 -0400 Subject: [PATCH 13/50] checkpoint: stop assembly at first new block --- data/pools/transactionPool.go | 55 +++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 524e4d7af1..c4c1466092 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -135,7 +135,7 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo } pool.cond.L = &pool.mu pool.assemblyCond.L = &pool.assemblyMu - pool.recomputeBlockEvaluator(nil, 0) + pool.recomputeBlockEvaluator(nil, 0, false) return &pool } @@ -231,7 +231,7 @@ func (pool *TransactionPool) Reset() { pool.numPendingWholeBlocks = 0 pool.pendingBlockEvaluator = nil pool.statusCache.reset() - pool.recomputeBlockEvaluator(nil, 0) + pool.recomputeBlockEvaluator(nil, 0, false) } // NumExpired returns the number of transactions that expired at the @@ -430,17 +430,18 @@ func (pool *TransactionPool) remember(txgroup []transactions.SignedTxn) error { params := poolIngestParams{ recomputing: false, } - return pool.ingest(txgroup, params) + _, err := pool.ingest(txgroup, params, false) + return err } // add tries to add the transaction group to the pool, bypassing the fee // priority checks. -func (pool *TransactionPool) add(txgroup []transactions.SignedTxn, stats *telemetryspec.AssembleBlockMetrics) error { +func (pool *TransactionPool) add(txgroup []transactions.SignedTxn, stats *telemetryspec.AssembleBlockMetrics, stopAtFirstFullBlock bool) (stoppedAtBlock bool, err error) { params := poolIngestParams{ recomputing: true, stats: stats, } - return pool.ingest(txgroup, params) + return pool.ingest(txgroup, params, stopAtFirstFullBlock) } // ingest checks whether a transaction group could be remembered in the pool, @@ -448,9 +449,9 @@ func (pool *TransactionPool) add(txgroup []transactions.SignedTxn, stats *teleme // // ingest assumes that pool.mu is locked. It might release the lock // while it waits for OnNewBlock() to be called. -func (pool *TransactionPool) ingest(txgroup []transactions.SignedTxn, params poolIngestParams) error { +func (pool *TransactionPool) ingest(txgroup []transactions.SignedTxn, params poolIngestParams, stopAtFirstFullBlock bool) (stoppedAtBlock bool, err error) { if pool.pendingBlockEvaluator == nil { - return fmt.Errorf("TransactionPool.ingest: no pending block evaluator") + return false, fmt.Errorf("TransactionPool.ingest: no pending block evaluator") } if !params.recomputing { @@ -462,26 +463,26 @@ func (pool *TransactionPool) ingest(txgroup []transactions.SignedTxn, params poo for pool.pendingBlockEvaluator.Round() <= latest && time.Now().Before(waitExpires) { condvar.TimedWait(&pool.cond, timeoutOnNewBlock) if pool.pendingBlockEvaluator == nil { - return fmt.Errorf("TransactionPool.ingest: no pending block evaluator") + return false, fmt.Errorf("TransactionPool.ingest: no pending block evaluator") } } - err := pool.checkSufficientFee(txgroup) + err = pool.checkSufficientFee(txgroup) if err != nil { - return err + return false, err } } - err := pool.addToPendingBlockEvaluator(txgroup, params.recomputing, params.stats) + stoppedAtBlock, err = pool.addToPendingBlockEvaluator(txgroup, params.recomputing, params.stats, stopAtFirstFullBlock) if err != nil { - return err + return false, err } pool.rememberedTxGroups = append(pool.rememberedTxGroups, txgroup) for _, t := range txgroup { pool.rememberedTxids[t.ID()] = t } - return nil + return stoppedAtBlock, nil } // RememberOne stores the provided transaction. @@ -540,7 +541,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt } // TODO(yossi): this would process _all_ pending transactions. We could, however, finish after we have a block. - speculativePool.OnNewBlock(block, delta) + speculativePool.onNewBlock(block, delta, true) if speculativePool.assemblyResults.err != nil { return @@ -571,8 +572,12 @@ func (pool *TransactionPool) TryReadSpeculativeBlock(branch bookkeeping.BlockHas } } -// OnNewBlock excises transactions from the pool that are included in the specified Block or if they've expired func (pool *TransactionPool) OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { + pool.onNewBlock(block, delta, false) +} + +// OnNewBlock excises transactions from the pool that are included in the specified Block or if they've expired +func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta, stopReprocessingAtFirstAsmBlock bool) { var stats telemetryspec.ProcessBlockMetrics var knownCommitted uint var unknownCommitted uint @@ -620,7 +625,7 @@ func (pool *TransactionPool) OnNewBlock(block bookkeeping.Block, delta ledgercor // Recompute the pool by starting from the new latest block. // This has the side-effect of discarding transactions that // have been committed (or that are otherwise no longer valid). - stats = pool.recomputeBlockEvaluator(committedTxids, knownCommitted) + stats = pool.recomputeBlockEvaluator(committedTxids, knownCommitted, false) } stats.KnownCommittedCount = knownCommitted @@ -717,20 +722,23 @@ func (pool *TransactionPool) addToPendingBlockEvaluatorOnce(txgroup []transactio return err } -func (pool *TransactionPool) addToPendingBlockEvaluator(txgroup []transactions.SignedTxn, recomputing bool, stats *telemetryspec.AssembleBlockMetrics) error { - err := pool.addToPendingBlockEvaluatorOnce(txgroup, recomputing, stats) +func (pool *TransactionPool) addToPendingBlockEvaluator(txgroup []transactions.SignedTxn, recomputing bool, stats *telemetryspec.AssembleBlockMetrics, addUntilFirstBlockFull bool) (shouldContinue bool, err error) { + err = pool.addToPendingBlockEvaluatorOnce(txgroup, recomputing, stats) if err == ledgercore.ErrNoSpace { pool.numPendingWholeBlocks++ + if addUntilFirstBlockFull { + return true, nil + } pool.pendingBlockEvaluator.ResetTxnBytes() err = pool.addToPendingBlockEvaluatorOnce(txgroup, recomputing, stats) } - return err + return false, err } // recomputeBlockEvaluator constructs a new BlockEvaluator and feeds all // in-pool transactions to it (removing any transactions that are rejected // by the BlockEvaluator). Expects that the pool.mu mutex would be already taken. -func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transactions.Txid]ledgercore.IncludedTransactions, knownCommitted uint) (stats telemetryspec.ProcessBlockMetrics) { +func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transactions.Txid]ledgercore.IncludedTransactions, knownCommitted uint, stopAtFirstFullBlock bool) (stats telemetryspec.ProcessBlockMetrics) { pool.pendingBlockEvaluator = nil latest := pool.ledger.Latest() @@ -813,7 +821,7 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact asmStats.EarlyCommittedCount++ continue } - err := pool.add(txgroup, &asmStats) + stoppedAtBlock, err := pool.add(txgroup, &asmStats, stopAtFirstFullBlock) if err != nil { for _, tx := range txgroup { pool.statusCache.put(tx, err.Error()) @@ -836,6 +844,9 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact pool.log.Warnf("Cannot re-add pending transaction to pool: %v", err) } } + if stoppedAtBlock { + break + } } pool.assemblyMu.Lock() @@ -1040,7 +1051,7 @@ func (pool *TransactionPool) AssembleDevModeBlock() (assembled *ledgercore.Valid defer pool.mu.Unlock() // drop the current block evaluator and start with a new one. - pool.recomputeBlockEvaluator(nil, 0) + pool.recomputeBlockEvaluator(nil, 0, false) // The above was already pregenerating the entire block, // so there won't be any waiting on this call. From e24af8ca10f5c2de09141911387c9066832ffc18 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 26 Jul 2022 16:29:06 -0400 Subject: [PATCH 14/50] cancel spec asm on any new block (speculative or not) --- data/pools/transactionPool.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index c4c1466092..cc431ef932 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -169,10 +169,6 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu - // cancel any pending speculative assembly - if pool.cancelSpec != nil { - pool.cancelSpec() - } pool.cancelSpec = cancel pool.specBlock = make(chan *ledgercore.ValidatedBlock, 1) @@ -572,11 +568,12 @@ func (pool *TransactionPool) TryReadSpeculativeBlock(branch bookkeeping.BlockHas } } +// OnNewBlock callback calls the internal implementation, onNewBlock with the ``false'' parameter to process all transactions. func (pool *TransactionPool) OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { pool.onNewBlock(block, delta, false) } -// OnNewBlock excises transactions from the pool that are included in the specified Block or if they've expired +// onNewBlock excises transactions from the pool that are included in the specified Block or if they've expired func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta, stopReprocessingAtFirstAsmBlock bool) { var stats telemetryspec.ProcessBlockMetrics var knownCommitted uint @@ -598,6 +595,12 @@ func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercor pool.mu.Lock() defer pool.mu.Unlock() defer pool.cond.Broadcast() + + // cancel any pending speculative assembly + if pool.cancelSpec != nil { + pool.cancelSpec() + } + if pool.pendingBlockEvaluator == nil || block.Round() >= pool.pendingBlockEvaluator.Round() { // Adjust the pool fee threshold. The rules are: // - If there was less than one full block in the pool, reduce From ea8744034ff9232e08ed7406fb18985ae8230f73 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 26 Jul 2022 20:11:58 -0400 Subject: [PATCH 15/50] plumbing to use spec block --- data/pools/transactionPool.go | 70 +++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index cc431ef932..286e932114 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -140,9 +140,6 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo } func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, error) { - pool.mu.Lock() - defer pool.mu.Unlock() - specLedger, err := ledger.MakeValidatedBlockAsLFE(block, pool.ledger) if err != nil { return nil, nil, err @@ -151,7 +148,6 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore ctx, cancel := context.WithCancel(pool.ctx) copy := TransactionPool{ - // TODO(yossi) verify this assumption pendingTxids: pool.pendingTxids, // it is safe to shallow copy pendingTxids since this map is only (atomically) swapped and never directly changed rememberedTxids: make(map[transactions.Txid]transactions.SignedTxn), expiredTxCount: make(map[basics.Round]int), @@ -529,14 +525,26 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig } func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { + + pool.mu.Lock() + // cancel any pending speculative assembly + if pool.cancelSpec != nil { + pool.cancelSpec() + } + + // move remembered txns to pending + pool.rememberCommit(false) + + // create shallow pool copy vb := ledgercore.MakeValidatedBlock(block, delta) speculativePool, outchan, err := pool.copyTransactionPoolOverSpecLedger(&vb) if err != nil { pool.log.Warnf("OnNewSpeculativeBlock: %v", err) return } + pool.mu.Unlock() - // TODO(yossi): this would process _all_ pending transactions. We could, however, finish after we have a block. + // process txns only until one block is full speculativePool.onNewBlock(block, delta, true) if speculativePool.assemblyResults.err != nil { @@ -550,7 +558,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt } } -func (pool *TransactionPool) TryReadSpeculativeBlock(branch bookkeeping.BlockHash) (*ledgercore.ValidatedBlock, error) { +func (pool *TransactionPool) tryReadSpeculativeBlock(branch bookkeeping.BlockHash) (*ledgercore.ValidatedBlock, error) { pool.mu.Lock() defer pool.mu.Unlock() @@ -596,11 +604,6 @@ func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercor defer pool.mu.Unlock() defer pool.cond.Broadcast() - // cancel any pending speculative assembly - if pool.cancelSpec != nil { - pool.cancelSpec() - } - if pool.pendingBlockEvaluator == nil || block.Round() >= pool.pendingBlockEvaluator.Round() { // Adjust the pool fee threshold. The rules are: // - If there was less than one full block in the pool, reduce @@ -647,10 +650,13 @@ func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercor } } -// isAssemblyTimedOut determines if we should keep attempting complete the block assembly by adding more transactions to the pending evaluator, -// or whether we've ran out of time. It takes into consideration the assemblyDeadline that was set by the AssembleBlock function as well as the -// projected time it's going to take to call the GenerateBlock function before the block assembly would be ready. -// The function expects that the pool.assemblyMu lock would be taken before being called. +// isAssemblyTimedOut determines if we should keep attempting complete the block +// assembly by adding more transactions to the pending evaluator, or whether +// we've ran out of time. It takes into consideration the assemblyDeadline that +// was set by the AssembleBlock function as well as the projected time it's +// going to take to call the GenerateBlock function before the block assembly +// would be ready. The function expects that the pool.assemblyMu lock would be +// taken before being called. func (pool *TransactionPool) isAssemblyTimedOut() bool { if pool.assemblyDeadline.IsZero() { // we have no deadline, so no reason to timeout. @@ -687,9 +693,12 @@ func (pool *TransactionPool) addToPendingBlockEvaluatorOnce(txgroup []transactio pool.assemblyMu.Lock() defer pool.assemblyMu.Unlock() if pool.assemblyRound > pool.pendingBlockEvaluator.Round() { - // the block we're assembling now isn't the one the the AssembleBlock is waiting for. While it would be really cool - // to finish generating the block, it would also be pointless to spend time on it. - // we're going to set the ok and assemblyCompletedOrAbandoned to "true" so we can complete this loop asap + // the block we're assembling now isn't the one the the + // AssembleBlock is waiting for. While it would be really cool + // to finish generating the block, it would also be pointless to + // spend time on it. we're going to set the ok and + // assemblyCompletedOrAbandoned to "true" so we can complete + // this loop asap pool.assemblyResults.ok = true pool.assemblyResults.assemblyCompletedOrAbandoned = true stats.StopReason = telemetryspec.AssembleBlockAbandon @@ -960,6 +969,13 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim return nil, ErrStaleBlockAssemblyRequest } + // TODO(yossi) check: maybe we have something assembled! + validatedBlock, err := pool.tryReadSpeculativeBlock(branch) + if err == nil { + assembled = validatedBlock + return assembled, nil + } + pool.assemblyDeadline = deadline pool.assemblyRound = round for time.Now().Before(deadline) && (!pool.assemblyResults.ok || pool.assemblyResults.roundStartedEvaluating != round) { @@ -967,17 +983,23 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim } if !pool.assemblyResults.ok { - // we've passed the deadline, so we're either going to have a partial block, or that we won't make it on time. - // start preparing an empty block in case we'll miss the extra time (assemblyWaitEps). - // the assembleEmptyBlock is using the database, so we want to unlock here and take the lock again later on. + // we've passed the deadline, so we're either going to have a partial + // block, or that we won't make it on time. start preparing an empty + // block in case we'll miss the extra time (assemblyWaitEps). the + // assembleEmptyBlock is using the database, so we want to unlock here + // and take the lock again later on. pool.assemblyMu.Unlock() emptyBlock, emptyBlockErr := pool.assembleEmptyBlock(round) pool.assemblyMu.Lock() if pool.assemblyResults.roundStartedEvaluating > round { - // this case is expected to happen only if the transaction pool was able to construct *two* rounds during the time we were trying to assemble the empty block. - // while this is extreamly unlikely, we need to handle this. the handling it quite straight-forward : - // since the network is already ahead of us, there is no issue here in not generating a block ( since the block would get discarded anyway ) + // this case is expected to happen only if the transaction pool was + // able to construct *two* rounds during the time we were trying to + // assemble the empty block. while this is extreamly unlikely, we + // need to handle this. the handling it quite straight-forward : + // since the network is already ahead of us, there is no issue here + // in not generating a block ( since the block would get discarded + // anyway ) pool.log.Infof("AssembleBlock: requested round is behind transaction pool round after timing out %d < %d", round, pool.assemblyResults.roundStartedEvaluating) return nil, ErrStaleBlockAssemblyRequest } From 3e72c9364fab1c372331fd2e96d79215743a6178 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 26 Jul 2022 20:33:31 -0400 Subject: [PATCH 16/50] get branch from ledger --- data/pools/transactionPool.go | 49 +++++++++++++++++++++-------------- ledger/speculative.go | 10 +++++++ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 286e932114..6471cec2aa 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -180,11 +180,13 @@ type poolAsmResults struct { blk *ledgercore.ValidatedBlock stats telemetryspec.AssembleBlockMetrics err error - // roundStartedEvaluating is the round which we were attempted to evaluate last. It's a good measure for - // which round we started evaluating, but not a measure to whether the evaluation is complete. + // roundStartedEvaluating is the round which we were attempted to evaluate + // last. It's a good measure for which round we started evaluating, but not + // a measure to whether the evaluation is complete. roundStartedEvaluating basics.Round - // assemblyCompletedOrAbandoned is *not* protected via the pool.assemblyMu lock and should be accessed only from the OnNewBlock goroutine. - // it's equivalent to the "ok" variable, and used for avoiding taking the lock. + // assemblyCompletedOrAbandoned is *not* protected via the pool.assemblyMu + // lock and should be accessed only from the OnNewBlock goroutine. it's + // equivalent to the "ok" variable, and used for avoiding taking the lock. assemblyCompletedOrAbandoned bool } @@ -559,6 +561,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt } func (pool *TransactionPool) tryReadSpeculativeBlock(branch bookkeeping.BlockHash) (*ledgercore.ValidatedBlock, error) { + // TODO(yossi) is taking the lock here necessary? is it safe (check lock taking order)? pool.mu.Lock() defer pool.mu.Unlock() @@ -961,19 +964,24 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim defer pool.assemblyMu.Unlock() if pool.assemblyResults.roundStartedEvaluating > round { - // we've already assembled a round in the future. Since we're clearly won't go backward, it means - // that the agreement is far behind us, so we're going to return here with error code to let - // the agreement know about it. - // since the network is already ahead of us, there is no issue here in not generating a block ( since the block would get discarded anyway ) + // we've already assembled a round in the future. Since we're clearly + // won't go backward, it means that the agreement is far behind us, so + // we're going to return here with error code to let the agreement know + // about it. since the network is already ahead of us, there is no issue + // here in not generating a block ( since the block would get discarded + // anyway ) pool.log.Infof("AssembleBlock: requested round is behind transaction pool round %d < %d", round, pool.assemblyResults.roundStartedEvaluating) return nil, ErrStaleBlockAssemblyRequest } - // TODO(yossi) check: maybe we have something assembled! - validatedBlock, err := pool.tryReadSpeculativeBlock(branch) + // Maybe we have a block already assembled + prev, err := pool.ledger.Block(round.SubSaturate(1)) if err == nil { - assembled = validatedBlock - return assembled, nil + validatedBlock, err := pool.tryReadSpeculativeBlock(prev.Hash()) + if err == nil { + assembled = validatedBlock + return assembled, nil + } } pool.assemblyDeadline = deadline @@ -1026,9 +1034,10 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim return nil, fmt.Errorf("AssemblyBlock: encountered error for round %d: %v", round, pool.assemblyResults.err) } if pool.assemblyResults.roundStartedEvaluating > round { - // this scenario should not happen unless the txpool is receiving the new blocks via OnNewBlock - // with "jumps" between consecutive blocks ( which is why it's a warning ) - // The "normal" usecase is evaluated on the top of the function. + // this scenario should not happen unless the txpool is receiving the + // new blocks via OnNewBlock with "jumps" between consecutive blocks ( + // which is why it's a warning ) The "normal" usecase is evaluated on + // the top of the function. pool.log.Warnf("AssembleBlock: requested round is behind transaction pool round %d < %d", round, pool.assemblyResults.roundStartedEvaluating) return nil, ErrStaleBlockAssemblyRequest } else if pool.assemblyResults.roundStartedEvaluating == round.SubSaturate(1) { @@ -1044,8 +1053,9 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim return pool.assemblyResults.blk, nil } -// assembleEmptyBlock construct a new block for the given round. Internally it's using the ledger database calls, so callers -// need to be aware that it might take a while before it would return. +// assembleEmptyBlock construct a new block for the given round. Internally it's +// using the ledger database calls, so callers need to be aware that it might +// take a while before it would return. func (pool *TransactionPool) assembleEmptyBlock(round basics.Round) (assembled *ledgercore.ValidatedBlock, err error) { prevRound := round - 1 prev, err := pool.ledger.BlockHdr(prevRound) @@ -1059,8 +1069,9 @@ func (pool *TransactionPool) assembleEmptyBlock(round basics.Round) (assembled * var nonSeqBlockEval ledgercore.ErrNonSequentialBlockEval if errors.As(err, &nonSeqBlockEval) { if nonSeqBlockEval.EvaluatorRound <= nonSeqBlockEval.LatestRound { - // in the case that the ledger have already moved beyond that round, just let the agreement know that - // we don't generate a block and it's perfectly fine. + // in the case that the ledger have already moved beyond that + // round, just let the agreement know that we don't generate a + // block and it's perfectly fine. return nil, ErrStaleBlockAssemblyRequest } } diff --git a/ledger/speculative.go b/ledger/speculative.go index 838414abe2..275185b76f 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -32,6 +32,7 @@ import ( type LedgerForEvaluator interface { // Needed for cow.go + Block(basics.Round) (bookkeeping.Block, error) BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error LookupWithoutRewards(basics.Round, basics.Address) (ledgercore.AccountData, basics.Round, error) @@ -102,6 +103,15 @@ func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator }, nil } +// Block implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) Block(r basics.Round) (bookkeeping.Block, error) { + if r == v.vb.Block().Round() { + return v.vb.Block(), nil + } + + return v.l.Block(r) +} + // BlockHdr implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) BlockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { if r == v.vb.Block().Round() { From 1900f784869a65ab4a95ce5f939a6c0c8eab248e Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 27 Jul 2022 09:45:20 -0400 Subject: [PATCH 17/50] release lock when copy fails --- data/pools/transactionPool.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 6471cec2aa..a667c0ab2e 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -95,9 +95,10 @@ type TransactionPool struct { // proposalAssemblyTime is the ProposalAssemblyTime configured for this node. proposalAssemblyTime time.Duration - ctx context.Context - cancelSpec context.CancelFunc - specBlock chan *ledgercore.ValidatedBlock + ctx context.Context + cancelSpec context.CancelFunc + specBlock chan *ledgercore.ValidatedBlock + specAsmDone chan struct{} cfg config.Local } @@ -161,11 +162,13 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore log: pool.log, //TODO(yossi) change the logger's copy to add a prefix indicating this is the pool's copy/speculation. So failures will be indicative cfg: pool.cfg, ctx: ctx, + specAsmDone: make(chan struct{}), } copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu pool.cancelSpec = cancel + pool.specAsmDone = copy.specAsmDone pool.specBlock = make(chan *ledgercore.ValidatedBlock, 1) return ©, pool.specBlock, nil @@ -532,6 +535,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt // cancel any pending speculative assembly if pool.cancelSpec != nil { pool.cancelSpec() + <-pool.specAsmDone } // move remembered txns to pending @@ -540,11 +544,14 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt // create shallow pool copy vb := ledgercore.MakeValidatedBlock(block, delta) speculativePool, outchan, err := pool.copyTransactionPoolOverSpecLedger(&vb) + pool.mu.Unlock() + + defer close(pool.specAsmDone) + if err != nil { pool.log.Warnf("OnNewSpeculativeBlock: %v", err) return } - pool.mu.Unlock() // process txns only until one block is full speculativePool.onNewBlock(block, delta, true) From 7513a83a3aa1184bdc8eff6a35a4fd074a0b8a88 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Wed, 27 Jul 2022 11:33:09 -0400 Subject: [PATCH 18/50] don't flush changes from shallow pool copy to real pool --- data/pools/transactionPool.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index a667c0ab2e..659ec68091 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -98,7 +98,7 @@ type TransactionPool struct { ctx context.Context cancelSpec context.CancelFunc specBlock chan *ledgercore.ValidatedBlock - specAsmDone chan struct{} + specAsmDone <-chan struct{} cfg config.Local } @@ -140,10 +140,10 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, error) { +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, error) { specLedger, err := ledger.MakeValidatedBlockAsLFE(block, pool.ledger) if err != nil { - return nil, nil, err + return nil, nil, nil, err } ctx, cancel := context.WithCancel(pool.ctx) @@ -162,16 +162,16 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore log: pool.log, //TODO(yossi) change the logger's copy to add a prefix indicating this is the pool's copy/speculation. So failures will be indicative cfg: pool.cfg, ctx: ctx, - specAsmDone: make(chan struct{}), } copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu pool.cancelSpec = cancel - pool.specAsmDone = copy.specAsmDone + specDoneCh := make(chan struct{}) + pool.specAsmDone = specDoneCh pool.specBlock = make(chan *ledgercore.ValidatedBlock, 1) - return ©, pool.specBlock, nil + return ©, pool.specBlock, specDoneCh, nil } // poolAsmResults is used to syncronize the state of the block assembly process. @@ -543,15 +543,14 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt // create shallow pool copy vb := ledgercore.MakeValidatedBlock(block, delta) - speculativePool, outchan, err := pool.copyTransactionPoolOverSpecLedger(&vb) + speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(&vb) pool.mu.Unlock() - defer close(pool.specAsmDone) - if err != nil { pool.log.Warnf("OnNewSpeculativeBlock: %v", err) return } + defer close(specAsmDoneCh) // process txns only until one block is full speculativePool.onNewBlock(block, delta, true) @@ -563,7 +562,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt select { case outchan <- speculativePool.assemblyResults.blk: default: - speculativePool.log.Errorf("failed writing speculated block to channel, channel already has a block") + speculativePool.log.Errorf("failed writing speculative block to channel, channel already has a block") } } @@ -760,7 +759,7 @@ func (pool *TransactionPool) addToPendingBlockEvaluator(txgroup []transactions.S // recomputeBlockEvaluator constructs a new BlockEvaluator and feeds all // in-pool transactions to it (removing any transactions that are rejected // by the BlockEvaluator). Expects that the pool.mu mutex would be already taken. -func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transactions.Txid]ledgercore.IncludedTransactions, knownCommitted uint, stopAtFirstFullBlock bool) (stats telemetryspec.ProcessBlockMetrics) { +func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transactions.Txid]ledgercore.IncludedTransactions, knownCommitted uint, speculativeAsm bool) (stats telemetryspec.ProcessBlockMetrics) { pool.pendingBlockEvaluator = nil latest := pool.ledger.Latest() @@ -843,7 +842,7 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact asmStats.EarlyCommittedCount++ continue } - stoppedAtBlock, err := pool.add(txgroup, &asmStats, stopAtFirstFullBlock) + hasBlock, err := pool.add(txgroup, &asmStats, speculativeAsm) if err != nil { for _, tx := range txgroup { pool.statusCache.put(tx, err.Error()) @@ -865,8 +864,7 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact stats.RemovedInvalidCount++ pool.log.Warnf("Cannot re-add pending transaction to pool: %v", err) } - } - if stoppedAtBlock { + } else if hasBlock { break } } @@ -896,7 +894,10 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact } pool.assemblyMu.Unlock() - pool.rememberCommit(true) + // remember the changes made to the tx pool if we're not in speculative assembly + if !speculativeAsm { + pool.rememberCommit(true) + } return } From 1be951183ca671144d89abf3b6305895266acdb2 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 28 Jul 2022 10:42:09 -0400 Subject: [PATCH 19/50] add BlockHdrCache method to interface --- ledger/speculative.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ledger/speculative.go b/ledger/speculative.go index 275185b76f..70fe10f4d0 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -47,6 +47,7 @@ type LedgerForEvaluator interface { Latest() basics.Round CompactCertVoters(basics.Round) (*ledgercore.VotersForRound, error) GenesisProto() config.ConsensusParams + BlockHdrCached(rnd basics.Round) (hdr bookkeeping.BlockHeader, err error) } // validatedBlockAsLFE presents a LedgerForEvaluator interface on top of @@ -233,6 +234,14 @@ func (v *validatedBlockAsLFE) VerifiedTransactionCache() verify.VerifiedTransact return v.l.VerifiedTransactionCache() } +// VerifiedTransactionCache implements the ledgerForEvaluator interface. +func (v *validatedBlockAsLFE) BlockHdrCached(rnd basics.Round) (hdr bookkeeping.BlockHeader, err error) { + if rnd == v.vb.Block().Round() { + return v.vb.Block().BlockHeader, nil + } + return v.l.BlockHdrCached(rnd) +} + // StartEvaluator implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint, maxTxnBytesPerBlock int) (*internal.BlockEvaluator, error) { if hdr.Round.SubSaturate(1) != v.Latest() { From a7e289c5ce4c5f0e486810b8c9b96f5b204a10f5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 28 Jul 2022 19:40:05 -0400 Subject: [PATCH 20/50] simpler error handling --- data/pools/transactionPool.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 659ec68091..c60837f946 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -143,7 +143,7 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, error) { specLedger, err := ledger.MakeValidatedBlockAsLFE(block, pool.ledger) if err != nil { - return nil, nil, nil, err + return nil, nil, make(chan struct{}), err } ctx, cancel := context.WithCancel(pool.ctx) @@ -544,13 +544,13 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt // create shallow pool copy vb := ledgercore.MakeValidatedBlock(block, delta) speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(&vb) + defer close(specAsmDoneCh) pool.mu.Unlock() if err != nil { pool.log.Warnf("OnNewSpeculativeBlock: %v", err) return } - defer close(specAsmDoneCh) // process txns only until one block is full speculativePool.onNewBlock(block, delta, true) From e6dc3dc1c1acfe9ef4dc62105763d98edad248ac Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 29 Jul 2022 12:17:13 -0400 Subject: [PATCH 21/50] checkpoint: plumbing through agreement --- agreement/abstractions.go | 3 +++ agreement/actions.go | 12 +++++++++++- agreement/demux.go | 10 ++++++++-- agreement/events.go | 5 ++++- agreement/player.go | 16 ++++++++++++++++ agreement/pseudonode.go | 5 +++++ agreement/service.go | 17 ++++++++++++----- agreement/types.go | 10 ++++++++++ 8 files changed, 69 insertions(+), 9 deletions(-) diff --git a/agreement/abstractions.go b/agreement/abstractions.go index cf58aa3725..8b1b3c875a 100644 --- a/agreement/abstractions.go +++ b/agreement/abstractions.go @@ -26,6 +26,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" + "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/protocol" ) @@ -85,6 +86,8 @@ type BlockFactory interface { // nodes on the network can assemble entries, the agreement protocol may // lose liveness. AssembleBlock(basics.Round) (ValidatedBlock, error) + + OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) } // A Ledger represents the sequence of Entries agreed upon by the protocol. diff --git a/agreement/actions.go b/agreement/actions.go index 42cfbcebb2..cd6170815c 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -55,6 +55,7 @@ const ( attest assemble repropose + speculativeAssembly // disk checkpoint @@ -308,7 +309,7 @@ func (a rezeroAction) do(ctx context.Context, s *Service) { } type pseudonodeAction struct { - // assemble, repropose, attest + // assemble, repropose, attest, speculativeAssembly T actionType Round round @@ -345,6 +346,15 @@ func (a pseudonodeAction) do(ctx context.Context, s *Service) { default: s.log.Errorf("pseudonode.MakeProposals call failed %v", err) } + case speculativeAssembly: + events, err := s.loopback.StartSpeculativeBlockAssembly(ctx, a.Round, a.Period) + switch err { + case nil: + s.demux.prioritize(events) + default: + s.log.Errorf("pseudonode.StartSpeculativeBlockAssembly call failed %v", err) + } + case repropose: logEvent := logspec.AgreementEvent{ Type: logspec.VoteAttest, diff --git a/agreement/demux.go b/agreement/demux.go index 70d7fc9abe..bc5fabfdb4 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -190,7 +190,7 @@ func (d *demux) verifyBundle(ctx context.Context, m message, r round, p period, // next blocks until it observes an external input event of interest for the state machine. // // If ok is false, there are no more events so the agreement service should quit. -func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Duration, currentRound round) (e externalEvent, ok bool) { +func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Duration, speculationDeadline time.Duration, currentRound round) (e externalEvent, ok bool) { defer func() { if !ok { return @@ -249,6 +249,7 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat ledgerNextRoundCh := s.Ledger.Wait(nextRound) deadlineCh := s.Clock.TimeoutAt(deadline) fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline) + speculationDeadlineCh := s.Clock.TimeoutAt(speculationDeadline) d.UpdateEventsQueue(eventQueueDemux, 0) d.monitor.dec(demuxCoserviceType) @@ -266,7 +267,7 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat // the pseudonode channel got closed. remove it from the queue and try again. d.queue = d.queue[1:] d.UpdateEventsQueue(eventQueuePseudonode, 0) - return d.next(s, deadline, fastDeadline, currentRound) + return d.next(s, deadline, fastDeadline, speculationDeadline, currentRound) // control case <-s.quit: @@ -300,6 +301,11 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat d.UpdateEventsQueue(eventQueueDemux, 1) d.monitor.inc(demuxCoserviceType) d.monitor.dec(clockCoserviceType) + case <-speculationDeadlineCh: + e = timeoutEvent{T: speculationTimeout, RandomEntropy: s.RandomSource.Uint64(), Round: nextRound} + d.UpdateEventsQueue(eventQueueDemux, 1) + d.monitor.inc(demuxCoserviceType) + d.monitor.dec(clockCoserviceType) // raw case m, open := <-rawVotes: diff --git a/agreement/events.go b/agreement/events.go index a75a32774b..3591687516 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -121,8 +121,11 @@ const ( // period. // // fastTimeout is like timeout but for fast partition recovery. + // speculation timeout marks when the player should start speculative + // block assembly. timeout fastTimeout + speculationTimeout // Other events are delivered from one state machine to another to // communicate some message or as a reply to some message. These events @@ -358,7 +361,7 @@ func (e roundInterruptionEvent) AttachConsensusVersion(v ConsensusVersionView) e } type timeoutEvent struct { - // {timeout,fastTimeout} + // {timeout,fastTimeout,speculationTimeout} T eventType RandomEntropy uint64 diff --git a/agreement/player.go b/agreement/player.go index 2add5711e4..ef4e127733 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -79,6 +79,9 @@ func (p *player) handle(r routerHandle, e event) []action { if e.T == fastTimeout { return p.handleFastTimeout(r, e) } + if e.T == speculationTimeout { + return p.handleSpeculationTimeout(r, e) + } if !p.Napping { r.t.logTimeout(*p) @@ -122,6 +125,15 @@ func (p *player) handle(r routerHandle, e event) []action { } } +func (p *player) handleSpeculationTimeout(r routerHandle, e timeoutEvent) []action { + // start speculative block assembly + if e.Proto.Err != nil { + r.t.log.Errorf("failed to read protocol version for speculationTimeout event (proto %v): %v", e.Proto.Version, e.Proto.Err) + return nil + } + return p.startSpeculativeBlockAsm(r) +} + func (p *player) handleFastTimeout(r routerHandle, e timeoutEvent) []action { if e.Proto.Err != nil { r.t.log.Errorf("failed to read protocol version for fastTimeout event (proto %v): %v", e.Proto.Version, e.Proto.Err) @@ -216,6 +228,10 @@ func (p *player) issueNextVote(r routerHandle) []action { return actions } +func (p *player) startSpeculativeBlockAsm(r routerHandle) (actions []action) { + return append(actions, pseudonodeAction{T: speculativeAssembly, Round: p.Round, Period: p.Period}) +} + func (p *player) issueFastVote(r routerHandle) (actions []action) { actions = p.partitionPolicy(r) diff --git a/agreement/pseudonode.go b/agreement/pseudonode.go index 06a91b210e..9571bf54a4 100644 --- a/agreement/pseudonode.go +++ b/agreement/pseudonode.go @@ -60,6 +60,8 @@ type pseudonode interface { // It returns an error if the pseudonode is unable to perform this. MakeProposals(ctx context.Context, r round, p period) (<-chan externalEvent, error) + StartSpeculativeBlockAssembly(ctx context.Context, r round, p period) (<-chan externalEvent, error) + // MakeVotes returns a vote for a given proposal in some round, period, and step. // // The passed-in context may be used to cancel vote creation and close the channel immediately. @@ -185,6 +187,9 @@ func (n asyncPseudonode) MakeProposals(ctx context.Context, r round, p period) ( } } +func (n asyncPseudonode) StartSpeculativeBlockAssembly(ctx context.Context, r round, p period) (<-chan externalEvent, error) { +} + func (n asyncPseudonode) MakeVotes(ctx context.Context, r round, p period, s step, prop proposalValue, persistStateDone chan error) (chan externalEvent, error) { proposalTask := n.makeVotesTask(ctx, r, p, s, prop, persistStateDone) if len(proposalTask.participation) == 0 { diff --git a/agreement/service.go b/agreement/service.go index 3462349509..e293048241 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -80,9 +80,10 @@ type parameters Parameters // externalDemuxSignals used to syncronize the external signals that goes to the demux with the main loop. type externalDemuxSignals struct { - Deadline time.Duration - FastRecoveryDeadline time.Duration - CurrentRound round + Deadline time.Duration + FastRecoveryDeadline time.Duration + SpeculativeBlockAsmDeadline time.Duration + CurrentRound round } // MakeService creates a new Agreement Service instance given a set of Parameters. @@ -162,7 +163,7 @@ func (s *Service) demuxLoop(ctx context.Context, input chan<- externalEvent, out for a := range output { s.do(ctx, a) extSignals := <-ready - e, ok := s.demux.next(s, extSignals.Deadline, extSignals.FastRecoveryDeadline, extSignals.CurrentRound) + e, ok := s.demux.next(s, extSignals.Deadline, extSignals.FastRecoveryDeadline, extSignals.SpeculativeBlockAsmDeadline, extSignals.CurrentRound) if !ok { close(input) break @@ -220,9 +221,15 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.Clock = clock } + nextVersion, err := s.Ledger.ConsensusVersion(status.Round) + speculativeBlockAsmTime := time.Duration(0) + if err == nil { + speculativeBlockAsmTime = SpeculativeBlockAsmTime(0, nextVersion) + } + for { output <- a - ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, CurrentRound: status.Round} + ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, SpeculativeBlockAsmDeadline: speculativeBlockAsmTime, CurrentRound: status.Round} e, ok := <-input if !ok { break diff --git a/agreement/types.go b/agreement/types.go index 51793aa077..0d86b865dc 100644 --- a/agreement/types.go +++ b/agreement/types.go @@ -38,6 +38,16 @@ func FilterTimeout(p period, v protocol.ConsensusVersion) time.Duration { return config.Consensus[v].AgreementFilterTimeout } +func SpeculativeBlockAsmTime(p period, v protocol.ConsensusVersion) time.Duration { + hardwait := FilterTimeout(p, v) + // TODO(yossi) change from default config to actual config + if hardwait > config.GetDefaultLocal().ProposalAssemblyTime+time.Duration(100*time.Millisecond) { + // TODO(yossi) consider another check that makes sure we waited at least for a bit, so we don't start spec asm for nothing + return hardwait - (config.GetDefaultLocal().ProposalAssemblyTime + time.Duration(100*time.Millisecond)) + } + return time.Duration(0) +} + // DeadlineTimeout is the duration of the second agreement step. func DeadlineTimeout() time.Duration { return deadlineTimeout From ee1641522e4fcdf907e3784dabdefcf017fbe6a5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 1 Aug 2022 15:31:12 -0400 Subject: [PATCH 22/50] checkpoint: MakeValidatedBlockAsLFE --> MakeBlockAsLFE --- agreement/actiontype_string.go | 7 ++-- agreement/eventtype_string.go | 63 +++++++++++++++++----------------- agreement/pseudonode.go | 3 ++ data/pools/transactionPool.go | 7 ++-- ledger/speculative.go | 34 +++++++++++------- ledger/speculative_test.go | 15 ++------ 6 files changed, 65 insertions(+), 64 deletions(-) diff --git a/agreement/actiontype_string.go b/agreement/actiontype_string.go index c27b9138bb..c90d81d46d 100644 --- a/agreement/actiontype_string.go +++ b/agreement/actiontype_string.go @@ -23,12 +23,13 @@ func _() { _ = x[attest-12] _ = x[assemble-13] _ = x[repropose-14] - _ = x[checkpoint-15] + _ = x[speculativeAssembly-15] + _ = x[checkpoint-16] } -const _actionType_name = "noopignorebroadcastrelaydisconnectbroadcastVotesverifyVoteverifyPayloadverifyBundleensurestageDigestrezeroattestassemblereproposecheckpoint" +const _actionType_name = "noopignorebroadcastrelaydisconnectbroadcastVotesverifyVoteverifyPayloadverifyBundleensurestageDigestrezeroattestassemblereproposespeculativeAssemblycheckpoint" -var _actionType_index = [...]uint8{0, 4, 10, 19, 24, 34, 48, 58, 71, 83, 89, 100, 106, 112, 120, 129, 139} +var _actionType_index = [...]uint8{0, 4, 10, 19, 24, 34, 48, 58, 71, 83, 89, 100, 106, 112, 120, 129, 148, 158} func (i actionType) String() string { if i < 0 || i >= actionType(len(_actionType_index)-1) { diff --git a/agreement/eventtype_string.go b/agreement/eventtype_string.go index f8ee701d80..6944ce5862 100644 --- a/agreement/eventtype_string.go +++ b/agreement/eventtype_string.go @@ -18,40 +18,41 @@ func _() { _ = x[roundInterruption-7] _ = x[timeout-8] _ = x[fastTimeout-9] - _ = x[softThreshold-10] - _ = x[certThreshold-11] - _ = x[nextThreshold-12] - _ = x[proposalCommittable-13] - _ = x[proposalAccepted-14] - _ = x[voteFiltered-15] - _ = x[voteMalformed-16] - _ = x[bundleFiltered-17] - _ = x[bundleMalformed-18] - _ = x[payloadRejected-19] - _ = x[payloadMalformed-20] - _ = x[payloadPipelined-21] - _ = x[payloadAccepted-22] - _ = x[proposalFrozen-23] - _ = x[voteAccepted-24] - _ = x[newRound-25] - _ = x[newPeriod-26] - _ = x[readStaging-27] - _ = x[readPinned-28] - _ = x[voteFilterRequest-29] - _ = x[voteFilteredStep-30] - _ = x[nextThresholdStatusRequest-31] - _ = x[nextThresholdStatus-32] - _ = x[freshestBundleRequest-33] - _ = x[freshestBundle-34] - _ = x[dumpVotesRequest-35] - _ = x[dumpVotes-36] - _ = x[wrappedAction-37] - _ = x[checkpointReached-38] + _ = x[speculationTimeout-10] + _ = x[softThreshold-11] + _ = x[certThreshold-12] + _ = x[nextThreshold-13] + _ = x[proposalCommittable-14] + _ = x[proposalAccepted-15] + _ = x[voteFiltered-16] + _ = x[voteMalformed-17] + _ = x[bundleFiltered-18] + _ = x[bundleMalformed-19] + _ = x[payloadRejected-20] + _ = x[payloadMalformed-21] + _ = x[payloadPipelined-22] + _ = x[payloadAccepted-23] + _ = x[proposalFrozen-24] + _ = x[voteAccepted-25] + _ = x[newRound-26] + _ = x[newPeriod-27] + _ = x[readStaging-28] + _ = x[readPinned-29] + _ = x[voteFilterRequest-30] + _ = x[voteFilteredStep-31] + _ = x[nextThresholdStatusRequest-32] + _ = x[nextThresholdStatus-33] + _ = x[freshestBundleRequest-34] + _ = x[freshestBundle-35] + _ = x[dumpVotesRequest-36] + _ = x[dumpVotes-37] + _ = x[wrappedAction-38] + _ = x[checkpointReached-39] } -const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" +const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutspeculationTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" -var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 131, 144, 157, 176, 192, 204, 217, 231, 246, 261, 277, 293, 308, 322, 334, 342, 351, 362, 372, 389, 405, 431, 450, 471, 485, 501, 510, 523, 540} +var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 136, 149, 162, 175, 194, 210, 222, 235, 249, 264, 279, 295, 311, 326, 340, 352, 360, 369, 380, 390, 407, 423, 449, 468, 489, 503, 519, 528, 541, 558} func (i eventType) String() string { if i < 0 || i >= eventType(len(_eventType_index)-1) { diff --git a/agreement/pseudonode.go b/agreement/pseudonode.go index 9571bf54a4..3862d859db 100644 --- a/agreement/pseudonode.go +++ b/agreement/pseudonode.go @@ -188,6 +188,9 @@ func (n asyncPseudonode) MakeProposals(ctx context.Context, r round, p period) ( } func (n asyncPseudonode) StartSpeculativeBlockAssembly(ctx context.Context, r round, p period) (<-chan externalEvent, error) { + return nil, nil + // n.validator. + // go n.factory.OnNewSpeculativeBlock(block, delta) } func (n asyncPseudonode) MakeVotes(ctx context.Context, r round, p period, s step, prop proposalValue, persistStateDone chan error) (chan externalEvent, error) { diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index c60837f946..1adf2635d5 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -140,8 +140,8 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, error) { - specLedger, err := ledger.MakeValidatedBlockAsLFE(block, pool.ledger) +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block bookkeeping.Block) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, error) { + specLedger, err := ledger.MakeBlockAsLFE(block, pool.ledger) if err != nil { return nil, nil, make(chan struct{}), err } @@ -542,8 +542,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt pool.rememberCommit(false) // create shallow pool copy - vb := ledgercore.MakeValidatedBlock(block, delta) - speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(&vb) + speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(block) defer close(specAsmDoneCh) pool.mu.Unlock() diff --git a/ledger/speculative.go b/ledger/speculative.go index 70fe10f4d0..5e41fefbdb 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -17,6 +17,7 @@ package ledger import ( + "context" "fmt" "github.com/algorand/go-algorand/config" @@ -41,6 +42,7 @@ type LedgerForEvaluator interface { LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) VerifiedTransactionCache() verify.VerifiedTransactionCache + LatestTotals() (basics.Round, ledgercore.AccountTotals, error) // Needed for the evaluator GenesisHash() crypto.Digest @@ -84,23 +86,29 @@ type validatedBlockAsLFE struct { vb *ledgercore.ValidatedBlock } -// makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a -// ValidatedBlock. -func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { +// makedBlockAsLFE constructs a new BlockAsLFE from a Block. +func MakeBlockAsLFE(blk bookkeeping.Block, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { latestRound := l.Latest() - if vb.Block().Round().SubSaturate(1) != latestRound { - return nil, fmt.Errorf("MakeValidatedBlockAsLFE: Ledger round %d mismatches next block round %d", latestRound, vb.Block().Round()) + if blk.Round().SubSaturate(1) != latestRound { + return nil, fmt.Errorf("MakeBlockAsLFE: Ledger round %d mismatches next block round %d", latestRound, blk.Round()) } hdr, err := l.BlockHdr(latestRound) if err != nil { return nil, err } - if vb.Block().Branch != hdr.Hash() { - return nil, fmt.Errorf("MakeValidatedBlockAsLFE: Ledger latest block hash %x mismatches block's prev hash %x", hdr.Hash(), vb.Block().Branch) + if blk.Branch != hdr.Hash() { + return nil, fmt.Errorf("MakeBlockAsLFE: Ledger latest block hash %x mismatches block's prev hash %x", hdr.Hash(), blk.Branch) } + + state, err := internal.Eval(context.Background(), l, blk, false, l.VerifiedTransactionCache(), nil) + if err != nil { + return nil, fmt.Errorf("error computing deltas for block %d round %d: %v", blk.Hash(), blk.Round(), err) + } + + vb := ledgercore.MakeValidatedBlock(blk, state) return &validatedBlockAsLFE{ l: l, - vb: vb, + vb: &vb, }, nil } @@ -141,6 +149,11 @@ func (v *validatedBlockAsLFE) Latest() basics.Round { return v.vb.Block().Round() } +// LatestTotals returns the totals of all accounts for the most recent round, as well as the round number. +func (v *validatedBlockAsLFE) LatestTotals() (basics.Round, ledgercore.AccountTotals, error) { + return v.Latest(), v.vb.Delta().Totals, nil +} + // CompactCertVoters implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) CompactCertVoters(r basics.Round) (*ledgercore.VotersForRound, error) { if r >= v.vb.Block().Round() { @@ -171,11 +184,6 @@ func (v *validatedBlockAsLFE) GetCreatorForRound(r basics.Round, cidx basics.Cre return v.l.GetCreatorForRound(r, cidx, ctype) } -// Totals implements the ledgerForEvaluator interface. -func (v *validatedBlockAsLFE) LatestTotals() (basics.Round, ledgercore.AccountTotals, error) { - return v.vb.Block().Round(), v.vb.Delta().Totals, nil -} - // GenesisProto returns the initial protocol for this ledger. func (v *validatedBlockAsLFE) GenesisProto() config.ConsensusParams { return v.l.GenesisProto() diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go index c1f539b6d0..fb96724c5e 100644 --- a/ledger/speculative_test.go +++ b/ledger/speculative_test.go @@ -17,7 +17,6 @@ package ledger import ( - "context" "testing" "github.com/stretchr/testify/require" @@ -26,8 +25,6 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" - "github.com/algorand/go-algorand/ledger/internal" - "github.com/algorand/go-algorand/ledger/ledgercore" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -56,13 +53,9 @@ func TestSpeculative(t *testing.T) { //sl, err := MakeSpeculativeLedger(l) //require.NoError(t, err) - state, err := internal.Eval(context.Background(), l, blk1, false, l.VerifiedTransactionCache(), nil) - require.NoError(t, err) - vblk1 := ledgercore.MakeValidatedBlock(blk1, state) - - blk1aslfe, err := MakeValidatedBlockAsLFE(&vblk1, l) + blk1aslfe, err := MakeBlockAsLFE(blk1, l) require.NoError(t, err) blk2 := blk1 @@ -97,11 +90,7 @@ func TestSpeculative(t *testing.T) { HasGenesisID: true, }) - state, err = internal.Eval(context.Background(), blk1aslfe, blk2, false, l.VerifiedTransactionCache(), nil) - require.NoError(t, err) - - vblk2 := ledgercore.MakeValidatedBlock(blk2, state) - blk2aslfe, err := MakeValidatedBlockAsLFE(&vblk2, blk1aslfe) + blk2aslfe, err := MakeBlockAsLFE(blk2, blk1aslfe) require.NoError(t, err) ad11, rnd, err := blk2aslfe.LookupWithoutRewards(blk1.Round(), addr1) From 36879609a12e6e5aaf06a94123281eb10e98bf87 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 1 Aug 2022 15:39:56 -0400 Subject: [PATCH 23/50] return statedelta --- data/pools/transactionPool.go | 12 ++++++------ ledger/speculative.go | 12 ++++++------ ledger/speculative_test.go | 6 ++++-- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 1adf2635d5..8a6705c0d5 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -140,10 +140,10 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block bookkeeping.Block) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, error) { - specLedger, err := ledger.MakeBlockAsLFE(block, pool.ledger) +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block bookkeeping.Block) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, ledgercore.StateDelta, error) { + specLedger, statedelta, err := ledger.MakeBlockAsLFE(block, pool.ledger) if err != nil { - return nil, nil, make(chan struct{}), err + return nil, nil, make(chan struct{}), ledgercore.StateDelta{}, err } ctx, cancel := context.WithCancel(pool.ctx) @@ -171,7 +171,7 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block bookkeeping pool.specAsmDone = specDoneCh pool.specBlock = make(chan *ledgercore.ValidatedBlock, 1) - return ©, pool.specBlock, specDoneCh, nil + return ©, pool.specBlock, specDoneCh, statedelta, nil } // poolAsmResults is used to syncronize the state of the block assembly process. @@ -529,7 +529,7 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } -func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { +func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block) { pool.mu.Lock() // cancel any pending speculative assembly @@ -542,7 +542,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block, delt pool.rememberCommit(false) // create shallow pool copy - speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(block) + speculativePool, outchan, specAsmDoneCh, delta, err := pool.copyTransactionPoolOverSpecLedger(block) defer close(specAsmDoneCh) pool.mu.Unlock() diff --git a/ledger/speculative.go b/ledger/speculative.go index 5e41fefbdb..d7b0ec40e3 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -87,29 +87,29 @@ type validatedBlockAsLFE struct { } // makedBlockAsLFE constructs a new BlockAsLFE from a Block. -func MakeBlockAsLFE(blk bookkeeping.Block, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { +func MakeBlockAsLFE(blk bookkeeping.Block, l LedgerForEvaluator) (*validatedBlockAsLFE, ledgercore.StateDelta, error) { latestRound := l.Latest() if blk.Round().SubSaturate(1) != latestRound { - return nil, fmt.Errorf("MakeBlockAsLFE: Ledger round %d mismatches next block round %d", latestRound, blk.Round()) + return nil, ledgercore.StateDelta{}, fmt.Errorf("MakeBlockAsLFE: Ledger round %d mismatches next block round %d", latestRound, blk.Round()) } hdr, err := l.BlockHdr(latestRound) if err != nil { - return nil, err + return nil, ledgercore.StateDelta{}, err } if blk.Branch != hdr.Hash() { - return nil, fmt.Errorf("MakeBlockAsLFE: Ledger latest block hash %x mismatches block's prev hash %x", hdr.Hash(), blk.Branch) + return nil, ledgercore.StateDelta{}, fmt.Errorf("MakeBlockAsLFE: Ledger latest block hash %x mismatches block's prev hash %x", hdr.Hash(), blk.Branch) } state, err := internal.Eval(context.Background(), l, blk, false, l.VerifiedTransactionCache(), nil) if err != nil { - return nil, fmt.Errorf("error computing deltas for block %d round %d: %v", blk.Hash(), blk.Round(), err) + return nil, ledgercore.StateDelta{}, fmt.Errorf("error computing deltas for block %d round %d: %v", blk.Hash(), blk.Round(), err) } vb := ledgercore.MakeValidatedBlock(blk, state) return &validatedBlockAsLFE{ l: l, vb: &vb, - }, nil + }, state, nil } // Block implements the ledgerForEvaluator interface. diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go index fb96724c5e..8d4bd5ce00 100644 --- a/ledger/speculative_test.go +++ b/ledger/speculative_test.go @@ -55,8 +55,9 @@ func TestSpeculative(t *testing.T) { require.NoError(t, err) - blk1aslfe, err := MakeBlockAsLFE(blk1, l) + blk1aslfe, statedelta, err := MakeBlockAsLFE(blk1, l) require.NoError(t, err) + require.Equal(t, blk1aslfe.vb.Delta(), statedelta) blk2 := blk1 blk2.BlockHeader.Round++ @@ -90,8 +91,9 @@ func TestSpeculative(t *testing.T) { HasGenesisID: true, }) - blk2aslfe, err := MakeBlockAsLFE(blk2, blk1aslfe) + blk2aslfe, statedelta, err := MakeBlockAsLFE(blk2, blk1aslfe) require.NoError(t, err) + require.Equal(t, blk2aslfe.vb.Delta(), statedelta) ad11, rnd, err := blk2aslfe.LookupWithoutRewards(blk1.Round(), addr1) require.NoError(t, err) From 63c03b907764a583132c55e255d3816d2aad49f8 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 2 Aug 2022 09:56:44 -0400 Subject: [PATCH 24/50] checkpoint: adding readlowest to agreement --- agreement/abstractions.go | 3 +- agreement/actions.go | 17 ++++------ agreement/agreementtest/simulate_test.go | 2 ++ agreement/common_test.go | 4 +++ agreement/demux_test.go | 3 +- agreement/events.go | 42 +++++++++++++++++++++++- agreement/eventtype_string.go | 26 ++++++++------- agreement/fuzzer/ledger_test.go | 2 ++ agreement/player.go | 18 +++++++--- agreement/proposal.go | 2 +- agreement/proposalStore.go | 9 +++++ agreement/proposalTracker.go | 5 +++ agreement/pseudonode.go | 10 +++--- ledger/speculative.go | 28 +++++++++------- 14 files changed, 123 insertions(+), 48 deletions(-) diff --git a/agreement/abstractions.go b/agreement/abstractions.go index 8b1b3c875a..c36c4903e7 100644 --- a/agreement/abstractions.go +++ b/agreement/abstractions.go @@ -26,7 +26,6 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" - "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/protocol" ) @@ -87,7 +86,7 @@ type BlockFactory interface { // lose liveness. AssembleBlock(basics.Round) (ValidatedBlock, error) - OnNewSpeculativeBlock(block bookkeeping.Block, delta ledgercore.StateDelta) + OnNewSpeculativeBlock(block bookkeeping.Block) } // A Ledger represents the sequence of Entries agreed upon by the protocol. diff --git a/agreement/actions.go b/agreement/actions.go index cd6170815c..6b33234ce3 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -312,10 +312,11 @@ type pseudonodeAction struct { // assemble, repropose, attest, speculativeAssembly T actionType - Round round - Period period - Step step - Proposal proposalValue + Round round + Period period + Step step + Proposal proposalValue + ValidatedBlock ValidatedBlock } func (a pseudonodeAction) t() actionType { @@ -347,13 +348,7 @@ func (a pseudonodeAction) do(ctx context.Context, s *Service) { s.log.Errorf("pseudonode.MakeProposals call failed %v", err) } case speculativeAssembly: - events, err := s.loopback.StartSpeculativeBlockAssembly(ctx, a.Round, a.Period) - switch err { - case nil: - s.demux.prioritize(events) - default: - s.log.Errorf("pseudonode.StartSpeculativeBlockAssembly call failed %v", err) - } + s.loopback.StartSpeculativeBlockAssembly(ctx, a.ValidatedBlock) case repropose: logEvent := logspec.AgreementEvent{ diff --git a/agreement/agreementtest/simulate_test.go b/agreement/agreementtest/simulate_test.go index 173f91fad8..d742d89a0e 100644 --- a/agreement/agreementtest/simulate_test.go +++ b/agreement/agreementtest/simulate_test.go @@ -96,6 +96,8 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (agreement.ValidatedBloc return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } +func (f testBlockFactory) OnNewSpeculativeBlock(blk bookkeeping.Block) {} + // If we try to read from high rounds, we panic and do not emit an error to find bugs during testing. type testLedger struct { mu deadlock.Mutex diff --git a/agreement/common_test.go b/agreement/common_test.go index fa43d10912..4998d386a7 100644 --- a/agreement/common_test.go +++ b/agreement/common_test.go @@ -183,6 +183,10 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (ValidatedBlock, error) return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } +func (f testBlockFactory) OnNewSpeculativeBlock(block bookkeeping.Block) { + return +} + // If we try to read from high rounds, we panic and do not emit an error to find bugs during testing. type testLedger struct { mu deadlock.Mutex diff --git a/agreement/demux_test.go b/agreement/demux_test.go index 7d7bbc71bd..6cfdfe114d 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -37,6 +37,7 @@ import ( ) const fastTimeoutChTime = 2 +const speculativeAsmTime = 1 type demuxTester struct { *testing.T @@ -675,7 +676,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { close(s.quit) } - e, ok := dmx.next(s, time.Second, fastTimeoutChTime, 300) + e, ok := dmx.next(s, time.Second, fastTimeoutChTime, speculativeAsmTime, 300) if !assert.Equal(t, testcase.ok, ok) { return false diff --git a/agreement/events.go b/agreement/events.go index 3591687516..7e20447cfc 100644 --- a/agreement/events.go +++ b/agreement/events.go @@ -199,6 +199,14 @@ const ( // readPinned is sent to the proposalStore to read the pinned value, if it exists. readPinned + // readLowestValue is sent to the proposalPeriodMachine to read the + // proposal-vote with the lowest credential. + readLowestValue + + // readLowestPayload is sent to the proposalStore to read the payload + // corresponding to the lowest-credential proposal-vote, if it exists. + readLowestPayload + /* * The following are event types that replace queries, and may warrant * a revision to make them more state-machine-esque. @@ -405,6 +413,38 @@ func (e newRoundEvent) ComparableStr() string { return e.String() } +type readLowestEvent struct { + // T is either readLowestValue or readLowestPayload + T eventType + + // Round and Period are the round and period for which to query + // the lowest-credential value and payload. This type of event + // is only sent for pipelining, which only makes sense for period + // 0, but the Period is here anyway to route to the appropriate + // proposalMachinePeriod. + Round round + Period period + + // Proposal holds the lowest-credential value. + Proposal proposalValue + // Payload holds the payload, if one exists (which is the case if PayloadOK is set). + Payload proposal + // PayloadOK is set if and only if a payload was received for the lowest-credential value. + PayloadOK bool +} + +func (e readLowestEvent) t() eventType { + return e.T +} + +func (e readLowestEvent) String() string { + return fmt.Sprintf("%v: %.5v", e.t().String(), e.Proposal.BlockDigest.String()) +} + +func (e readLowestEvent) ComparableStr() string { + return e.String() +} + type newPeriodEvent struct { // Period holds the latest period relevant to the proposalRoundMachine. Period period @@ -741,7 +781,7 @@ func zeroEvent(t eventType) event { return messageEvent{} case roundInterruption: return roundInterruptionEvent{} - case timeout, fastTimeout: + case timeout, fastTimeout, speculationTimeout: return timeoutEvent{} case newRound: return newRoundEvent{} diff --git a/agreement/eventtype_string.go b/agreement/eventtype_string.go index 6944ce5862..ba8cd99960 100644 --- a/agreement/eventtype_string.go +++ b/agreement/eventtype_string.go @@ -38,21 +38,23 @@ func _() { _ = x[newPeriod-27] _ = x[readStaging-28] _ = x[readPinned-29] - _ = x[voteFilterRequest-30] - _ = x[voteFilteredStep-31] - _ = x[nextThresholdStatusRequest-32] - _ = x[nextThresholdStatus-33] - _ = x[freshestBundleRequest-34] - _ = x[freshestBundle-35] - _ = x[dumpVotesRequest-36] - _ = x[dumpVotes-37] - _ = x[wrappedAction-38] - _ = x[checkpointReached-39] + _ = x[readLowestValue-30] + _ = x[readLowestPayload-31] + _ = x[voteFilterRequest-32] + _ = x[voteFilteredStep-33] + _ = x[nextThresholdStatusRequest-34] + _ = x[nextThresholdStatus-35] + _ = x[freshestBundleRequest-36] + _ = x[freshestBundle-37] + _ = x[dumpVotesRequest-38] + _ = x[dumpVotes-39] + _ = x[wrappedAction-40] + _ = x[checkpointReached-41] } -const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutspeculationTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" +const _eventType_name = "nonevotePresentpayloadPresentbundlePresentvoteVerifiedpayloadVerifiedbundleVerifiedroundInterruptiontimeoutfastTimeoutspeculationTimeoutsoftThresholdcertThresholdnextThresholdproposalCommittableproposalAcceptedvoteFilteredvoteMalformedbundleFilteredbundleMalformedpayloadRejectedpayloadMalformedpayloadPipelinedpayloadAcceptedproposalFrozenvoteAcceptednewRoundnewPeriodreadStagingreadPinnedreadLowestValuereadLowestPayloadvoteFilterRequestvoteFilteredStepnextThresholdStatusRequestnextThresholdStatusfreshestBundleRequestfreshestBundledumpVotesRequestdumpVoteswrappedActioncheckpointReached" -var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 136, 149, 162, 175, 194, 210, 222, 235, 249, 264, 279, 295, 311, 326, 340, 352, 360, 369, 380, 390, 407, 423, 449, 468, 489, 503, 519, 528, 541, 558} +var _eventType_index = [...]uint16{0, 4, 15, 29, 42, 54, 69, 83, 100, 107, 118, 136, 149, 162, 175, 194, 210, 222, 235, 249, 264, 279, 295, 311, 326, 340, 352, 360, 369, 380, 390, 405, 422, 439, 455, 481, 500, 521, 535, 551, 560, 573, 590} func (i eventType) String() string { if i < 0 || i >= eventType(len(_eventType_index)-1) { diff --git a/agreement/fuzzer/ledger_test.go b/agreement/fuzzer/ledger_test.go index 2ae73dd694..ab3bb4c5b4 100644 --- a/agreement/fuzzer/ledger_test.go +++ b/agreement/fuzzer/ledger_test.go @@ -112,6 +112,8 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (agreement.ValidatedBloc return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } +func (f testBlockFactory) OnNewSpeculativeBlock(bookkeeping.Block) {} + type testLedgerSyncFunc func(l *testLedger, r basics.Round, c agreement.Certificate) bool // If we try to read from high rounds, we panic and do not emit an error to find bugs during testing. diff --git a/agreement/player.go b/agreement/player.go index ef4e127733..cee7ff0a15 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -126,12 +126,21 @@ func (p *player) handle(r routerHandle, e event) []action { } func (p *player) handleSpeculationTimeout(r routerHandle, e timeoutEvent) []action { - // start speculative block assembly if e.Proto.Err != nil { r.t.log.Errorf("failed to read protocol version for speculationTimeout event (proto %v): %v", e.Proto.Version, e.Proto.Err) return nil } - return p.startSpeculativeBlockAsm(r) + + // get the best proposal we have + re := readLowestEvent{T: readLowestPayload, Round: p.Round} + re = r.dispatch(*p, re, proposalMachineRound, p.Round, 0, 0).(readLowestEvent) + + // if we have its payload and its been validated already, start speculating + // on top of it + if re.PayloadOK && re.Payload.ve != nil { + return p.startSpeculativeBlockAsm(r, re.Payload.ve) + } + return nil } func (p *player) handleFastTimeout(r routerHandle, e timeoutEvent) []action { @@ -228,8 +237,9 @@ func (p *player) issueNextVote(r routerHandle) []action { return actions } -func (p *player) startSpeculativeBlockAsm(r routerHandle) (actions []action) { - return append(actions, pseudonodeAction{T: speculativeAssembly, Round: p.Round, Period: p.Period}) +func (p *player) startSpeculativeBlockAsm(r routerHandle, ve ValidatedBlock) (actions []action) { + // TODO(yossi) extend the action to carry the speculative block payload + return append(actions, pseudonodeAction{T: speculativeAssembly, Round: p.Round, Period: p.Period, ValidatedBlock: ve}) } func (p *player) issueFastVote(r routerHandle) (actions []action) { diff --git a/agreement/proposal.go b/agreement/proposal.go index 232fa6f5f4..75918c11a9 100644 --- a/agreement/proposal.go +++ b/agreement/proposal.go @@ -79,7 +79,7 @@ func (p unauthenticatedProposal) value() proposalValue { } } -// A proposal is an Block along with everything needed to validate it. +// A proposal is a Block along with everything needed to validate it. type proposal struct { unauthenticatedProposal diff --git a/agreement/proposalStore.go b/agreement/proposalStore.go index 973f909d03..ccfadc33c8 100644 --- a/agreement/proposalStore.go +++ b/agreement/proposalStore.go @@ -337,6 +337,15 @@ func (store *proposalStore) handle(r routerHandle, p player, e event) event { se.Committable = ea.Assembled se.Payload = ea.Payload return se + case readLowestPayload: + re := e.(readLowestEvent) + re.T = readLowestValue + re = r.dispatch(p, re, proposalMachinePeriod, re.Round, re.Period, 0).(readLowestEvent) + re.T = readLowestPayload + ea := store.Assemblers[re.Proposal] + re.PayloadOK = ea.Assembled + re.Payload = ea.Payload + return re case readPinned: se := e.(pinnedValueEvent) ea := store.Assemblers[store.Pinned] // If pinned is bottom, assembled/payloadOK = false, payload = bottom diff --git a/agreement/proposalTracker.go b/agreement/proposalTracker.go index c76c5c9fda..04a5dd5fca 100644 --- a/agreement/proposalTracker.go +++ b/agreement/proposalTracker.go @@ -163,6 +163,11 @@ func (t *proposalTracker) handle(r routerHandle, p player, e event) event { t.Freezer = t.Freezer.freeze() return e + case readLowestValue: + e := e.(readLowestEvent) + e.Proposal = t.Freezer.Lowest.R.Proposal + return e + case softThreshold, certThreshold: e := e.(thresholdEvent) t.Staging = e.Proposal diff --git a/agreement/pseudonode.go b/agreement/pseudonode.go index 3862d859db..deca7fd062 100644 --- a/agreement/pseudonode.go +++ b/agreement/pseudonode.go @@ -60,7 +60,8 @@ type pseudonode interface { // It returns an error if the pseudonode is unable to perform this. MakeProposals(ctx context.Context, r round, p period) (<-chan externalEvent, error) - StartSpeculativeBlockAssembly(ctx context.Context, r round, p period) (<-chan externalEvent, error) + // TODO(yossi) pass in context all the way to the pool, so we can cancel spec asm if needed + StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock) // MakeVotes returns a vote for a given proposal in some round, period, and step. // @@ -187,10 +188,9 @@ func (n asyncPseudonode) MakeProposals(ctx context.Context, r round, p period) ( } } -func (n asyncPseudonode) StartSpeculativeBlockAssembly(ctx context.Context, r round, p period) (<-chan externalEvent, error) { - return nil, nil - // n.validator. - // go n.factory.OnNewSpeculativeBlock(block, delta) +func (n asyncPseudonode) StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock) { + // TODO(yossi) change to use ve directly, to avoid recomputing statedeltas + go n.factory.OnNewSpeculativeBlock(ve.Block()) } func (n asyncPseudonode) MakeVotes(ctx context.Context, r round, p period, s step, prop proposalValue, persistStateDone chan error) (chan externalEvent, error) { diff --git a/ledger/speculative.go b/ledger/speculative.go index d7b0ec40e3..c15d4671ab 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -86,30 +86,36 @@ type validatedBlockAsLFE struct { vb *ledgercore.ValidatedBlock } -// makedBlockAsLFE constructs a new BlockAsLFE from a Block. -func MakeBlockAsLFE(blk bookkeeping.Block, l LedgerForEvaluator) (*validatedBlockAsLFE, ledgercore.StateDelta, error) { +// makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a ValidatedBlock. +func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { latestRound := l.Latest() - if blk.Round().SubSaturate(1) != latestRound { - return nil, ledgercore.StateDelta{}, fmt.Errorf("MakeBlockAsLFE: Ledger round %d mismatches next block round %d", latestRound, blk.Round()) + if vb.Block().Round().SubSaturate(1) != latestRound { + return nil, fmt.Errorf("MakeBlockAsLFE: Ledger round %d mismatches next block round %d", latestRound, vb.Block().Round()) } hdr, err := l.BlockHdr(latestRound) if err != nil { - return nil, ledgercore.StateDelta{}, err + return nil, err } - if blk.Branch != hdr.Hash() { - return nil, ledgercore.StateDelta{}, fmt.Errorf("MakeBlockAsLFE: Ledger latest block hash %x mismatches block's prev hash %x", hdr.Hash(), blk.Branch) + if vb.Block().Branch != hdr.Hash() { + return nil, fmt.Errorf("MakeBlockAsLFE: Ledger latest block hash %x mismatches block's prev hash %x", hdr.Hash(), vb.Block().Branch) } + return &validatedBlockAsLFE{ + l: l, + vb: vb, + }, nil +} + +// makeBlockAsLFE constructs a new validatedBlockAsLFE from a Block. +func MakeBlockAsLFE(blk bookkeeping.Block, l LedgerForEvaluator) (*validatedBlockAsLFE, ledgercore.StateDelta, error) { state, err := internal.Eval(context.Background(), l, blk, false, l.VerifiedTransactionCache(), nil) if err != nil { return nil, ledgercore.StateDelta{}, fmt.Errorf("error computing deltas for block %d round %d: %v", blk.Hash(), blk.Round(), err) } vb := ledgercore.MakeValidatedBlock(blk, state) - return &validatedBlockAsLFE{ - l: l, - vb: &vb, - }, state, nil + lfe, err := MakeValidatedBlockAsLFE(&vb, l) + return lfe, vb.Delta(), nil } // Block implements the ledgerForEvaluator interface. From 633bf49b37bb19a03e5f9109ea9a54c71a95783b Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 2 Aug 2022 10:28:32 -0400 Subject: [PATCH 25/50] treat 0 timeout as no timeout --- agreement/demux.go | 5 +++++ agreement/demux_test.go | 2 +- agreement/player.go | 7 ++++--- agreement/proposalTrackerContract.go | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/agreement/demux.go b/agreement/demux.go index bc5fabfdb4..e7eecb758a 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -249,7 +249,12 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat ledgerNextRoundCh := s.Ledger.Wait(nextRound) deadlineCh := s.Clock.TimeoutAt(deadline) fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline) + speculationDeadlineCh := s.Clock.TimeoutAt(speculationDeadline) + // zero timeout means we don't have enough time to speculate on block assembly + if speculationDeadline == 0 { + speculationDeadlineCh = nil + } d.UpdateEventsQueue(eventQueueDemux, 0) d.monitor.dec(demuxCoserviceType) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index 6cfdfe114d..984f6719a1 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -37,7 +37,7 @@ import ( ) const fastTimeoutChTime = 2 -const speculativeAsmTime = 1 +const speculativeAsmTime = fastTimeoutChTime + 1 type demuxTester struct { *testing.T diff --git a/agreement/player.go b/agreement/player.go index cee7ff0a15..5dfb6d9512 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -79,14 +79,15 @@ func (p *player) handle(r routerHandle, e event) []action { if e.T == fastTimeout { return p.handleFastTimeout(r, e) } - if e.T == speculationTimeout { - return p.handleSpeculationTimeout(r, e) - } if !p.Napping { r.t.logTimeout(*p) } + if e.T == speculationTimeout { + return p.handleSpeculationTimeout(r, e) + } + switch p.Step { case soft: // precondition: nap = false diff --git a/agreement/proposalTrackerContract.go b/agreement/proposalTrackerContract.go index c33feb8413..0b32499242 100644 --- a/agreement/proposalTrackerContract.go +++ b/agreement/proposalTrackerContract.go @@ -30,7 +30,7 @@ type proposalTrackerContract struct { // TODO check concrete types of events func (c *proposalTrackerContract) pre(p player, in event) (pre []error) { switch in.t() { - case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging: + case voteVerified, proposalFrozen, softThreshold, certThreshold, voteFilterRequest, readStaging, readLowestValue, readLowestPayload: default: pre = append(pre, fmt.Errorf("incoming event has invalid type: %v", in.t())) } From 57edac81a172baff2737b8867dbd5bfa8fd7e59c Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 2 Aug 2022 11:13:58 -0400 Subject: [PATCH 26/50] x --- agreement/player.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agreement/player.go b/agreement/player.go index 5dfb6d9512..741964a8f3 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -239,7 +239,6 @@ func (p *player) issueNextVote(r routerHandle) []action { } func (p *player) startSpeculativeBlockAsm(r routerHandle, ve ValidatedBlock) (actions []action) { - // TODO(yossi) extend the action to carry the speculative block payload return append(actions, pseudonodeAction{T: speculativeAssembly, Round: p.Round, Period: p.Period, ValidatedBlock: ve}) } From 7a0fdf71dd5f6127184de1b484d0ec65ba462423 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 2 Aug 2022 16:04:45 -0400 Subject: [PATCH 27/50] zero spec timeout --- agreement/demux_test.go | 3 +-- agreement/player.go | 13 +++++++++++-- agreement/service.go | 13 ++++++------- agreement/types.go | 3 +++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index 984f6719a1..92d1a713e7 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -37,7 +37,6 @@ import ( ) const fastTimeoutChTime = 2 -const speculativeAsmTime = fastTimeoutChTime + 1 type demuxTester struct { *testing.T @@ -676,7 +675,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { close(s.quit) } - e, ok := dmx.next(s, time.Second, fastTimeoutChTime, speculativeAsmTime, 300) + e, ok := dmx.next(s, time.Second, fastTimeoutChTime, 0, 300) if !assert.Equal(t, testcase.ok, ok) { return false diff --git a/agreement/player.go b/agreement/player.go index 741964a8f3..9958ab7d24 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -48,9 +48,15 @@ type player struct { // partition recovery. FastRecoveryDeadline time.Duration + // SpeculativeAssemblyDeadline contains the next timeout expected for + // speculative block assembly. + SpeculativeAssemblyDeadline time.Duration + // Pending holds the player's proposalTable, which stores proposals that // must be verified after some vote has been verified. Pending proposalTable + + ConcensusVersion protocol.ConsensusVersion } func (p *player) T() stateMachineTag { @@ -127,14 +133,15 @@ func (p *player) handle(r routerHandle, e event) []action { } func (p *player) handleSpeculationTimeout(r routerHandle, e timeoutEvent) []action { + p.SpeculativeAssemblyDeadline = 0 if e.Proto.Err != nil { r.t.log.Errorf("failed to read protocol version for speculationTimeout event (proto %v): %v", e.Proto.Version, e.Proto.Err) return nil } // get the best proposal we have - re := readLowestEvent{T: readLowestPayload, Round: p.Round} - re = r.dispatch(*p, re, proposalMachineRound, p.Round, 0, 0).(readLowestEvent) + re := readLowestEvent{T: readLowestPayload, Round: p.Round, Period: p.Period} + re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) // if we have its payload and its been validated already, start speculating // on top of it @@ -353,6 +360,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Napping = false p.FastRecoveryDeadline = 0 // set immediately p.Deadline = FilterTimeout(target, source.Proto) + p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(target, p.ConcensusVersion) // update tracer state to match player r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step}) @@ -397,6 +405,7 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately + p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(0, p.ConcensusVersion) switch source := source.(type) { case roundInterruptionEvent: diff --git a/agreement/service.go b/agreement/service.go index e293048241..ea3a11c9e0 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -221,15 +221,14 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.Clock = clock } - nextVersion, err := s.Ledger.ConsensusVersion(status.Round) - speculativeBlockAsmTime := time.Duration(0) - if err == nil { - speculativeBlockAsmTime = SpeculativeBlockAsmTime(0, nextVersion) - } - for { + status.ConcensusVersion, err = s.Ledger.ConsensusVersion(status.Round) + if err != nil { + s.Panicf("cannot read latest concensus version, round %d: %v", status.Round, err) + } + output <- a - ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, SpeculativeBlockAsmDeadline: speculativeBlockAsmTime, CurrentRound: status.Round} + ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, SpeculativeBlockAsmDeadline: status.SpeculativeAssemblyDeadline, CurrentRound: status.Round} e, ok := <-input if !ok { break diff --git a/agreement/types.go b/agreement/types.go index 0d86b865dc..d49b9faa67 100644 --- a/agreement/types.go +++ b/agreement/types.go @@ -39,6 +39,9 @@ func FilterTimeout(p period, v protocol.ConsensusVersion) time.Duration { } func SpeculativeBlockAsmTime(p period, v protocol.ConsensusVersion) time.Duration { + if p != 0 { + return time.Duration(0) + } hardwait := FilterTimeout(p, v) // TODO(yossi) change from default config to actual config if hardwait > config.GetDefaultLocal().ProposalAssemblyTime+time.Duration(100*time.Millisecond) { From 2ca5107e8618e4d60d312a42c6cb7f77032102f3 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 2 Aug 2022 16:56:00 -0400 Subject: [PATCH 28/50] txpool interface --- data/datatest/impls.go | 2 ++ node/node.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/data/datatest/impls.go b/data/datatest/impls.go index 7a8f804eaf..344c30cc1c 100644 --- a/data/datatest/impls.go +++ b/data/datatest/impls.go @@ -64,6 +64,8 @@ func (i entryFactoryImpl) AssembleBlock(round basics.Round) (agreement.Validated return validatedBlock{blk: &b}, nil } +func (i entryFactoryImpl) OnNewSpeculativeBlock(bookkeeping.Block) {} + // WithSeed implements the agreement.ValidatedBlock interface. func (ve validatedBlock) WithSeed(s committee.Seed) agreement.ValidatedBlock { newblock := ve.blk.WithSeed(s) diff --git a/node/node.go b/node/node.go index 5d9c50beed..2f972cc48a 100644 --- a/node/node.go +++ b/node/node.go @@ -1265,6 +1265,10 @@ func (node *AlgorandFullNode) AssembleBlock(round basics.Round) (agreement.Valid return validatedBlock{vb: lvb}, nil } +func (node *AlgorandFullNode) OnNewSpeculativeBlock(blk bookkeeping.Block) { + node.transactionPool.OnNewSpeculativeBlock(blk) +} + // getOfflineClosedStatus will return an int with the appropriate bit(s) set if it is offline and/or online func getOfflineClosedStatus(acctData basics.OnlineAccountData) int { rval := 0 From 28a19a9599b84e98e8c807170ed6a4a946fac41e Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 4 Aug 2022 09:08:34 -0400 Subject: [PATCH 29/50] demux event test --- agreement/demux_test.go | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/agreement/demux_test.go b/agreement/demux_test.go index 92d1a713e7..a37f622028 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -37,6 +37,7 @@ import ( ) const fastTimeoutChTime = 2 +const speculativeBlockAsmTime = time.Duration(5 * time.Microsecond) type demuxTester struct { *testing.T @@ -65,8 +66,9 @@ type demuxTestUsecase struct { verifiedProposal testChanState verifiedBundle testChanState // expected output - e event - ok bool + e event + ok bool + speculativeAsmTime time.Duration } var demuxTestUsecases = []demuxTestUsecase{ @@ -412,6 +414,25 @@ var demuxTestUsecases = []demuxTestUsecase{ e: messageEvent{T: votePresent}, ok: true, }, + { + queue: []testChanState{{eventCount: 0, closed: false}}, + rawVotes: testChanState{eventCount: 0, closed: false}, + rawProposals: testChanState{eventCount: 0, closed: false}, + rawBundles: testChanState{eventCount: 0, closed: false}, + compoundProposals: false, + quit: false, + voteChannelFull: false, + proposalChannelFull: false, + bundleChannelFull: false, + ledgerRoundReached: false, + deadlineReached: false, + verifiedVote: testChanState{eventCount: 0, closed: false}, + verifiedProposal: testChanState{eventCount: 0, closed: false}, + verifiedBundle: testChanState{eventCount: 0, closed: false}, + e: timeoutEvent{T: speculationTimeout}, + ok: true, + speculativeAsmTime: speculativeBlockAsmTime, + }, } func TestDemuxNext(t *testing.T) { @@ -432,6 +453,9 @@ func (t *demuxTester) TimeoutAt(delta time.Duration) <-chan time.Time { if delta == fastTimeoutChTime { return nil } + if delta == speculativeBlockAsmTime { + return time.After(delta) + } c := make(chan time.Time, 2) if t.currentUsecase.deadlineReached { @@ -674,8 +698,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { if testcase.quit { close(s.quit) } - - e, ok := dmx.next(s, time.Second, fastTimeoutChTime, 0, 300) + e, ok := dmx.next(s, time.Second, fastTimeoutChTime, testcase.speculativeAsmTime, 300) if !assert.Equal(t, testcase.ok, ok) { return false From 3d8413545a266169655a56ae85e352cd561c0971 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 4 Aug 2022 14:56:05 -0400 Subject: [PATCH 30/50] fix todos --- agreement/abstractions.go | 2 +- agreement/agreementtest/simulate_test.go | 2 +- agreement/common_test.go | 2 +- agreement/fuzzer/ledger_test.go | 2 +- agreement/player.go | 13 ++- agreement/pseudonode.go | 5 +- agreement/service.go | 2 + agreement/types.go | 11 +-- config/localTemplate.go | 7 +- config/local_defaults.go | 3 +- data/datatest/impls.go | 2 +- data/pools/transactionPool.go | 45 ++++++---- installer/config.json.example | 3 +- ledger/speculative.go | 31 +------ node/node.go | 5 +- test/testdata/configs/config-v24.json | 107 +++++++++++++++++++++++ 16 files changed, 173 insertions(+), 69 deletions(-) create mode 100644 test/testdata/configs/config-v24.json diff --git a/agreement/abstractions.go b/agreement/abstractions.go index c36c4903e7..e3e3a9870b 100644 --- a/agreement/abstractions.go +++ b/agreement/abstractions.go @@ -86,7 +86,7 @@ type BlockFactory interface { // lose liveness. AssembleBlock(basics.Round) (ValidatedBlock, error) - OnNewSpeculativeBlock(block bookkeeping.Block) + OnNewSpeculativeBlock(context.Context, ValidatedBlock) } // A Ledger represents the sequence of Entries agreed upon by the protocol. diff --git a/agreement/agreementtest/simulate_test.go b/agreement/agreementtest/simulate_test.go index d742d89a0e..a50e052a42 100644 --- a/agreement/agreementtest/simulate_test.go +++ b/agreement/agreementtest/simulate_test.go @@ -96,7 +96,7 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (agreement.ValidatedBloc return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } -func (f testBlockFactory) OnNewSpeculativeBlock(blk bookkeeping.Block) {} +func (f testBlockFactory) OnNewSpeculativeBlock(context.Context, agreement.ValidatedBlock) {} // If we try to read from high rounds, we panic and do not emit an error to find bugs during testing. type testLedger struct { diff --git a/agreement/common_test.go b/agreement/common_test.go index 4998d386a7..2cb6ea3227 100644 --- a/agreement/common_test.go +++ b/agreement/common_test.go @@ -183,7 +183,7 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (ValidatedBlock, error) return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } -func (f testBlockFactory) OnNewSpeculativeBlock(block bookkeeping.Block) { +func (f testBlockFactory) OnNewSpeculativeBlock(context.Context, ValidatedBlock) { return } diff --git a/agreement/fuzzer/ledger_test.go b/agreement/fuzzer/ledger_test.go index ab3bb4c5b4..d2b2059456 100644 --- a/agreement/fuzzer/ledger_test.go +++ b/agreement/fuzzer/ledger_test.go @@ -112,7 +112,7 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (agreement.ValidatedBloc return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } -func (f testBlockFactory) OnNewSpeculativeBlock(bookkeeping.Block) {} +func (f testBlockFactory) OnNewSpeculativeBlock(context.Context, agreement.ValidatedBlock) {} type testLedgerSyncFunc func(l *testLedger, r basics.Round, c agreement.Certificate) bool diff --git a/agreement/player.go b/agreement/player.go index 9958ab7d24..eecbd6f63a 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -56,7 +56,11 @@ type player struct { // must be verified after some vote has been verified. Pending proposalTable + // the current concensus version ConcensusVersion protocol.ConsensusVersion + + // the time offset before the filter deadline to start speculative block assembly + SpeculativeAsmTimeDuration time.Duration } func (p *player) T() stateMachineTag { @@ -360,7 +364,12 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Napping = false p.FastRecoveryDeadline = 0 // set immediately p.Deadline = FilterTimeout(target, source.Proto) - p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(target, p.ConcensusVersion) + if target == 0 { + p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(target, p.ConcensusVersion, p.SpeculativeAsmTimeDuration) + } else { + // only speculate on block assembly in period 0 + p.SpeculativeAssemblyDeadline = 0 + } // update tracer state to match player r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step}) @@ -405,7 +414,7 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately - p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(0, p.ConcensusVersion) + p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(0, p.ConcensusVersion, p.SpeculativeAsmTimeDuration) switch source := source.(type) { case roundInterruptionEvent: diff --git a/agreement/pseudonode.go b/agreement/pseudonode.go index deca7fd062..1d6aa53f1b 100644 --- a/agreement/pseudonode.go +++ b/agreement/pseudonode.go @@ -60,7 +60,7 @@ type pseudonode interface { // It returns an error if the pseudonode is unable to perform this. MakeProposals(ctx context.Context, r round, p period) (<-chan externalEvent, error) - // TODO(yossi) pass in context all the way to the pool, so we can cancel spec asm if needed + // StartSpeculativeBlockAssembly calls the BlockFactory's StartSpeculativeBlockAssembly on a new go routine. StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock) // MakeVotes returns a vote for a given proposal in some round, period, and step. @@ -189,8 +189,7 @@ func (n asyncPseudonode) MakeProposals(ctx context.Context, r round, p period) ( } func (n asyncPseudonode) StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock) { - // TODO(yossi) change to use ve directly, to avoid recomputing statedeltas - go n.factory.OnNewSpeculativeBlock(ve.Block()) + go n.factory.OnNewSpeculativeBlock(ctx, ve) } func (n asyncPseudonode) MakeVotes(ctx context.Context, r round, p period, s step, prop proposalValue, persistStateDone chan error) (chan externalEvent, error) { diff --git a/agreement/service.go b/agreement/service.go index ea3a11c9e0..7f23374d88 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -221,6 +221,8 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.Clock = clock } + // set speculative block assembly based on the current local configuration + status.SpeculativeAsmTimeDuration = s.parameters.Local.ProposalAssemblyTime + s.parameters.Local.SpeculativeBlockAssemblyGraceTime for { status.ConcensusVersion, err = s.Ledger.ConsensusVersion(status.Round) if err != nil { diff --git a/agreement/types.go b/agreement/types.go index d49b9faa67..2d11ded305 100644 --- a/agreement/types.go +++ b/agreement/types.go @@ -38,15 +38,10 @@ func FilterTimeout(p period, v protocol.ConsensusVersion) time.Duration { return config.Consensus[v].AgreementFilterTimeout } -func SpeculativeBlockAsmTime(p period, v protocol.ConsensusVersion) time.Duration { - if p != 0 { - return time.Duration(0) - } +func SpeculativeBlockAsmTime(p period, v protocol.ConsensusVersion, speculativeAsmTimeDuration time.Duration) time.Duration { hardwait := FilterTimeout(p, v) - // TODO(yossi) change from default config to actual config - if hardwait > config.GetDefaultLocal().ProposalAssemblyTime+time.Duration(100*time.Millisecond) { - // TODO(yossi) consider another check that makes sure we waited at least for a bit, so we don't start spec asm for nothing - return hardwait - (config.GetDefaultLocal().ProposalAssemblyTime + time.Duration(100*time.Millisecond)) + if hardwait > speculativeAsmTimeDuration { + return hardwait - speculativeAsmTimeDuration } return time.Duration(0) } diff --git a/config/localTemplate.go b/config/localTemplate.go index 8a8120c5f9..10de4cff9b 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -41,7 +41,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24"` // environmental (may be overridden) // When enabled, stores blocks indefinitely, otherwise, only the most recent blocks @@ -452,6 +452,11 @@ type Local struct { // MaxAcctLookback sets the maximum lookback range for account states, // i.e. the ledger can answer account states questions for the range Latest-MaxAcctLookback...Latest MaxAcctLookback uint64 `version[23]:"4"` + + // SpeculativeBlockAssemblyGraceTime sets additional time, on top of + // ProposalAssemblyTime, that this node allows for speculative block + // assembly. + SpeculativeBlockAssemblyGraceTime time.Duration `version[24]:"50000000"` } // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers diff --git a/config/local_defaults.go b/config/local_defaults.go index 4b6017050d..89426e73f8 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,7 +20,7 @@ package config var defaultLocal = Local{ - Version: 23, + Version: 24, AccountUpdatesStatsInterval: 5000000000, AccountsRebuildSynchronousMode: 1, AgreementIncomingBundlesQueueLength: 7, @@ -110,6 +110,7 @@ var defaultLocal = Local{ RestReadTimeoutSeconds: 15, RestWriteTimeoutSeconds: 120, RunHosted: false, + SpeculativeBlockAssemblyGraceTime: 50000000, SuggestedFeeBlockHistory: 3, SuggestedFeeSlidingWindowSize: 50, TLSCertFile: "", diff --git a/data/datatest/impls.go b/data/datatest/impls.go index 344c30cc1c..b043cf628d 100644 --- a/data/datatest/impls.go +++ b/data/datatest/impls.go @@ -64,7 +64,7 @@ func (i entryFactoryImpl) AssembleBlock(round basics.Round) (agreement.Validated return validatedBlock{blk: &b}, nil } -func (i entryFactoryImpl) OnNewSpeculativeBlock(bookkeeping.Block) {} +func (i entryFactoryImpl) OnNewSpeculativeBlock(context.Context, agreement.ValidatedBlock) {} // WithSeed implements the agreement.ValidatedBlock interface. func (ve validatedBlock) WithSeed(s committee.Seed) agreement.ValidatedBlock { diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 8a6705c0d5..6fc0e7860c 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -95,9 +95,14 @@ type TransactionPool struct { // proposalAssemblyTime is the ProposalAssemblyTime configured for this node. proposalAssemblyTime time.Duration - ctx context.Context - cancelSpec context.CancelFunc - specBlock chan *ledgercore.ValidatedBlock + ctx context.Context + cancelSpec context.CancelFunc + + // specBlockCh has an assembled speculative block + specBlockCh chan *ledgercore.ValidatedBlock + // specBlockMu protects setting/getting specBlockCh only + specBlockMu deadlock.Mutex + // specAsmDone channel is closed when there is no speculative assembly specAsmDone <-chan struct{} cfg config.Local @@ -119,6 +124,7 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo if cfg.TxPoolExponentialIncreaseFactor < 1 { cfg.TxPoolExponentialIncreaseFactor = 1 } + pool := TransactionPool{ pendingTxids: make(map[transactions.Txid]transactions.SignedTxn), rememberedTxids: make(map[transactions.Txid]transactions.SignedTxn), @@ -140,13 +146,13 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block bookkeeping.Block) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, ledgercore.StateDelta, error) { - specLedger, statedelta, err := ledger.MakeBlockAsLFE(block, pool.ledger) +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Context, vb *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, error) { + specLedger, err := ledger.MakeValidatedBlockAsLFE(vb, pool.ledger) if err != nil { - return nil, nil, make(chan struct{}), ledgercore.StateDelta{}, err + return nil, nil, make(chan struct{}), err } - ctx, cancel := context.WithCancel(pool.ctx) + copyPoolctx, cancel := context.WithCancel(ctx) copy := TransactionPool{ pendingTxids: pool.pendingTxids, // it is safe to shallow copy pendingTxids since this map is only (atomically) swapped and never directly changed @@ -159,9 +165,9 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block bookkeeping expFeeFactor: pool.cfg.TxPoolExponentialIncreaseFactor, txPoolMaxSize: pool.cfg.TxPoolSize, proposalAssemblyTime: pool.cfg.ProposalAssemblyTime, - log: pool.log, //TODO(yossi) change the logger's copy to add a prefix indicating this is the pool's copy/speculation. So failures will be indicative + log: pool.log, cfg: pool.cfg, - ctx: ctx, + ctx: copyPoolctx, } copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu @@ -169,9 +175,13 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(block bookkeeping pool.cancelSpec = cancel specDoneCh := make(chan struct{}) pool.specAsmDone = specDoneCh - pool.specBlock = make(chan *ledgercore.ValidatedBlock, 1) - return ©, pool.specBlock, specDoneCh, statedelta, nil + specBlockCh := make(chan *ledgercore.ValidatedBlock, 1) + pool.specBlockMu.Lock() + pool.specBlockCh = specBlockCh + pool.specBlockMu.Unlock() + + return ©, specBlockCh, specDoneCh, nil } // poolAsmResults is used to syncronize the state of the block assembly process. @@ -529,7 +539,7 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } -func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block) { +func (pool *TransactionPool) OnNewSpeculativeBlock(ctx context.Context, vb *ledgercore.ValidatedBlock) { pool.mu.Lock() // cancel any pending speculative assembly @@ -542,7 +552,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block) { pool.rememberCommit(false) // create shallow pool copy - speculativePool, outchan, specAsmDoneCh, delta, err := pool.copyTransactionPoolOverSpecLedger(block) + speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(ctx, vb) defer close(specAsmDoneCh) pool.mu.Unlock() @@ -552,7 +562,7 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block) { } // process txns only until one block is full - speculativePool.onNewBlock(block, delta, true) + speculativePool.onNewBlock(vb.Block(), vb.Delta(), true) if speculativePool.assemblyResults.err != nil { return @@ -566,12 +576,11 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(block bookkeeping.Block) { } func (pool *TransactionPool) tryReadSpeculativeBlock(branch bookkeeping.BlockHash) (*ledgercore.ValidatedBlock, error) { - // TODO(yossi) is taking the lock here necessary? is it safe (check lock taking order)? - pool.mu.Lock() - defer pool.mu.Unlock() + pool.specBlockMu.Lock() + defer pool.specBlockMu.Unlock() select { - case vb := <-pool.specBlock: + case vb := <-pool.specBlockCh: if vb == nil { return nil, fmt.Errorf("speculation block is nil") } diff --git a/installer/config.json.example b/installer/config.json.example index 76b1700a7b..6d22d3239c 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,5 @@ { - "Version": 23, + "Version": 24, "AccountUpdatesStatsInterval": 5000000000, "AccountsRebuildSynchronousMode": 1, "AgreementIncomingBundlesQueueLength": 7, @@ -89,6 +89,7 @@ "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, + "SpeculativeBlockAssemblyGraceTime": 50000000, "SuggestedFeeBlockHistory": 3, "SuggestedFeeSlidingWindowSize": 50, "TLSCertFile": "", diff --git a/ledger/speculative.go b/ledger/speculative.go index c15d4671ab..c96baa8cab 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -17,7 +17,6 @@ package ledger import ( - "context" "fmt" "github.com/algorand/go-algorand/config" @@ -57,24 +56,13 @@ type LedgerForEvaluator interface { // on top, which in turn allows speculatively constructing a subsequent // block, before the ValidatedBlock is committed to the ledger. // -// This is what the state looks like: -// -// previousLFE <--------- roundCowBase -// ^ l ^ -// | | lookupParent -// | | -// | roundCowState -// | ^ -// | | state -// | | -// | ValidatedBlock -------> Block +// ledger ValidatedBlock -------> Block // | ^ blk // | | vb // | l | // \---------- validatedBlockAsLFE // -// where previousLFE might be the full ledger, or might be another -// validatedBlockAsLFE. +// where ledger is the full ledger. type validatedBlockAsLFE struct { // l points to the underlying ledger; it might be another instance // of validatedBlockAsLFE if we are speculating on a chain of many @@ -86,7 +74,7 @@ type validatedBlockAsLFE struct { vb *ledgercore.ValidatedBlock } -// makeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a ValidatedBlock. +// MakeValidatedBlockAsLFE constructs a new validatedBlockAsLFE from a ValidatedBlock. func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator) (*validatedBlockAsLFE, error) { latestRound := l.Latest() if vb.Block().Round().SubSaturate(1) != latestRound { @@ -106,18 +94,6 @@ func MakeValidatedBlockAsLFE(vb *ledgercore.ValidatedBlock, l LedgerForEvaluator }, nil } -// makeBlockAsLFE constructs a new validatedBlockAsLFE from a Block. -func MakeBlockAsLFE(blk bookkeeping.Block, l LedgerForEvaluator) (*validatedBlockAsLFE, ledgercore.StateDelta, error) { - state, err := internal.Eval(context.Background(), l, blk, false, l.VerifiedTransactionCache(), nil) - if err != nil { - return nil, ledgercore.StateDelta{}, fmt.Errorf("error computing deltas for block %d round %d: %v", blk.Hash(), blk.Round(), err) - } - - vb := ledgercore.MakeValidatedBlock(blk, state) - lfe, err := MakeValidatedBlockAsLFE(&vb, l) - return lfe, vb.Delta(), nil -} - // Block implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) Block(r basics.Round) (bookkeeping.Block, error) { if r == v.vb.Block().Round() { @@ -243,7 +219,6 @@ func (v *validatedBlockAsLFE) LookupWithoutRewards(rnd basics.Round, a basics.Ad } // VerifiedTransactionCache implements the ledgerForEvaluator interface. -// TODO(yossi) should we forward this cache? func (v *validatedBlockAsLFE) VerifiedTransactionCache() verify.VerifiedTransactionCache { return v.l.VerifiedTransactionCache() } diff --git a/node/node.go b/node/node.go index 2f972cc48a..9f27d10d27 100644 --- a/node/node.go +++ b/node/node.go @@ -1265,8 +1265,9 @@ func (node *AlgorandFullNode) AssembleBlock(round basics.Round) (agreement.Valid return validatedBlock{vb: lvb}, nil } -func (node *AlgorandFullNode) OnNewSpeculativeBlock(blk bookkeeping.Block) { - node.transactionPool.OnNewSpeculativeBlock(blk) +func (node *AlgorandFullNode) OnNewSpeculativeBlock(ctx context.Context, vb agreement.ValidatedBlock) { + + node.transactionPool.OnNewSpeculativeBlock(ctx, vb.(validatedBlock).vb) } // getOfflineClosedStatus will return an int with the appropriate bit(s) set if it is offline and/or online diff --git a/test/testdata/configs/config-v24.json b/test/testdata/configs/config-v24.json new file mode 100644 index 0000000000..f95df3b337 --- /dev/null +++ b/test/testdata/configs/config-v24.json @@ -0,0 +1,107 @@ +{ + "Version": 23, + "AccountUpdatesStatsInterval": 5000000000, + "AccountsRebuildSynchronousMode": 1, + "AgreementIncomingBundlesQueueLength": 7, + "AgreementIncomingProposalsQueueLength": 25, + "AgreementIncomingVotesQueueLength": 10000, + "AnnounceParticipationKey": true, + "Archival": false, + "BaseLoggerDebugLevel": 4, + "BlockServiceCustomFallbackEndpoints": "", + "BroadcastConnectionsLimit": -1, + "CadaverSizeTarget": 1073741824, + "CatchpointFileHistoryLength": 365, + "CatchpointInterval": 10000, + "CatchpointTracking": 0, + "CatchupBlockDownloadRetryAttempts": 1000, + "CatchupBlockValidateMode": 0, + "CatchupFailurePeerRefreshRate": 10, + "CatchupGossipBlockFetchTimeoutSec": 4, + "CatchupHTTPBlockFetchTimeoutSec": 4, + "CatchupLedgerDownloadRetryAttempts": 50, + "CatchupParallelBlocks": 16, + "ConnectionsRateLimitingCount": 60, + "ConnectionsRateLimitingWindowSeconds": 1, + "DNSBootstrapID": ".algorand.network", + "DNSSecurityFlags": 1, + "DeadlockDetection": 0, + "DeadlockDetectionThreshold": 30, + "DisableLocalhostConnectionRateLimit": true, + "DisableNetworking": false, + "DisableOutgoingConnectionThrottling": false, + "EnableAccountUpdatesStats": false, + "EnableAgreementReporting": false, + "EnableAgreementTimeMetrics": false, + "EnableAssembleStats": false, + "EnableBlockService": false, + "EnableBlockServiceFallbackToArchiver": true, + "EnableCatchupFromArchiveServers": false, + "EnableDeveloperAPI": false, + "EnableGossipBlockService": true, + "EnableIncomingMessageFilter": false, + "EnableLedgerService": false, + "EnableMetricReporting": false, + "EnableOutgoingNetworkMessageFiltering": true, + "EnablePingHandler": true, + "EnableProcessBlockStats": false, + "EnableProfiler": false, + "EnableRequestLogger": false, + "EnableRuntimeMetrics": false, + "EnableTopAccountsReporting": false, + "EnableVerbosedTransactionSyncLogging": false, + "EndpointAddress": "127.0.0.1:0", + "FallbackDNSResolverAddress": "", + "ForceFetchTransactions": false, + "ForceRelayMessages": false, + "GossipFanout": 4, + "IncomingConnectionsLimit": 800, + "IncomingMessageFilterBucketCount": 5, + "IncomingMessageFilterBucketSize": 512, + "IsIndexerActive": false, + "LedgerSynchronousMode": 2, + "LogArchiveMaxAge": "", + "LogArchiveName": "node.archive.log", + "LogSizeLimit": 1073741824, + "MaxAcctLookback": 4, + "MaxAPIResourcesPerAccount": 100000, + "MaxCatchpointDownloadDuration": 7200000000000, + "MaxConnectionsPerIP": 30, + "MinCatchpointFileDownloadBytesPerSecond": 20480, + "NetAddress": "", + "NetworkMessageTraceServer": "", + "NetworkProtocolVersion": "", + "NodeExporterListenAddress": ":9100", + "NodeExporterPath": "./node_exporter", + "OptimizeAccountsDatabaseOnStartup": false, + "OutgoingMessageFilterBucketCount": 3, + "OutgoingMessageFilterBucketSize": 128, + "ParticipationKeysRefreshInterval": 60000000000, + "PeerConnectionsUpdateInterval": 3600, + "PeerPingPeriodSeconds": 0, + "PriorityPeers": {}, + "ProposalAssemblyTime": 500000000, + "PublicAddress": "", + "ReconnectTime": 60000000000, + "ReservedFDs": 256, + "RestConnectionsHardLimit": 2048, + "RestConnectionsSoftLimit": 1024, + "RestReadTimeoutSeconds": 15, + "RestWriteTimeoutSeconds": 120, + "RunHosted": false, + "SuggestedFeeBlockHistory": 3, + "SuggestedFeeSlidingWindowSize": 50, + "TLSCertFile": "", + "TLSKeyFile": "", + "TelemetryToLog": true, + "TransactionSyncDataExchangeRate": 0, + "TransactionSyncSignificantMessageThreshold": 0, + "TxPoolExponentialIncreaseFactor": 2, + "TxPoolSize": 75000, + "TxSyncIntervalSeconds": 60, + "TxSyncServeResponseSize": 1000000, + "TxSyncTimeoutSeconds": 30, + "UseXForwardedForAddressField": "", + "VerifiedTranscationsCacheSize": 150000, + "SpeculativeBlockAssemblyGraceTime": 50000000 +} From aea39caad51143c12af223af8c3b1070e4b2e7c7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Fri, 5 Aug 2022 10:36:39 -0400 Subject: [PATCH 31/50] check casting --- node/node.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/node/node.go b/node/node.go index 9f27d10d27..1aabd202da 100644 --- a/node/node.go +++ b/node/node.go @@ -1265,9 +1265,13 @@ func (node *AlgorandFullNode) AssembleBlock(round basics.Round) (agreement.Valid return validatedBlock{vb: lvb}, nil } -func (node *AlgorandFullNode) OnNewSpeculativeBlock(ctx context.Context, vb agreement.ValidatedBlock) { - - node.transactionPool.OnNewSpeculativeBlock(ctx, vb.(validatedBlock).vb) +func (node *AlgorandFullNode) OnNewSpeculativeBlock(ctx context.Context, avb agreement.ValidatedBlock) { + vb, ok := avb.(validatedBlock) + if ok { + node.transactionPool.OnNewSpeculativeBlock(ctx, vb.vb) + } else { + node.log.Errorf("cannot convert agreement ValidatedBlock to ValidateBlock") + } } // getOfflineClosedStatus will return an int with the appropriate bit(s) set if it is offline and/or online From e883f2ed1490d5a551129aa0843ca4cf585e4d4d Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 8 Aug 2022 13:24:21 -0400 Subject: [PATCH 32/50] bug fix + unit test --- data/pools/transactionPool.go | 4 +- data/pools/transactionPool_test.go | 89 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 6fc0e7860c..218ec273e5 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -156,6 +156,7 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Conte copy := TransactionPool{ pendingTxids: pool.pendingTxids, // it is safe to shallow copy pendingTxids since this map is only (atomically) swapped and never directly changed + pendingTxGroups: pool.pendingTxGroups, rememberedTxids: make(map[transactions.Txid]transactions.SignedTxn), expiredTxCount: make(map[basics.Round]int), ledger: specLedger, @@ -551,7 +552,8 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(ctx context.Context, vb *ledg // move remembered txns to pending pool.rememberCommit(false) - // create shallow pool copy + // create shallow pool copy, close the done channel when we're done with + // speculative block assembly. speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(ctx, vb) defer close(specAsmDoneCh) pool.mu.Unlock() diff --git a/data/pools/transactionPool_test.go b/data/pools/transactionPool_test.go index 2f1bd75b6c..ebd2f2e1dc 100644 --- a/data/pools/transactionPool_test.go +++ b/data/pools/transactionPool_test.go @@ -17,6 +17,7 @@ package pools import ( + "context" "fmt" "math/rand" "strings" @@ -1271,3 +1272,91 @@ func TestTxPoolSizeLimits(t *testing.T) { } } } + +func TestSepculativeBlockAssembly(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfAccounts := 10 + // Generate accounts + secrets := make([]*crypto.SignatureSecrets, numOfAccounts) + addresses := make([]basics.Address, numOfAccounts) + + for i := 0; i < numOfAccounts; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + } + + mockLedger := makeMockLedger(t, initAccFixed(addresses, 1<<32)) + cfg := config.GetDefaultLocal() + cfg.TxPoolSize = testPoolSize + cfg.EnableProcessBlockStats = false + transactionPool := MakeTransactionPool(mockLedger, cfg, logging.Base()) + + savedTransactions := 0 + for i, sender := range addresses { + amount := uint64(0) + for _, receiver := range addresses { + if sender != receiver { + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: sender, + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee + amount}, + FirstValid: 0, + LastValid: 10, + Note: make([]byte, 0), + GenesisHash: mockLedger.GenesisHash(), + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: receiver, + Amount: basics.MicroAlgos{Raw: 0}, + }, + } + amount++ + + signedTx := tx.Sign(secrets[i]) + require.NoError(t, transactionPool.RememberOne(signedTx)) + savedTransactions++ + } + } + } + pending := transactionPool.PendingTxGroups() + require.Len(t, pending, savedTransactions) + + secret := keypair() + recv := basics.Address(secret.SignatureVerifier) + + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: addresses[0], + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee}, + FirstValid: 0, + LastValid: 10, + Note: []byte{1}, + GenesisHash: mockLedger.GenesisHash(), + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: recv, + Amount: basics.MicroAlgos{Raw: 0}, + }, + } + signedTx := tx.Sign(secrets[0]) + + blockEval := newBlockEvaluator(t, mockLedger) + err := blockEval.Transaction(signedTx, transactions.ApplyData{}) + require.NoError(t, err) + + // simulate this transaction was applied + block, err := blockEval.GenerateBlock() + require.NoError(t, err) + + transactionPool.OnNewSpeculativeBlock(context.Background(), block) + <-transactionPool.specAsmDone + specBlock, err := transactionPool.tryReadSpeculativeBlock(block.Block().Hash()) + require.NoError(t, err) + require.NotNil(t, specBlock) + require.Len(t, specBlock.Block().Payset, savedTransactions) +} From 68c3b77ab64d4585464f16d1d469650cdac12b8d Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 8 Aug 2022 15:03:00 -0400 Subject: [PATCH 33/50] speculative txn not in block --- data/pools/transactionPool_test.go | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/data/pools/transactionPool_test.go b/data/pools/transactionPool_test.go index ebd2f2e1dc..e2ee821147 100644 --- a/data/pools/transactionPool_test.go +++ b/data/pools/transactionPool_test.go @@ -1360,3 +1360,76 @@ func TestSepculativeBlockAssembly(t *testing.T) { require.NotNil(t, specBlock) require.Len(t, specBlock.Block().Payset, savedTransactions) } + +func TestSepculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfAccounts := 10 + // Generate accounts + secrets := make([]*crypto.SignatureSecrets, numOfAccounts) + addresses := make([]basics.Address, numOfAccounts) + + for i := 0; i < numOfAccounts; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + } + + mockLedger := makeMockLedger(t, initAccFixed(addresses, 1<<32)) + cfg := config.GetDefaultLocal() + cfg.TxPoolSize = testPoolSize + cfg.EnableProcessBlockStats = false + transactionPool := MakeTransactionPool(mockLedger, cfg, logging.Base()) + + savedTransactions := 0 + pendingTxn := transactions.SignedTxn{} + for i, sender := range addresses { + amount := uint64(0) + for _, receiver := range addresses { + if sender != receiver { + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: sender, + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee + amount}, + FirstValid: 0, + LastValid: 10, + Note: make([]byte, 0), + GenesisHash: mockLedger.GenesisHash(), + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: receiver, + Amount: basics.MicroAlgos{Raw: 0}, + }, + } + amount++ + + signedTx := tx.Sign(secrets[i]) + require.NoError(t, transactionPool.RememberOne(signedTx)) + pendingTxn = signedTx + savedTransactions++ + } + } + } + pending := transactionPool.PendingTxGroups() + require.Len(t, pending, savedTransactions) + + blockEval := newBlockEvaluator(t, mockLedger) + err := blockEval.Transaction(pendingTxn, transactions.ApplyData{}) + require.NoError(t, err) + + // simulate this transaction was applied + block, err := blockEval.GenerateBlock() + require.NoError(t, err) + + transactionPool.OnNewSpeculativeBlock(context.Background(), block) + <-transactionPool.specAsmDone + specBlock, err := transactionPool.tryReadSpeculativeBlock(block.Block().Hash()) + require.NoError(t, err) + require.NotNil(t, specBlock) + // assembled block doesn't have txn in the speculated block + require.Len(t, specBlock.Block().Payset, savedTransactions-1) + // tx pool unaffected + require.Len(t, transactionPool.PendingTxIDs(), savedTransactions) +} From edb6e4dc858d45526ca79dc0099c630ed0f8ecb5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 8 Aug 2022 15:23:34 -0400 Subject: [PATCH 34/50] asm round to next round --- data/pools/transactionPool.go | 1 + 1 file changed, 1 insertion(+) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 218ec273e5..9518cf705c 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -166,6 +166,7 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Conte expFeeFactor: pool.cfg.TxPoolExponentialIncreaseFactor, txPoolMaxSize: pool.cfg.TxPoolSize, proposalAssemblyTime: pool.cfg.ProposalAssemblyTime, + assemblyRound: specLedger.Latest() + 1, log: pool.log, cfg: pool.cfg, ctx: copyPoolctx, From 94b9292fcf307c67840280d0641346bf610f79e7 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 8 Aug 2022 15:28:00 -0400 Subject: [PATCH 35/50] better member name --- data/pools/transactionPool.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 9518cf705c..a4e0aababd 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -95,8 +95,8 @@ type TransactionPool struct { // proposalAssemblyTime is the ProposalAssemblyTime configured for this node. proposalAssemblyTime time.Duration - ctx context.Context - cancelSpec context.CancelFunc + ctx context.Context + cancelSpeculativeAssembly context.CancelFunc // specBlockCh has an assembled speculative block specBlockCh chan *ledgercore.ValidatedBlock @@ -174,7 +174,7 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Conte copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu - pool.cancelSpec = cancel + pool.cancelSpeculativeAssembly = cancel specDoneCh := make(chan struct{}) pool.specAsmDone = specDoneCh @@ -545,8 +545,8 @@ func (pool *TransactionPool) OnNewSpeculativeBlock(ctx context.Context, vb *ledg pool.mu.Lock() // cancel any pending speculative assembly - if pool.cancelSpec != nil { - pool.cancelSpec() + if pool.cancelSpeculativeAssembly != nil { + pool.cancelSpeculativeAssembly() <-pool.specAsmDone } From 50560d80a02c3176a3a57fecc6e2296c1a817d27 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 9 Aug 2022 14:04:50 -0400 Subject: [PATCH 36/50] check good txns --- data/pools/transactionPool.go | 1 + data/pools/transactionPool_test.go | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index a4e0aababd..e11f798da4 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -853,6 +853,7 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact asmStats.EarlyCommittedCount++ continue } + hasBlock, err := pool.add(txgroup, &asmStats, speculativeAsm) if err != nil { for _, tx := range txgroup { diff --git a/data/pools/transactionPool_test.go b/data/pools/transactionPool_test.go index e2ee821147..527d6fef49 100644 --- a/data/pools/transactionPool_test.go +++ b/data/pools/transactionPool_test.go @@ -1384,6 +1384,7 @@ func TestSepculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { savedTransactions := 0 pendingTxn := transactions.SignedTxn{} + pendingTxIDSet := make(map[crypto.Signature]bool) for i, sender := range addresses { amount := uint64(0) for _, receiver := range addresses { @@ -1405,15 +1406,16 @@ func TestSepculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { } amount++ - signedTx := tx.Sign(secrets[i]) - require.NoError(t, transactionPool.RememberOne(signedTx)) - pendingTxn = signedTx + pendingTxn = tx.Sign(secrets[i]) + require.NoError(t, transactionPool.RememberOne(pendingTxn)) + pendingTxIDSet[pendingTxn.Sig] = true savedTransactions++ } } } pending := transactionPool.PendingTxGroups() require.Len(t, pending, savedTransactions) + require.Len(t, pendingTxIDSet, savedTransactions) blockEval := newBlockEvaluator(t, mockLedger) err := blockEval.Transaction(pendingTxn, transactions.ApplyData{}) @@ -1430,6 +1432,12 @@ func TestSepculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { require.NotNil(t, specBlock) // assembled block doesn't have txn in the speculated block require.Len(t, specBlock.Block().Payset, savedTransactions-1) + // tx pool unaffected require.Len(t, transactionPool.PendingTxIDs(), savedTransactions) + + for _, txn := range specBlock.Block().Payset { + require.NotEqual(t, txn.SignedTxn.Sig, pendingTxn.Sig) + require.True(t, pendingTxIDSet[txn.SignedTxn.Sig]) + } } From beaf0c6a6ecdd0bce6fa37ab2dcc92dc74506262 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 9 Aug 2022 19:59:40 -0400 Subject: [PATCH 37/50] better test --- data/pools/transactionPool_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/data/pools/transactionPool_test.go b/data/pools/transactionPool_test.go index 527d6fef49..751f01ce8a 100644 --- a/data/pools/transactionPool_test.go +++ b/data/pools/transactionPool_test.go @@ -22,6 +22,7 @@ import ( "math/rand" "strings" "testing" + "time" "github.com/stretchr/testify/require" @@ -1355,8 +1356,18 @@ func TestSepculativeBlockAssembly(t *testing.T) { transactionPool.OnNewSpeculativeBlock(context.Background(), block) <-transactionPool.specAsmDone - specBlock, err := transactionPool.tryReadSpeculativeBlock(block.Block().Hash()) + + // add the block + mockLedger.AddBlock(block.Block(), agreement.Certificate{}) + + // empty tx pool + transactionPool.pendingTxids = make(map[transactions.Txid]transactions.SignedTxn) + transactionPool.pendingTxGroups = nil + + // check that we still assemble the block + specBlock, err := transactionPool.AssembleBlock(block.Block().Round()+1, time.Now().Add(10*time.Millisecond)) require.NoError(t, err) + require.Equal(t, specBlock.Block().Branch, block.Block().Hash()) require.NotNil(t, specBlock) require.Len(t, specBlock.Block().Payset, savedTransactions) } From 735abb638c238a9731e1d7609a3636452312c1b5 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 9 Aug 2022 20:14:40 -0400 Subject: [PATCH 38/50] reset clears speculative results --- data/pools/transactionPool.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index e11f798da4..98002a8fa6 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -241,6 +241,13 @@ func (pool *TransactionPool) Reset() { pool.pendingBlockEvaluator = nil pool.statusCache.reset() pool.recomputeBlockEvaluator(nil, 0, false) + + // cancel speculative assembly and clear its result + if pool.cancelSpeculativeAssembly != nil { + pool.cancelSpeculativeAssembly() + <-pool.specAsmDone + } + pool.specBlockCh = nil } // NumExpired returns the number of transactions that expired at the From 6b580c67f89fd8f248460301f9a4a56cfecdcf33 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 17 Aug 2022 16:58:06 -0400 Subject: [PATCH 39/50] change the way pendingTxids and pendingTxGroups are copied --- data/pools/transactionPool.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 98002a8fa6..83602a3d1c 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -155,8 +155,8 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Conte copyPoolctx, cancel := context.WithCancel(ctx) copy := TransactionPool{ - pendingTxids: pool.pendingTxids, // it is safe to shallow copy pendingTxids since this map is only (atomically) swapped and never directly changed - pendingTxGroups: pool.pendingTxGroups, + pendingTxids: make(map[transactions.Txid]transactions.SignedTxn), // pendingTxIds is only used for stats and hints + pendingTxGroups: pool.pendingTxGroups[:], rememberedTxids: make(map[transactions.Txid]transactions.SignedTxn), expiredTxCount: make(map[basics.Round]int), ledger: specLedger, From fbc2718a47abca19b513cb5e5f5f7ef1f8958855 Mon Sep 17 00:00:00 2001 From: Nicholas Guo Date: Thu, 15 Sep 2022 12:00:43 -0700 Subject: [PATCH 40/50] add test --- agreement/player.go | 8 +- agreement/service.go | 4 +- data/pools/transactionPool_test.go | 122 ++++++++++++++++++++++++++++- ledger/ledger_test.go | 2 +- 4 files changed, 127 insertions(+), 9 deletions(-) diff --git a/agreement/player.go b/agreement/player.go index eecbd6f63a..300c0fc2be 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -56,8 +56,8 @@ type player struct { // must be verified after some vote has been verified. Pending proposalTable - // the current concensus version - ConcensusVersion protocol.ConsensusVersion + // the current consensus version + ConsensusVersion protocol.ConsensusVersion // the time offset before the filter deadline to start speculative block assembly SpeculativeAsmTimeDuration time.Duration @@ -365,7 +365,7 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.FastRecoveryDeadline = 0 // set immediately p.Deadline = FilterTimeout(target, source.Proto) if target == 0 { - p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(target, p.ConcensusVersion, p.SpeculativeAsmTimeDuration) + p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(target, p.ConsensusVersion, p.SpeculativeAsmTimeDuration) } else { // only speculate on block assembly in period 0 p.SpeculativeAssemblyDeadline = 0 @@ -414,7 +414,7 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately - p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(0, p.ConcensusVersion, p.SpeculativeAsmTimeDuration) + p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(0, p.ConsensusVersion, p.SpeculativeAsmTimeDuration) switch source := source.(type) { case roundInterruptionEvent: diff --git a/agreement/service.go b/agreement/service.go index 7f23374d88..44f6bf0001 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -224,9 +224,9 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r // set speculative block assembly based on the current local configuration status.SpeculativeAsmTimeDuration = s.parameters.Local.ProposalAssemblyTime + s.parameters.Local.SpeculativeBlockAssemblyGraceTime for { - status.ConcensusVersion, err = s.Ledger.ConsensusVersion(status.Round) + status.ConsensusVersion, err = s.Ledger.ConsensusVersion(status.Round) if err != nil { - s.Panicf("cannot read latest concensus version, round %d: %v", status.Round, err) + s.Panicf("cannot read latest consensus version, round %d: %v", status.Round, err) } output <- a diff --git a/data/pools/transactionPool_test.go b/data/pools/transactionPool_test.go index 01cd121c30..8969e9032e 100644 --- a/data/pools/transactionPool_test.go +++ b/data/pools/transactionPool_test.go @@ -23,6 +23,7 @@ import ( "fmt" "math/rand" "strings" + "sync" "testing" "time" @@ -1507,7 +1508,7 @@ func generateProofForTesting( return proof } -func TestSepculativeBlockAssembly(t *testing.T) { +func TestSpeculativeBlockAssembly(t *testing.T) { partitiontest.PartitionTest(t) numOfAccounts := 10 @@ -1605,7 +1606,7 @@ func TestSepculativeBlockAssembly(t *testing.T) { require.Len(t, specBlock.Block().Payset, savedTransactions) } -func TestSepculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { +func TestSpeculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { partitiontest.PartitionTest(t) numOfAccounts := 10 @@ -1685,3 +1686,120 @@ func TestSepculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { require.True(t, pendingTxIDSet[txn.SignedTxn.Sig]) } } + +// This test runs the speculative block assembly and adds txns to the pool in another thread +func TestSpeculativeBlockAssemblyDataRace(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfAccounts := 10 + // Generate accounts + secrets := make([]*crypto.SignatureSecrets, numOfAccounts) + addresses := make([]basics.Address, numOfAccounts) + + for i := 0; i < numOfAccounts; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + } + + mockLedger := makeMockLedger(t, initAccFixed(addresses, 1<<32)) + cfg := config.GetDefaultLocal() + cfg.TxPoolSize = testPoolSize + cfg.EnableProcessBlockStats = false + transactionPool := MakeTransactionPool(mockLedger, cfg, logging.Base()) + + savedTransactions := 0 + pendingTxn := transactions.SignedTxn{} + pendingTxIDSet := make(map[crypto.Signature]bool) + for i, sender := range addresses { + amount := uint64(0) + for _, receiver := range addresses { + if sender != receiver { + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: sender, + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee + amount}, + FirstValid: 0, + LastValid: 10, + Note: make([]byte, 0), + GenesisHash: mockLedger.GenesisHash(), + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: receiver, + Amount: basics.MicroAlgos{Raw: 0}, + }, + } + amount++ + + pendingTxn = tx.Sign(secrets[i]) + require.NoError(t, transactionPool.RememberOne(pendingTxn)) + pendingTxIDSet[pendingTxn.Sig] = true + savedTransactions++ + } + } + } + pending := transactionPool.PendingTxGroups() + require.Len(t, pending, savedTransactions) + require.Len(t, pendingTxIDSet, savedTransactions) + + blockEval := newBlockEvaluator(t, mockLedger) + err := blockEval.Transaction(pendingTxn, transactions.ApplyData{}) + require.NoError(t, err) + + // simulate this transaction was applied + block, err := blockEval.GenerateBlock() + require.NoError(t, err) + + newSavedTransactions := 0 + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i, sender := range addresses { + amount := uint64(0) + for _, receiver := range addresses { + if sender != receiver { + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: sender, + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee + amount}, + FirstValid: 0, + LastValid: 11, + Note: make([]byte, 0), + GenesisHash: mockLedger.GenesisHash(), + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: receiver, + Amount: basics.MicroAlgos{Raw: 0}, + }, + } + amount++ + + pendingTxn = tx.Sign(secrets[i]) + require.NoError(t, transactionPool.RememberOne(pendingTxn)) + pendingTxIDSet[pendingTxn.Sig] = true + newSavedTransactions++ + } + } + } + }() + transactionPool.OnNewSpeculativeBlock(context.Background(), block) + wg.Wait() + <-transactionPool.specAsmDone + specBlock, err := transactionPool.tryReadSpeculativeBlock(block.Block().Hash()) + require.NoError(t, err) + require.NotNil(t, specBlock) + // assembled block doesn't have txn in the speculated block + require.Len(t, specBlock.Block().Payset, savedTransactions-1) + + // tx pool should have old txns and new txns + require.Len(t, transactionPool.PendingTxIDs(), savedTransactions+newSavedTransactions) + + for _, txn := range specBlock.Block().Payset { + require.NotEqual(t, txn.SignedTxn.Sig, pendingTxn.Sig) + require.True(t, pendingTxIDSet[txn.SignedTxn.Sig]) + } +} \ No newline at end of file diff --git a/ledger/ledger_test.go b/ledger/ledger_test.go index 95a0a84e22..adc1ab6b06 100644 --- a/ledger/ledger_test.go +++ b/ledger/ledger_test.go @@ -1095,7 +1095,7 @@ func testLedgerSingleTxApplyData(t *testing.T, version protocol.ConsensusVersion VoteLast: 10000, } - // depends on what the concensus is need to generate correct KeyregTxnFields. + // depends on what the consensus is need to generate correct KeyregTxnFields. if proto.EnableStateProofKeyregCheck { frst, lst := uint64(correctKeyregFields.VoteFirst), uint64(correctKeyregFields.VoteLast) store, err := db.MakeAccessor("test-DB", false, true) From 6d9bae1a594ea39fcb2f8b39a9d929f9b14b9938 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 19 Sep 2022 16:00:50 -0400 Subject: [PATCH 41/50] bug fix: return correct round number --- ledger/speculative.go | 3 ++- ledger/speculative_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ledger/speculative.go b/ledger/speculative.go index 857268d0fa..20434352f0 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -215,7 +215,8 @@ func (v *validatedBlockAsLFE) LookupWithoutRewards(rnd basics.Round, a basics.Ad } // account didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger - return v.l.LookupWithoutRewards(rnd, a) + acctData, _, err := v.l.LookupWithoutRewards(rnd, a) + return acctData, rnd, err } // VerifiedTransactionCache implements the ledgerForEvaluator interface. diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go index b42d87ef11..f00edc1d45 100644 --- a/ledger/speculative_test.go +++ b/ledger/speculative_test.go @@ -107,7 +107,7 @@ func TestSpeculative(t *testing.T) { ad11, rnd, err := blk2aslfe.LookupWithoutRewards(blk1.Round(), addr1) require.NoError(t, err) // account was never changed - require.Equal(t, rnd, basics.Round(0)) + require.Equal(t, rnd, blk1.Round()) ad22, rnd, err := blk2aslfe.LookupWithoutRewards(blk2.Round(), addr1) require.NoError(t, err) From f34825a2a7ed557cfbe07b4e6f2c7f2410d7642f Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Mon, 19 Sep 2022 16:07:42 -0400 Subject: [PATCH 42/50] preserve round number response in case of an error --- ledger/speculative.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ledger/speculative.go b/ledger/speculative.go index 20434352f0..cb42f36ab3 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -215,7 +215,10 @@ func (v *validatedBlockAsLFE) LookupWithoutRewards(rnd basics.Round, a basics.Ad } // account didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger - acctData, _, err := v.l.LookupWithoutRewards(rnd, a) + acctData, fallbackrnd, err := v.l.LookupWithoutRewards(rnd, a) + if err != nil { + return acctData, fallbackrnd, err + } return acctData, rnd, err } From 3599437f8222bce60608e8a8f00ed9dca6ceadb3 Mon Sep 17 00:00:00 2001 From: Nicholas Guo Date: Mon, 19 Sep 2022 16:25:11 -0400 Subject: [PATCH 43/50] use round from 2 rounds ago --- agreement/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agreement/service.go b/agreement/service.go index 7f23374d88..883d03f361 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -224,9 +224,9 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r // set speculative block assembly based on the current local configuration status.SpeculativeAsmTimeDuration = s.parameters.Local.ProposalAssemblyTime + s.parameters.Local.SpeculativeBlockAssemblyGraceTime for { - status.ConcensusVersion, err = s.Ledger.ConsensusVersion(status.Round) + status.ConcensusVersion, err = s.Ledger.ConsensusVersion(status.Round.SubSaturate(2)) if err != nil { - s.Panicf("cannot read latest concensus version, round %d: %v", status.Round, err) + s.Panicf("cannot read latest concensus version, round %d: %v", status.Round.SubSaturate(2), err) } output <- a From 1d81a56eb2580e67d3b867cbb65bce9c98194cda Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Tue, 20 Sep 2022 10:48:05 -0400 Subject: [PATCH 44/50] setting clock for first period after boot --- agreement/service.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agreement/service.go b/agreement/service.go index 7f23374d88..f1a97d06e1 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -229,8 +229,10 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.Panicf("cannot read latest concensus version, round %d: %v", status.Round, err) } + specClock := SpeculativeBlockAsmTime(status.Period, status.ConcensusVersion, status.SpeculativeAsmTimeDuration) + output <- a - ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, SpeculativeBlockAsmDeadline: status.SpeculativeAssemblyDeadline, CurrentRound: status.Round} + ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, SpeculativeBlockAsmDeadline: specClock, CurrentRound: status.Round} e, ok := <-input if !ok { break From 044a72a04df1cd4f5230ce472f6b740297c94321 Mon Sep 17 00:00:00 2001 From: Nicholas Guo Date: Tue, 4 Oct 2022 12:16:31 -0400 Subject: [PATCH 45/50] optimize --- agreement/service.go | 4 ++-- data/pools/transactionPool.go | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/agreement/service.go b/agreement/service.go index 7c4c4971ff..4dd94e2a12 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -221,14 +221,14 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r s.Clock = clock } - // set speculative block assembly based on the current local configuration - status.SpeculativeAsmTimeDuration = s.parameters.Local.ProposalAssemblyTime + s.parameters.Local.SpeculativeBlockAssemblyGraceTime for { status.ConcensusVersion, err = s.Ledger.ConsensusVersion(status.Round.SubSaturate(2)) if err != nil { s.Panicf("cannot read latest concensus version, round %d: %v", status.Round.SubSaturate(2), err) } + // set speculative block assembly based on the current local configuration + status.SpeculativeAsmTimeDuration = s.parameters.Local.ProposalAssemblyTime + s.parameters.Local.SpeculativeBlockAssemblyGraceTime specClock := SpeculativeBlockAsmTime(status.Period, status.ConcensusVersion, status.SpeculativeAsmTimeDuration) output <- a diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 11d8d99008..66eaf98c18 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -645,6 +645,10 @@ func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercor defer pool.mu.Unlock() defer pool.cond.Broadcast() + if !stopReprocessingAtFirstAsmBlock && pool.cancelSpeculativeAssembly != nil { + pool.cancelSpeculativeAssembly() + } + if pool.pendingBlockEvaluator == nil || block.Round() >= pool.pendingBlockEvaluator.Round() { // Adjust the pool fee threshold. The rules are: // - If there was less than one full block in the pool, reduce @@ -672,7 +676,7 @@ func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercor // Recompute the pool by starting from the new latest block. // This has the side-effect of discarding transactions that // have been committed (or that are otherwise no longer valid). - stats = pool.recomputeBlockEvaluator(committedTxids, knownCommitted, false) + stats = pool.recomputeBlockEvaluator(committedTxids, knownCommitted, stopReprocessingAtFirstAsmBlock) } stats.KnownCommittedCount = knownCommitted From 3cbfe53a418c4a2e6ea81ac030128aa39b164614 Mon Sep 17 00:00:00 2001 From: Nicholas Guo Date: Wed, 12 Oct 2022 16:47:19 -0400 Subject: [PATCH 46/50] lint --- agreement/service.go | 2 +- agreement/types.go | 1 + data/pools/transactionPool.go | 1 + ledger/ledgercore/validatedBlock.go | 2 ++ ledger/speculative.go | 1 + node/node.go | 1 + 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/agreement/service.go b/agreement/service.go index 9972d39947..22d6145499 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -224,7 +224,7 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r for { status.ConsensusVersion, err = s.Ledger.ConsensusVersion(status.Round.SubSaturate(2)) if err != nil { - s.Panicf("cannot read latest concensus version, round %d: %v", status.Round.SubSaturate(2), err) + s.Panicf("cannot read latest consensus version, round %d: %v", status.Round.SubSaturate(2), err) } // set speculative block assembly based on the current local configuration diff --git a/agreement/types.go b/agreement/types.go index 2d11ded305..ace6dfcb46 100644 --- a/agreement/types.go +++ b/agreement/types.go @@ -38,6 +38,7 @@ func FilterTimeout(p period, v protocol.ConsensusVersion) time.Duration { return config.Consensus[v].AgreementFilterTimeout } +// SpeculativeBlockAsmTime is the time at which we would like to begin speculative assembly func SpeculativeBlockAsmTime(p period, v protocol.ConsensusVersion, speculativeAsmTimeDuration time.Duration) time.Duration { hardwait := FilterTimeout(p, v) if hardwait > speculativeAsmTimeDuration { diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 66eaf98c18..074859c392 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -562,6 +562,7 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } +// OnNewSpeculativeBlock handles creating a speculative block func (pool *TransactionPool) OnNewSpeculativeBlock(ctx context.Context, vb *ledgercore.ValidatedBlock) { pool.mu.Lock() diff --git a/ledger/ledgercore/validatedBlock.go b/ledger/ledgercore/validatedBlock.go index 400fa3ea15..84d55fb9ae 100644 --- a/ledger/ledgercore/validatedBlock.go +++ b/ledger/ledgercore/validatedBlock.go @@ -53,6 +53,7 @@ func (vb ValidatedBlock) WithSeed(s committee.Seed) ValidatedBlock { } } +// CheckDup checks whether a txn is a duplicate func (vb ValidatedBlock) CheckDup(currentProto config.ConsensusParams, firstValid, lastValid basics.Round, txid transactions.Txid, txl Txlease) error { _, present := vb.delta.Txids[txid] if present { @@ -68,6 +69,7 @@ func (vb ValidatedBlock) CheckDup(currentProto config.ConsensusParams, firstVali return nil } +// Hash returns the hash of the block func (vb ValidatedBlock) Hash() bookkeeping.BlockHash { return vb.blk.Hash() } diff --git a/ledger/speculative.go b/ledger/speculative.go index cb42f36ab3..809d6c6956 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -30,6 +30,7 @@ import ( "github.com/algorand/go-algorand/logging" ) +// LedgerForEvaluator defines the ledger interface needed by the evaluator. type LedgerForEvaluator interface { // Needed for cow.go Block(basics.Round) (bookkeeping.Block, error) diff --git a/node/node.go b/node/node.go index e57faed3f0..eb71ca85e7 100644 --- a/node/node.go +++ b/node/node.go @@ -1264,6 +1264,7 @@ func (node *AlgorandFullNode) AssembleBlock(round basics.Round) (agreement.Valid return validatedBlock{vb: lvb}, nil } +// OnNewSpeculativeBlock handles creating a speculative block func (node *AlgorandFullNode) OnNewSpeculativeBlock(ctx context.Context, avb agreement.ValidatedBlock) { vb, ok := avb.(validatedBlock) if ok { From 4bc1194c0f1835fad529774c2ef36bd80b8d4e24 Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy Date: Wed, 14 Dec 2022 19:29:25 -0500 Subject: [PATCH 47/50] fix linter and msgp --- agreement/msgp_gen.go | 59 +++++++++++++++++++++++++++++++++++--- config/localTemplate.go | 2 +- ledger/speculative.go | 4 +-- ledger/speculative_test.go | 2 +- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 3c396226b8..fe456df8c0 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3552,9 +3552,12 @@ func (z *periodRouter) MsgIsZero() bool { // MarshalMsg implements msgp.Marshaler func (z *player) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - // map header, size 8 + // map header, size 11 + // string "ConsensusVersion" + o = append(o, 0x8b, 0xb0, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = (*z).ConsensusVersion.MarshalMsg(o) // string "Deadline" - o = append(o, 0x88, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) + o = append(o, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) o = msgp.AppendDuration(o, (*z).Deadline) // string "FastRecoveryDeadline" o = append(o, 0xb4, 0x46, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) @@ -3574,6 +3577,12 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { // string "Round" o = append(o, 0xa5, 0x52, 0x6f, 0x75, 0x6e, 0x64) o = (*z).Round.MarshalMsg(o) + // string "SpeculativeAsmTimeDuration" + o = append(o, 0xba, 0x53, 0x70, 0x65, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x73, 0x6d, 0x54, 0x69, 0x6d, 0x65, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e) + o = msgp.AppendDuration(o, (*z).SpeculativeAsmTimeDuration) + // string "SpeculativeAssemblyDeadline" + o = append(o, 0xbb, 0x53, 0x70, 0x65, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x73, 0x73, 0x65, 0x6d, 0x62, 0x6c, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) + o = msgp.AppendDuration(o, (*z).SpeculativeAssemblyDeadline) // string "Step" o = append(o, 0xa4, 0x53, 0x74, 0x65, 0x70) o = msgp.AppendUint64(o, uint64((*z).Step)) @@ -3666,6 +3675,14 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + (*z).SpeculativeAssemblyDeadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SpeculativeAssemblyDeadline") + return + } + } if zb0001 > 0 { zb0001-- bts, err = (*z).Pending.UnmarshalMsg(bts) @@ -3674,6 +3691,22 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).ConsensusVersion.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "ConsensusVersion") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).SpeculativeAsmTimeDuration, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SpeculativeAsmTimeDuration") + return + } + } if zb0001 > 0 { err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { @@ -3751,12 +3784,30 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "FastRecoveryDeadline") return } + case "SpeculativeAssemblyDeadline": + (*z).SpeculativeAssemblyDeadline, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SpeculativeAssemblyDeadline") + return + } case "Pending": bts, err = (*z).Pending.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "Pending") return } + case "ConsensusVersion": + bts, err = (*z).ConsensusVersion.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ConsensusVersion") + return + } + case "SpeculativeAsmTimeDuration": + (*z).SpeculativeAsmTimeDuration, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SpeculativeAsmTimeDuration") + return + } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -3777,13 +3828,13 @@ func (_ *player) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *player) Msgsize() (s int) { - s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 28 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + 17 + (*z).ConsensusVersion.Msgsize() + 27 + msgp.DurationSize return } // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).SpeculativeAssemblyDeadline == 0) && ((*z).Pending.MsgIsZero()) && ((*z).ConsensusVersion.MsgIsZero()) && ((*z).SpeculativeAsmTimeDuration == 0) } // MarshalMsg implements msgp.Marshaler diff --git a/config/localTemplate.go b/config/localTemplate.go index 2a57bcddd8..75be20af66 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -467,7 +467,7 @@ type Local struct { // 0x01 (txFilterRawMsg) - check for raw tx message duplicates // 0x02 (txFilterCanonical) - check for canonical tx group duplicates TxIncomingFilteringFlags uint32 `version[26]:"1"` - + // SpeculativeBlockAssemblyGraceTime sets additional time, on top of // ProposalAssemblyTime, that this node allows for speculative block // assembly. diff --git a/ledger/speculative.go b/ledger/speculative.go index 0dbcbab04e..a67a4e5f2e 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Algorand, Inc. +// Copyright (C) 2019-2022 Algorand, Inc. // This file is part of go-algorand // // go-algorand is free software: you can redistribute it and/or modify @@ -31,7 +31,7 @@ import ( ) // LedgerForEvaluator defines the ledger interface needed by the evaluator. -type LedgerForEvaluator interface { +type LedgerForEvaluator interface { //nolint:revive //LedgerForEvaluator is a long established but newly leaking-out name, and there really isn't a better name for it despite how lint dislikes ledger.LedgerForEvaluator // Needed for cow.go Block(basics.Round) (bookkeeping.Block, error) BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go index f00edc1d45..607d1efcf4 100644 --- a/ledger/speculative_test.go +++ b/ledger/speculative_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Algorand, Inc. +// Copyright (C) 2019-2022 Algorand, Inc. // This file is part of go-algorand // // go-algorand is free software: you can redistribute it and/or modify From d42feb0fb7a9c27f64c7ddedec6674ceb24af32f Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Thu, 26 Jan 2023 10:10:47 -0500 Subject: [PATCH 48/50] performance: Speculative block assembly (#4861) * workaround interface changes * small tweaks, note TODOs * cleanup s/OnNewSpeculativeBlock/StartSpeculativeBlockAssembly/g rm player.SpeculativeAssemblyDeadline * simplify demux * test fixes, works at least -short * comment * TODO comments * expand StartSpeculativeBlockAssembly to take the block header digest * wip: pool rehacking passes tests * fix TestSpeculativeBlockAssemblyDataRace better locking in tryReadSpeculativeBlock() start thread in StartSpeculativeBlockAssembly() * fix * simplify SpeculativeBlockAssemblyGraceTime to SpeculativeAsmTimeOffset remove player.SpeculativeAsmTimeDuration field * improve speculative block assembly starting start spec asm at filterTimeout-offset and again at soft-vote cleanup * lint cleanup * nolint comment * fix copyright header * Post merge fixes * fix GetCreatorForRound which was breaking e2e test * make generate * review comment * SpeculativeAsmTimeOffset typo, 400ms * algod_speculative_assembly_discarded metric * start speculative assembly on proposal validation * StartSpeculativeBlockAssembly(...,onlyIfStarted) * algod_speculative_assembly_discarded_{start,read,ONB} * code review feedback * fix a test by ignoring intermittent speculative block action Co-authored-by: Pavel Zbitskiy --- agreement/abstractions.go | 2 +- agreement/actions.go | 9 +- agreement/actiontype_string.go | 7 +- agreement/agreeInstall.go | 2 +- agreement/agreementtest/simulate_test.go | 3 +- agreement/common_test.go | 2 +- agreement/coservice.go | 12 +- agreement/demux.go | 19 +- agreement/demux_test.go | 6 +- agreement/fuzzer/ledger_test.go | 3 +- agreement/msgp_gen.go | 42 +--- agreement/persistence_test.go | 13 +- agreement/player.go | 62 ++--- agreement/player_test.go | 41 +++- agreement/pseudonode.go | 8 +- agreement/service.go | 11 +- agreement/service_test.go | 223 ++++++++++-------- agreement/types.go | 6 +- config/localTemplate.go | 9 +- config/local_defaults.go | 5 +- data/account/rootInstall.go | 2 +- data/datatest/impls.go | 3 +- data/pools/transactionPool.go | 189 +++++++++------ data/pools/transactionPool_test.go | 41 ++-- installer/config.json.example | 3 +- ledger/speculative.go | 82 ++++--- logging/logspec/agreement.go | 1 + node/node.go | 8 +- test/scripts/e2e_client_runner.py | 10 +- .../e2e_subs/e2e-app-real-assets-round.sh | 23 +- .../e2e_subs/tealprogs/assetround.teal | 26 +- test/testdata/configs/config-v27.json | 2 +- 32 files changed, 516 insertions(+), 359 deletions(-) diff --git a/agreement/abstractions.go b/agreement/abstractions.go index e3e3a9870b..3b4478c411 100644 --- a/agreement/abstractions.go +++ b/agreement/abstractions.go @@ -86,7 +86,7 @@ type BlockFactory interface { // lose liveness. AssembleBlock(basics.Round) (ValidatedBlock, error) - OnNewSpeculativeBlock(context.Context, ValidatedBlock) + StartSpeculativeBlockAssembly(context.Context, ValidatedBlock, crypto.Digest, bool) } // A Ledger represents the sequence of Entries agreed upon by the protocol. diff --git a/agreement/actions.go b/agreement/actions.go index cdb84746b2..c5801311cf 100644 --- a/agreement/actions.go +++ b/agreement/actions.go @@ -55,6 +55,7 @@ const ( assemble repropose speculativeAssembly + speculativeAssemblyIfStarted // disk checkpoint @@ -310,7 +311,7 @@ func (a rezeroAction) do(ctx context.Context, s *Service) { } type pseudonodeAction struct { - // assemble, repropose, attest, speculativeAssembly + // assemble, repropose, attest, speculativeAssembly, speculativeAssemblyIfStarted T actionType Round round @@ -349,7 +350,9 @@ func (a pseudonodeAction) do(ctx context.Context, s *Service) { s.log.Errorf("pseudonode.MakeProposals call failed %v", err) } case speculativeAssembly: - s.loopback.StartSpeculativeBlockAssembly(ctx, a.ValidatedBlock) + s.loopback.StartSpeculativeBlockAssembly(ctx, a.ValidatedBlock, a.Proposal.BlockDigest, false) + case speculativeAssemblyIfStarted: + s.loopback.StartSpeculativeBlockAssembly(ctx, a.ValidatedBlock, a.Proposal.BlockDigest, true) case repropose: logEvent := logspec.AgreementEvent{ @@ -469,7 +472,7 @@ func zeroAction(t actionType) action { return ensureAction{} case rezero: return rezeroAction{} - case attest, assemble, repropose: + case attest, assemble, repropose, speculativeAssembly, speculativeAssemblyIfStarted: return pseudonodeAction{} case checkpoint: return checkpointAction{} diff --git a/agreement/actiontype_string.go b/agreement/actiontype_string.go index b127acbd2b..e6cee56ece 100644 --- a/agreement/actiontype_string.go +++ b/agreement/actiontype_string.go @@ -24,12 +24,13 @@ func _() { _ = x[assemble-13] _ = x[repropose-14] _ = x[speculativeAssembly-15] - _ = x[checkpoint-16] + _ = x[speculativeAssemblyIfStarted-16] + _ = x[checkpoint-17] } -const _actionType_name = "noopignorebroadcastrelaydisconnectbroadcastVotesverifyVoteverifyPayloadverifyBundleensurestageDigestrezeroattestassemblereproposespeculativeAssemblycheckpoint" +const _actionType_name = "noopignorebroadcastrelaydisconnectbroadcastVotesverifyVoteverifyPayloadverifyBundleensurestageDigestrezeroattestassemblereproposespeculativeAssemblyspeculativeAssemblyIfStartedcheckpoint" -var _actionType_index = [...]uint8{0, 4, 10, 19, 24, 34, 48, 58, 71, 83, 89, 100, 106, 112, 120, 129, 148, 158} +var _actionType_index = [...]uint8{0, 4, 10, 19, 24, 34, 48, 58, 71, 83, 89, 100, 106, 112, 120, 129, 148, 176, 186} func (i actionType) String() string { if i >= actionType(len(_actionType_index)-1) { diff --git a/agreement/agreeInstall.go b/agreement/agreeInstall.go index 1028eedb99..28b1985b22 100644 --- a/agreement/agreeInstall.go +++ b/agreement/agreeInstall.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2022 Algorand, Inc. +// Copyright (C) 2019-2023 Algorand, Inc. // This file is part of go-algorand // // go-algorand is free software: you can redistribute it and/or modify diff --git a/agreement/agreementtest/simulate_test.go b/agreement/agreementtest/simulate_test.go index a50e052a42..67f05c169f 100644 --- a/agreement/agreementtest/simulate_test.go +++ b/agreement/agreementtest/simulate_test.go @@ -96,7 +96,8 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (agreement.ValidatedBloc return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } -func (f testBlockFactory) OnNewSpeculativeBlock(context.Context, agreement.ValidatedBlock) {} +func (f testBlockFactory) StartSpeculativeBlockAssembly(context.Context, agreement.ValidatedBlock, crypto.Digest, bool) { +} // If we try to read from high rounds, we panic and do not emit an error to find bugs during testing. type testLedger struct { diff --git a/agreement/common_test.go b/agreement/common_test.go index 2cb6ea3227..28e2ad5ce7 100644 --- a/agreement/common_test.go +++ b/agreement/common_test.go @@ -183,7 +183,7 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (ValidatedBlock, error) return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } -func (f testBlockFactory) OnNewSpeculativeBlock(context.Context, ValidatedBlock) { +func (f testBlockFactory) StartSpeculativeBlockAssembly(context.Context, ValidatedBlock, crypto.Digest, bool) { return } diff --git a/agreement/coservice.go b/agreement/coservice.go index 0126cb85d3..4fa9514d80 100644 --- a/agreement/coservice.go +++ b/agreement/coservice.go @@ -17,9 +17,10 @@ package agreement import ( - "github.com/algorand/go-deadlock" + "runtime" + "testing" - "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-deadlock" ) //go:generate stringer -type=coserviceType @@ -35,10 +36,12 @@ const ( //msgp:ignore coserviceType type coserviceType int +// coserviceMonitor is unit test instrumentation type coserviceMonitor struct { deadlock.Mutex id int + t *testing.T c map[coserviceType]uint coserviceListener @@ -79,7 +82,10 @@ func (m *coserviceMonitor) dec(t coserviceType) { m.c = make(map[coserviceType]uint) } if m.c[t] == 0 { - logging.Base().Panicf("%d: tried to decrement empty coservice queue %v", m.id, t) + var stack [1000]byte + sl := runtime.Stack(stack[:], false) + m.t.Log(string(stack[:sl])) + m.t.Fatalf("%d: tried to decrement empty coservice queue %v", m.id, t) } m.c[t]-- diff --git a/agreement/demux.go b/agreement/demux.go index 309b3de258..8909054106 100644 --- a/agreement/demux.go +++ b/agreement/demux.go @@ -60,10 +60,13 @@ type demux struct { queue []<-chan externalEvent processingMonitor EventsProcessingMonitor - monitor *coserviceMonitor cancelTokenizers context.CancelFunc log logging.Logger + + // coserviceMonitor is unit test instrumentation. + // should be fast no-op if monitor == nil + monitor *coserviceMonitor } // demuxParams contains the parameters required to initliaze a new demux object @@ -74,7 +77,10 @@ type demuxParams struct { voteVerifier *AsyncVoteVerifier processingMonitor EventsProcessingMonitor log logging.Logger - monitor *coserviceMonitor + + // coserviceMonitor is unit test instrumentation. + // should be fast no-op if monitor == nil + monitor *coserviceMonitor } // makeDemux initializes the goroutines needed to process external events, setting up the appropriate channels. @@ -253,12 +259,14 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat deadlineCh := s.Clock.TimeoutAt(deadline) fastDeadlineCh := s.Clock.TimeoutAt(fastDeadline) - speculationDeadlineCh := s.Clock.TimeoutAt(speculationDeadline) + var speculationDeadlineCh <-chan time.Time // zero timeout means we don't have enough time to speculate on block assembly - if speculationDeadline == 0 { - speculationDeadlineCh = nil + if speculationDeadline != 0 { + speculationDeadlineCh = s.Clock.TimeoutAt(speculationDeadline) } + //d.log.Infof("demux deadline %d, fastD %d, specD %d, d.monitor %v", deadline, fastDeadline, speculationDeadline, d.monitor) // not threadsafe in some tests + d.UpdateEventsQueue(eventQueueDemux, 0) d.monitor.dec(demuxCoserviceType) @@ -369,6 +377,7 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat } // setupCompoundMessage processes compound messages: distinct messages which are delivered together +// TODO: does this ever really see something other than empty .Vote? func setupCompoundMessage(l LedgerReader, m message) (res externalEvent) { compound := m.CompoundMessage if compound.Vote == (unauthenticatedVote{}) { diff --git a/agreement/demux_test.go b/agreement/demux_test.go index a37f622028..4589e2f630 100644 --- a/agreement/demux_test.go +++ b/agreement/demux_test.go @@ -69,6 +69,7 @@ type demuxTestUsecase struct { e event ok bool speculativeAsmTime time.Duration + desc string } var demuxTestUsecases = []demuxTestUsecase{ @@ -197,6 +198,7 @@ var demuxTestUsecases = []demuxTestUsecase{ verifiedBundle: testChanState{eventCount: 0, closed: false}, e: messageEvent{T: votePresent}, ok: true, + desc: "one vote one prop", }, { queue: []testChanState{}, @@ -413,6 +415,7 @@ var demuxTestUsecases = []demuxTestUsecase{ verifiedBundle: testChanState{eventCount: 0, closed: false}, e: messageEvent{T: votePresent}, ok: true, + desc: "one prop", }, { queue: []testChanState{{eventCount: 0, closed: false}}, @@ -677,6 +680,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { dmx := &demux{} + dmx.log = logging.TestingLog(t) dmx.crypto = t dmx.ledger = t dmx.rawVotes = t.makeRawChannel(protocol.AgreementVoteTag, testcase.rawVotes, false) @@ -704,7 +708,7 @@ func (t *demuxTester) TestUsecase(testcase demuxTestUsecase) bool { return false } - if !assert.Equalf(t, strings.Replace(testcase.e.String(), "{test_index}", fmt.Sprintf("%d", t.testIdx), 1), e.String(), "Test case %d failed.", t.testIdx+1) { + if !assert.Equalf(t, strings.Replace(testcase.e.String(), "{test_index}", fmt.Sprintf("%d", t.testIdx), 1), e.String(), "Test case %d (%s) failed.", t.testIdx+1, testcase.desc) { return false } diff --git a/agreement/fuzzer/ledger_test.go b/agreement/fuzzer/ledger_test.go index d2b2059456..7332c91cdc 100644 --- a/agreement/fuzzer/ledger_test.go +++ b/agreement/fuzzer/ledger_test.go @@ -112,7 +112,8 @@ func (f testBlockFactory) AssembleBlock(r basics.Round) (agreement.ValidatedBloc return testValidatedBlock{Inside: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: r}}}, nil } -func (f testBlockFactory) OnNewSpeculativeBlock(context.Context, agreement.ValidatedBlock) {} +func (f testBlockFactory) StartSpeculativeBlockAssembly(context.Context, agreement.ValidatedBlock, crypto.Digest, bool) { +} type testLedgerSyncFunc func(l *testLedger, r basics.Round, c agreement.Certificate) bool diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index fe456df8c0..38caefdff2 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3552,9 +3552,9 @@ func (z *periodRouter) MsgIsZero() bool { // MarshalMsg implements msgp.Marshaler func (z *player) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - // map header, size 11 + // map header, size 9 // string "ConsensusVersion" - o = append(o, 0x8b, 0xb0, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = append(o, 0x89, 0xb0, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) o = (*z).ConsensusVersion.MarshalMsg(o) // string "Deadline" o = append(o, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) @@ -3577,12 +3577,6 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { // string "Round" o = append(o, 0xa5, 0x52, 0x6f, 0x75, 0x6e, 0x64) o = (*z).Round.MarshalMsg(o) - // string "SpeculativeAsmTimeDuration" - o = append(o, 0xba, 0x53, 0x70, 0x65, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x73, 0x6d, 0x54, 0x69, 0x6d, 0x65, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e) - o = msgp.AppendDuration(o, (*z).SpeculativeAsmTimeDuration) - // string "SpeculativeAssemblyDeadline" - o = append(o, 0xbb, 0x53, 0x70, 0x65, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x73, 0x73, 0x65, 0x6d, 0x62, 0x6c, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = msgp.AppendDuration(o, (*z).SpeculativeAssemblyDeadline) // string "Step" o = append(o, 0xa4, 0x53, 0x74, 0x65, 0x70) o = msgp.AppendUint64(o, uint64((*z).Step)) @@ -3675,14 +3669,6 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } - if zb0001 > 0 { - zb0001-- - (*z).SpeculativeAssemblyDeadline, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "SpeculativeAssemblyDeadline") - return - } - } if zb0001 > 0 { zb0001-- bts, err = (*z).Pending.UnmarshalMsg(bts) @@ -3699,14 +3685,6 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } - if zb0001 > 0 { - zb0001-- - (*z).SpeculativeAsmTimeDuration, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "SpeculativeAsmTimeDuration") - return - } - } if zb0001 > 0 { err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { @@ -3784,12 +3762,6 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "FastRecoveryDeadline") return } - case "SpeculativeAssemblyDeadline": - (*z).SpeculativeAssemblyDeadline, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "SpeculativeAssemblyDeadline") - return - } case "Pending": bts, err = (*z).Pending.UnmarshalMsg(bts) if err != nil { @@ -3802,12 +3774,6 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "ConsensusVersion") return } - case "SpeculativeAsmTimeDuration": - (*z).SpeculativeAsmTimeDuration, bts, err = msgp.ReadDurationBytes(bts) - if err != nil { - err = msgp.WrapError(err, "SpeculativeAsmTimeDuration") - return - } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -3828,13 +3794,13 @@ func (_ *player) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *player) Msgsize() (s int) { - s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 28 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + 17 + (*z).ConsensusVersion.Msgsize() + 27 + msgp.DurationSize + s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + 17 + (*z).ConsensusVersion.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).SpeculativeAssemblyDeadline == 0) && ((*z).Pending.MsgIsZero()) && ((*z).ConsensusVersion.MsgIsZero()) && ((*z).SpeculativeAsmTimeDuration == 0) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) && ((*z).ConsensusVersion.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler diff --git a/agreement/persistence_test.go b/agreement/persistence_test.go index 7a4ec3db4d..d866bb44f2 100644 --- a/agreement/persistence_test.go +++ b/agreement/persistence_test.go @@ -183,15 +183,18 @@ func randomizeDiskState() (rr rootRouter, p player) { func TestRandomizedEncodingFullDiskState(t *testing.T) { partitiontest.PartitionTest(t) for i := 0; i < 5000; i++ { + if i%100 == 0 { + t.Logf("i=%d", i) + } router, player := randomizeDiskState() a := []action{} clock := timers.MakeMonotonicClock(time.Date(2015, 1, 2, 5, 6, 7, 8, time.UTC)) log := makeServiceLogger(logging.Base()) - e1 := encode(clock, router, player, a, true) - e2 := encode(clock, router, player, a, false) - require.Equalf(t, e1, e2, "msgp and go-codec encodings differ: len(msgp)=%v, len(reflect)=%v", len(e1), len(e2)) - _, rr1, p1, _, err1 := decode(e1, clock, log, true) - _, rr2, p2, _, err2 := decode(e1, clock, log, false) + eReflect := encode(clock, router, player, a, true) // reflect=true + eMsgp := encode(clock, router, player, a, false) + require.Equalf(t, eMsgp, eReflect, "msgp and go-codec encodings differ: len(msgp)=%v, len(reflect)=%v", len(eMsgp), len(eReflect)) + _, rr1, p1, _, err1 := decode(eReflect, clock, log, true) + _, rr2, p2, _, err2 := decode(eReflect, clock, log, false) require.NoErrorf(t, err1, "reflect decoding failed") require.NoErrorf(t, err2, "msgp decoding failed") require.Equalf(t, rr1, rr2, "rootRouters decoded differently") diff --git a/agreement/player.go b/agreement/player.go index 77db28a0a0..a90a37235c 100644 --- a/agreement/player.go +++ b/agreement/player.go @@ -49,19 +49,12 @@ type player struct { // partition recovery. FastRecoveryDeadline time.Duration - // SpeculativeAssemblyDeadline contains the next timeout expected for - // speculative block assembly. - SpeculativeAssemblyDeadline time.Duration - // Pending holds the player's proposalTable, which stores proposals that // must be verified after some vote has been verified. Pending proposalTable // the current consensus version ConsensusVersion protocol.ConsensusVersion - - // the time offset before the filter deadline to start speculative block assembly - SpeculativeAsmTimeDuration time.Duration } func (p *player) T() stateMachineTag { @@ -137,23 +130,14 @@ func (p *player) handle(r routerHandle, e event) []action { } } +// handleSpeculationTimeout TODO: rename this 'timeout' is the START of speculative assembly. func (p *player) handleSpeculationTimeout(r routerHandle, e timeoutEvent) []action { - p.SpeculativeAssemblyDeadline = 0 if e.Proto.Err != nil { r.t.log.Errorf("failed to read protocol version for speculationTimeout event (proto %v): %v", e.Proto.Version, e.Proto.Err) return nil } - // get the best proposal we have - re := readLowestEvent{T: readLowestPayload, Round: p.Round, Period: p.Period} - re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) - - // if we have its payload and its been validated already, start speculating - // on top of it - if re.PayloadOK && re.Payload.ve != nil { - return p.startSpeculativeBlockAsm(r, re.Payload.ve) - } - return nil + return p.startSpeculativeBlockAsm(r, nil, false) } func (p *player) handleFastTimeout(r routerHandle, e timeoutEvent) []action { @@ -194,6 +178,7 @@ func (p *player) issueSoftVote(r routerHandle) (actions []action) { // If we arrive due to fast-forward/soft threshold; then answer.Bottom = false and answer.Proposal = bottom // and we should soft-vote normally (not based on the starting value) a.Proposal = nextStatus.Proposal + // TODO: how do we speculative block assemble based on nextStatus.Proposal? return append(actions, a) } @@ -210,8 +195,10 @@ func (p *player) issueSoftVote(r routerHandle) (actions []action) { return nil } - // original proposal: vote for it - return append(actions, a) + // original proposal: vote for it, maybe build + actions = append(actions, a) + actions = p.startSpeculativeBlockAsm(r, actions, false) + return actions } // A committableEvent is the trigger for issuing a cert vote. @@ -250,8 +237,25 @@ func (p *player) issueNextVote(r routerHandle) []action { return actions } -func (p *player) startSpeculativeBlockAsm(r routerHandle, ve ValidatedBlock) (actions []action) { - return append(actions, pseudonodeAction{T: speculativeAssembly, Round: p.Round, Period: p.Period, ValidatedBlock: ve}) +func (p *player) startSpeculativeBlockAsm(r routerHandle, actions []action, onlyIfStarted bool) []action { + if p.Period != 0 { + // If not period 0, cautiously do a simpler protocol. + return actions + } + // get the best proposal we have + re := readLowestEvent{T: readLowestPayload, Round: p.Round, Period: p.Period} + re = r.dispatch(*p, re, proposalMachineRound, p.Round, p.Period, 0).(readLowestEvent) + + // if we have its payload and its been validated already, start speculating on top of it + if re.PayloadOK && re.Payload.ve != nil { + a := pseudonodeAction{T: speculativeAssembly, Round: p.Round, Period: p.Period, ValidatedBlock: re.Payload.ve, Proposal: re.Proposal} + if onlyIfStarted { + // only re-start speculation if we had already started it but got a better block + a.T = speculativeAssemblyIfStarted + } + return append(actions, a) + } + return actions } func (p *player) issueFastVote(r routerHandle) (actions []action) { @@ -365,12 +369,6 @@ func (p *player) enterPeriod(r routerHandle, source thresholdEvent, target perio p.Napping = false p.FastRecoveryDeadline = 0 // set immediately p.Deadline = FilterTimeout(target, source.Proto) - if target == 0 { - p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(target, p.ConsensusVersion, p.SpeculativeAsmTimeDuration) - } else { - // only speculate on block assembly in period 0 - p.SpeculativeAssemblyDeadline = 0 - } // update tracer state to match player r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step}) @@ -415,7 +413,6 @@ func (p *player) enterRound(r routerHandle, source event, target round) []action p.Step = soft p.Napping = false p.FastRecoveryDeadline = 0 // set immediately - p.SpeculativeAssemblyDeadline = SpeculativeBlockAsmTime(0, p.ConsensusVersion, p.SpeculativeAsmTimeDuration) switch source := source.(type) { case roundInterruptionEvent: @@ -629,6 +626,13 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a actions = append(actions, a) } + // StartSpeculativeBlockAssembly every time we validate a proposal + // TODO: maybe only do this if after speculation has started; interrupt speculation on a block when we get a better block + // TODO: maybe don't do this at all and just delete it? + if ef.t() == payloadAccepted { + actions = p.startSpeculativeBlockAsm(r, actions, true) + } + // If the payload is valid, check it against any received cert threshold. // Of course, this should only trigger for payloadVerified case. // This allows us to handle late payloads (relative to cert-bundles, i.e., certificates) without resorting to catchup. diff --git a/agreement/player_test.go b/agreement/player_test.go index 3e3cf8167b..b20a64d4c2 100644 --- a/agreement/player_test.go +++ b/agreement/player_test.go @@ -113,13 +113,28 @@ func simulateProposalPayloads(t *testing.T, router *rootRouter, player *player, } } +// most player code returns a fixed number of actions, but some player code sometimes adds an exctra speculative block assembly action that we filter out to make tests simpler +func ignoreSpeculativeAssembly(a []action) []action { + var out []action + for _, ia := range a { + switch ia.t() { + case speculativeAssembly, speculativeAssemblyIfStarted: + // do not copy out + default: + out = append(out, ia) + } + } + return out +} + func simulateProposals(t *testing.T, router *rootRouter, player *player, voteBatch []event, payloadBatch []event) { for i, e := range voteBatch { var res []action *player, res = router.submitTop(&playerTracer, *player, e) - earlier := res + earlier := ignoreSpeculativeAssembly(res) *player, res = router.submitTop(&playerTracer, *player, payloadBatch[i]) + res = ignoreSpeculativeAssembly(res) if len(res) != len(earlier) { panic("proposal action mismatch") } @@ -135,13 +150,25 @@ func simulateTimeoutExpectSoft(t *testing.T, router *rootRouter, player *player, var res []action e := makeTimeoutEvent() *player, res = router.submitTop(&playerTracer, *player, e) - if len(res) != 1 { - panic("wrong number of actions") + // one or two actions, one must be attest, other _may_ be speculativeAssembly + require.True(t, 1 <= len(res) && len(res) <= 2, "wrong number of actions") + + hasAttest := false + var a1x action + + for _, ax := range res { + switch ax.t() { + case attest: + hasAttest = true + a1x = ax + case speculativeAssembly: + // ok + default: + t.Fatalf("bad action type: %v", ax.t()) + } } - - a1x := res[0] - if a1x.t() != attest { - panic("action 1 is not attest") + if !hasAttest { + panic("missing attest") } a1 := a1x.(pseudonodeAction) diff --git a/agreement/pseudonode.go b/agreement/pseudonode.go index f7d223e9c0..ec20d526b4 100644 --- a/agreement/pseudonode.go +++ b/agreement/pseudonode.go @@ -23,6 +23,7 @@ import ( "sync" "time" + "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/account" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/logging" @@ -61,7 +62,7 @@ type pseudonode interface { MakeProposals(ctx context.Context, r round, p period) (<-chan externalEvent, error) // StartSpeculativeBlockAssembly calls the BlockFactory's StartSpeculativeBlockAssembly on a new go routine. - StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock) + StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock, blockHash crypto.Digest, onlyIfStarted bool) // MakeVotes returns a vote for a given proposal in some round, period, and step. // @@ -188,8 +189,9 @@ func (n asyncPseudonode) MakeProposals(ctx context.Context, r round, p period) ( } } -func (n asyncPseudonode) StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock) { - go n.factory.OnNewSpeculativeBlock(ctx, ve) +func (n asyncPseudonode) StartSpeculativeBlockAssembly(ctx context.Context, ve ValidatedBlock, blockHash crypto.Digest, onlyIfStarted bool) { + // TODO: make this synchronous instead of thread? (thread creation likely moving to inside transactionPool) + go n.factory.StartSpeculativeBlockAssembly(ctx, ve, blockHash, onlyIfStarted) } func (n asyncPseudonode) MakeVotes(ctx context.Context, r round, p period, s step, prop proposalValue, persistStateDone chan error) (chan externalEvent, error) { diff --git a/agreement/service.go b/agreement/service.go index 0bb4a3ec59..21c859faf6 100644 --- a/agreement/service.go +++ b/agreement/service.go @@ -144,6 +144,7 @@ func (s *Service) Start() { input := make(chan externalEvent) output := make(chan []action) ready := make(chan externalDemuxSignals) + // TODO: use demuxOne() inside of mainLoop() instead of this nonsense pair of threads go s.demuxLoop(ctx, input, output, ready) go s.mainLoop(input, output, ready) } @@ -176,6 +177,12 @@ func (s *Service) demuxLoop(ctx context.Context, input chan<- externalEvent, out close(s.done) } +// TODO: use demuxOne() inside mainLoop() instead of having a pair of synchronous go threads trading off via chan +func (s *Service) demuxOne(ctx context.Context, a []action, extSignals externalDemuxSignals) (e externalEvent, ok bool) { + s.do(ctx, a) + return s.demux.next(s, extSignals.Deadline, extSignals.FastRecoveryDeadline, extSignals.SpeculativeBlockAsmDeadline, extSignals.CurrentRound) +} + // mainLoop drives the state machine. // // After possibly restoring from disk and then initializing, it does the following in a loop: @@ -228,9 +235,9 @@ func (s *Service) mainLoop(input <-chan externalEvent, output chan<- []action, r } // set speculative block assembly based on the current local configuration - status.SpeculativeAsmTimeDuration = s.parameters.Local.ProposalAssemblyTime + s.parameters.Local.SpeculativeBlockAssemblyGraceTime - specClock := SpeculativeBlockAsmTime(status.Period, status.ConsensusVersion, status.SpeculativeAsmTimeDuration) + specClock := SpeculativeBlockAsmTime(status.Period, status.ConsensusVersion, s.parameters.Local.SpeculativeAsmTimeOffset) + // TODO: e, ok := s.demuxOne(ctx, a, externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, SpeculativeBlockAsmDeadline: specClock, CurrentRound: status.Round}) output <- a ready <- externalDemuxSignals{Deadline: status.Deadline, FastRecoveryDeadline: status.FastRecoveryDeadline, SpeculativeBlockAsmDeadline: specClock, CurrentRound: status.Round} e, ok := <-input diff --git a/agreement/service_test.go b/agreement/service_test.go index b5ec2ce613..685b8e3033 100644 --- a/agreement/service_test.go +++ b/agreement/service_test.go @@ -150,7 +150,7 @@ type testingNetworkEndpoint struct { type nodeID int // bufferCapacity is per channel -func makeTestingNetwork(nodes int, bufferCapacity int, validator BlockValidator) *testingNetwork { +func makeTestingNetwork(t *testing.T, nodes int, bufferCapacity int, validator BlockValidator) *testingNetwork { n := new(testingNetwork) n.validator = validator @@ -168,6 +168,7 @@ func makeTestingNetwork(nodes int, bufferCapacity int, validator BlockValidator) m := new(coserviceMonitor) m.id = i + m.t = t n.monitors[nodeID(i)] = m } @@ -504,10 +505,13 @@ type activityMonitor struct { quiet chan struct{} cb func(nodeID, map[coserviceType]uint) + + t *testing.T } -func makeActivityMonitor() (m *activityMonitor) { +func makeActivityMonitor(t *testing.T) (m *activityMonitor) { m = new(activityMonitor) + m.t = t m.sums = make(map[nodeID]uint) m.listeners = make(map[nodeID]coserviceListener) m.activity = make(chan struct{}, 1000) @@ -546,17 +550,29 @@ func (m *activityMonitor) waitForActivity() { } func (m *activityMonitor) waitForQuiet() { + if m.t.Failed() { + return + } + dt := time.NewTicker(100 * time.Millisecond) + defer dt.Stop() + ticksRemaining := 100 select { case <-m.quiet: - case <-time.After(10 * time.Second): - m.dump() + case <-dt.C: + if m.t.Failed() { + return + } + if ticksRemaining <= 0 { + m.dump() - var buf [1000000]byte - n := runtime.Stack(buf[:], true) - fmt.Println("Printing goroutine dump of size", n) - fmt.Println(string(buf[:n])) + var buf [1000000]byte + n := runtime.Stack(buf[:], true) + fmt.Println("Printing goroutine dump of size", n) + fmt.Println(string(buf[:n])) - panic("timed out waiting for quiet...") + panic("timed out waiting for quiet...") + } + ticksRemaining-- } } @@ -705,11 +721,7 @@ func setupAgreementWithValidator(t *testing.T, numNodes int, traceLevel traceLev accounts, balances := createTestAccountsAndBalances(t, numNodes, (&[32]byte{})[:]) baseLedger := ledgerFactory(balances) - // logging - log := logging.Base() - f, _ := os.Create(t.Name() + ".log") - log.SetJSONFormatter() - log.SetOutput(f) + log := logging.TestingLog(t) log.SetLevel(logging.Debug) // node setup @@ -717,8 +729,8 @@ func setupAgreementWithValidator(t *testing.T, numNodes int, traceLevel traceLev ledgers := make([]Ledger, numNodes) dbAccessors := make([]db.Accessor, numNodes) services := make([]*Service, numNodes) - baseNetwork := makeTestingNetwork(numNodes, bufCap, validator) - am := makeActivityMonitor() + baseNetwork := makeTestingNetwork(t, numNodes, bufCap, validator) + am := makeActivityMonitor(t) for i := 0; i < numNodes; i++ { accessor, err := db.MakeAccessor(t.Name()+"_"+strconv.Itoa(i)+"_crash.db", false, true) @@ -744,7 +756,7 @@ func setupAgreementWithValidator(t *testing.T, numNodes int, traceLevel traceLev BlockFactory: testBlockFactory{Owner: i}, Clock: clocks[i], Accessor: accessor, - Local: config.Local{CadaverSizeTarget: 10000000}, + Local: config.Local{CadaverSizeTarget: 10000000, SpeculativeAsmTimeOffset: 9999999999}, RandomSource: &testingRand{}, } @@ -805,12 +817,11 @@ func (m *coserviceMonitor) clearClock() { } } -func expectNewPeriod(clocks []timers.Clock, zeroes uint) (newzeroes uint) { +func expectNewPeriod(t *testing.T, clocks []timers.Clock, zeroes uint) (newzeroes uint) { zeroes++ for i := range clocks { if clocks[i].(*testingClock).zeroes != zeroes { - errstr := fmt.Sprintf("unexpected number of zeroes: %v != %v", clocks[i].(*testingClock).zeroes, zeroes) - panic(errstr) + require.Equal(t, zeroes, clocks[i].(*testingClock).zeroes, "clocks[%d] unexpected number of zeroes %v != %v", i, zeroes, clocks[i].(*testingClock).zeroes) } } return zeroes @@ -837,9 +848,10 @@ func triggerGlobalTimeout(d time.Duration, clocks []timers.Clock, activityMonito activityMonitor.waitForQuiet() } -func runRound(clocks []timers.Clock, activityMonitor *activityMonitor, zeroes uint, filterTimeout time.Duration) (newzeroes uint) { +func runRound(t *testing.T, clocks []timers.Clock, activityMonitor *activityMonitor, zeroes uint, filterTimeout time.Duration) (newzeroes uint) { triggerGlobalTimeout(filterTimeout, clocks, activityMonitor) - return expectNewPeriod(clocks, zeroes) + t.Log("runRound a") + return expectNewPeriod(t, clocks, zeroes) } func sanityCheck(startRound round, numRounds round, ledgers []Ledger) { @@ -878,16 +890,21 @@ func simulateAgreementWithLedgerFactory(t *testing.T, numNodes int, numRounds in for i := 0; i < numNodes; i++ { services[i].Start() } + t.Logf("%d nodes started", numNodes) activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + t.Log("all quiet") + zeroes := expectNewPeriod(t, clocks, 0) + t.Logf("runRound [%d]clocks", len(clocks)) // run round with round-specific consensus version first (since fix in #1896) version, _ := baseLedger.ConsensusVersion(ParamsRound(startRound)) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + t.Logf("zeroes %d", zeroes) for j := 1; j < numRounds; j++ { + t.Logf("test round j=%d", j) version, _ := baseLedger.ConsensusVersion(ParamsRound(baseLedger.NextRound() + basics.Round(j-1))) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { @@ -1023,11 +1040,11 @@ func TestAgreementFastRecoveryDownEarly(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // force fast partition recovery into bottom @@ -1045,19 +1062,19 @@ func TestAgreementFastRecoveryDownEarly(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(firstFPR, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { @@ -1081,11 +1098,11 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // force fast partition recovery into bottom @@ -1126,19 +1143,19 @@ func TestAgreementFastRecoveryDownMiss(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(secondFPR, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { @@ -1162,11 +1179,11 @@ func TestAgreementFastRecoveryLate(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // force fast partition recovery into value @@ -1228,14 +1245,14 @@ func TestAgreementFastRecoveryLate(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(secondFPR, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // terminate on period 1 { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } for _, l := range ledgers { @@ -1251,7 +1268,7 @@ func TestAgreementFastRecoveryLate(t *testing.T) { // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { @@ -1275,11 +1292,11 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // force fast partition recovery into value @@ -1341,7 +1358,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(secondFPR, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // fail period 1 with value again @@ -1382,14 +1399,14 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(secondFPR, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // terminate on period 2 { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } for _, l := range ledgers { @@ -1405,7 +1422,7 @@ func TestAgreementFastRecoveryRedo(t *testing.T) { // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { @@ -1429,11 +1446,11 @@ func TestAgreementBlockReplayBug_b29ea57(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // fail period 0 @@ -1443,7 +1460,7 @@ func TestAgreementBlockReplayBug_b29ea57(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // fail period 1 on bottom with block @@ -1452,19 +1469,19 @@ func TestAgreementBlockReplayBug_b29ea57(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // terminate on period 2 { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { @@ -1488,11 +1505,11 @@ func TestAgreementLateCertBug(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // delay minority cert votes to force period 1 @@ -1505,7 +1522,7 @@ func TestAgreementLateCertBug(t *testing.T) { baseNetwork.repairAll() triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // terminate on period 0 in period 1 @@ -1517,12 +1534,12 @@ func TestAgreementLateCertBug(t *testing.T) { baseNetwork.finishAllMulticast() activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { @@ -1546,11 +1563,11 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // force partition recovery into value @@ -1581,7 +1598,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { } triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1608,7 +1625,7 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { } triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 5, int(zeroes)) } @@ -1617,13 +1634,13 @@ func TestAgreementRecoverGlobalStartingValue(t *testing.T) { { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 6, int(zeroes)) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { services[i].Shutdown() @@ -1646,11 +1663,11 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // force partition recovery into value. @@ -1686,7 +1703,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { return params }) triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1712,7 +1729,7 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { } } triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } @@ -1720,13 +1737,13 @@ func TestAgreementRecoverGlobalStartingValueBadProposal(t *testing.T) { { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 6, int(zeroes)) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { services[i].Shutdown() @@ -1749,11 +1766,11 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // force partition recovery into both bottom and value. one node enters bottom, the rest enter value @@ -1811,7 +1828,7 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { lower, upper := (next + 1).nextVoteRanges() delta := time.Duration(testingRand{}.Uint64() % uint64(upper-lower)) triggerGlobalTimeout(lower+delta, clocks[1:], activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 4, int(zeroes)) } @@ -1838,20 +1855,20 @@ func TestAgreementRecoverBothVAndBotQuorums(t *testing.T) { } triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // Finish in period 2 { baseNetwork.repairAll() triggerGlobalTimeout(FilterTimeout(2, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 6, int(zeroes)) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { services[i].Shutdown() @@ -1874,11 +1891,11 @@ func TestAgreementSlowPayloadsPreDeadline(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // run round and then start pocketing payloads @@ -1886,7 +1903,7 @@ func TestAgreementSlowPayloadsPreDeadline(t *testing.T) { closeFn := baseNetwork.pocketAllCompound(pocket) // (takes effect next round) { triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run round with late payload @@ -1904,12 +1921,12 @@ func TestAgreementSlowPayloadsPreDeadline(t *testing.T) { baseNetwork.finishAllMulticast() activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { services[i].Shutdown() @@ -1932,11 +1949,11 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // run round and then start pocketing payloads @@ -1944,7 +1961,7 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { closeFn := baseNetwork.pocketAllCompound(pocket) // (takes effect next round) { triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // force network into period 1 by delaying proposals @@ -1952,7 +1969,7 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // recover in period 1 @@ -1969,12 +1986,12 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) { zeroes = expectNoNewPeriod(clocks, zeroes) triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { services[i].Shutdown() @@ -1997,11 +2014,11 @@ func TestAgreementLargePeriods(t *testing.T) { activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // partition the network, run until period 60 @@ -2013,7 +2030,7 @@ func TestAgreementLargePeriods(t *testing.T) { baseNetwork.repairAll() triggerGlobalTimeout(deadlineTimeout, clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) require.Equal(t, 4+p, int(zeroes)) } } @@ -2021,12 +2038,12 @@ func TestAgreementLargePeriods(t *testing.T) { // terminate { triggerGlobalTimeout(FilterTimeout(60, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // run two more rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } for i := 0; i < numNodes; i++ { services[i].Shutdown() @@ -2095,11 +2112,11 @@ func TestAgreementRegression_WrongPeriodPayloadVerificationCancellation_8ba23942 } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds for j := 0; j < 2; j++ { - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) } // run round and then start pocketing payloads, suspending validation @@ -2108,7 +2125,7 @@ func TestAgreementRegression_WrongPeriodPayloadVerificationCancellation_8ba23942 closeFn := baseNetwork.pocketAllCompound(pocket0) // (takes effect next round) { triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) } // force network into period 1 by failing period 0, entering with bottom and no soft threshold (to prevent proposal value pinning) @@ -2201,15 +2218,15 @@ func TestAgreementRegression_WrongPeriodPayloadVerificationCancellation_8ba23942 } baseNetwork.finishAllMulticast() - zeroes = expectNewPeriod(clocks, zeroes) + zeroes = expectNewPeriod(t, clocks, zeroes) activityMonitor.waitForQuiet() // run two more rounds //for j := 0; j < 2; j++ { - // zeroes = runRound(clocks, activityMonitor, zeroes, period(1-j)) + // zeroes = runRound(t, clocks, activityMonitor, zeroes, period(1-j)) //} - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(1, version)) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(1, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) for i := 0; i < numNodes; i++ { services[i].Shutdown() @@ -2252,9 +2269,9 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { } activityMonitor.waitForActivity() activityMonitor.waitForQuiet() - zeroes := expectNewPeriod(clocks, 0) + zeroes := expectNewPeriod(t, clocks, 0) // run two rounds - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) // make sure relay does not see block proposal for round 3 baseNetwork.intercept(func(params multicastParams) multicastParams { if params.tag == protocol.ProposalPayloadTag { @@ -2285,7 +2302,7 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { return params }) - zeroes = runRound(clocks, activityMonitor, zeroes, FilterTimeout(0, version)) + zeroes = runRound(t, clocks, activityMonitor, zeroes, FilterTimeout(0, version)) // Round 3: // First partition the relay to prevent it from seeing certificate or block @@ -2309,7 +2326,7 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { }) // And with some hypothetical second relay the network achieves consensus on a certificate and block. triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor) - zeroes = expectNewPeriod(clocks[1:], zeroes) + zeroes = expectNewPeriod(t, clocks[1:], zeroes) require.Equal(t, uint(3), clocks[0].(*testingClock).zeroes) close(pocketCert) @@ -2328,7 +2345,7 @@ func TestAgreementCertificateDoesNotStallSingleRelay(t *testing.T) { // this relay must still relay initial messages. Note that payloads were already relayed with // the previous global timeout. triggerGlobalTimeout(FilterTimeout(0, version), clocks[1:], activityMonitor) - zeroes = expectNewPeriod(clocks[1:], zeroes) + zeroes = expectNewPeriod(t, clocks[1:], zeroes) require.Equal(t, uint(3), clocks[0].(*testingClock).zeroes) for i := 0; i < numNodes; i++ { diff --git a/agreement/types.go b/agreement/types.go index ace6dfcb46..fb65a1d846 100644 --- a/agreement/types.go +++ b/agreement/types.go @@ -39,10 +39,10 @@ func FilterTimeout(p period, v protocol.ConsensusVersion) time.Duration { } // SpeculativeBlockAsmTime is the time at which we would like to begin speculative assembly -func SpeculativeBlockAsmTime(p period, v protocol.ConsensusVersion, speculativeAsmTimeDuration time.Duration) time.Duration { +func SpeculativeBlockAsmTime(p period, v protocol.ConsensusVersion, speculativeAsmTimeOffset time.Duration) time.Duration { hardwait := FilterTimeout(p, v) - if hardwait > speculativeAsmTimeDuration { - return hardwait - speculativeAsmTimeDuration + if hardwait > speculativeAsmTimeOffset { + return hardwait - speculativeAsmTimeOffset } return time.Duration(0) } diff --git a/config/localTemplate.go b/config/localTemplate.go index 75be20af66..83768a06f0 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -468,10 +468,11 @@ type Local struct { // 0x02 (txFilterCanonical) - check for canonical tx group duplicates TxIncomingFilteringFlags uint32 `version[26]:"1"` - // SpeculativeBlockAssemblyGraceTime sets additional time, on top of - // ProposalAssemblyTime, that this node allows for speculative block - // assembly. - SpeculativeBlockAssemblyGraceTime time.Duration `version[27]:"50000000"` + // SpeculativeAsmTimeOffset defines when speculative block assembly first starts, nanoseconds before consensus AgreementFilterTimeoutPeriod0 or AgreementFilterTimeout + // A huge value (greater than either AgreementFilterTimeout) disables this event. + SpeculativeAsmTimeOffset time.Duration `version[27]:"400000000"` + + SpeculativeAssemblyDisable bool `version[27]:"false"` } // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers diff --git a/config/local_defaults.go b/config/local_defaults.go index 6c0aa5197c..fe5ce0c810 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2022 Algorand, Inc. +// Copyright (C) 2019-2023 Algorand, Inc. // This file is part of go-algorand // // go-algorand is free software: you can redistribute it and/or modify @@ -112,7 +112,8 @@ var defaultLocal = Local{ RestReadTimeoutSeconds: 15, RestWriteTimeoutSeconds: 120, RunHosted: false, - SpeculativeBlockAssemblyGraceTime: 50000000, + SpeculativeAsmTimeOffset: 400000000, + SpeculativeAssemblyDisable: false, SuggestedFeeBlockHistory: 3, SuggestedFeeSlidingWindowSize: 50, TLSCertFile: "", diff --git a/data/account/rootInstall.go b/data/account/rootInstall.go index 4bf9a73e29..50f1c6e972 100644 --- a/data/account/rootInstall.go +++ b/data/account/rootInstall.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2022 Algorand, Inc. +// Copyright (C) 2019-2023 Algorand, Inc. // This file is part of go-algorand // // go-algorand is free software: you can redistribute it and/or modify diff --git a/data/datatest/impls.go b/data/datatest/impls.go index b043cf628d..97dd060e33 100644 --- a/data/datatest/impls.go +++ b/data/datatest/impls.go @@ -64,7 +64,8 @@ func (i entryFactoryImpl) AssembleBlock(round basics.Round) (agreement.Validated return validatedBlock{blk: &b}, nil } -func (i entryFactoryImpl) OnNewSpeculativeBlock(context.Context, agreement.ValidatedBlock) {} +func (i entryFactoryImpl) StartSpeculativeBlockAssembly(context.Context, agreement.ValidatedBlock, crypto.Digest, bool) { +} // WithSeed implements the agreement.ValidatedBlock interface. func (ve validatedBlock) WithSeed(s committee.Seed) agreement.ValidatedBlock { diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index d3c1ef3def..de92b6633a 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -27,6 +27,7 @@ import ( "github.com/algorand/go-deadlock" "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" @@ -36,8 +37,11 @@ import ( "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/condvar" + "github.com/algorand/go-algorand/util/metrics" ) +var speculativeAssemblyDiscarded = metrics.NewTagCounter("algod_speculative_assembly_discarded", "started speculative block assembly but a different block won later") + // A TransactionPool prepares valid blocks for proposal and caches // validated transaction groups. // @@ -95,15 +99,20 @@ type TransactionPool struct { // proposalAssemblyTime is the ProposalAssemblyTime configured for this node. proposalAssemblyTime time.Duration - ctx context.Context - cancelSpeculativeAssembly context.CancelFunc + ctx context.Context - // specBlockCh has an assembled speculative block - specBlockCh chan *ledgercore.ValidatedBlock - // specBlockMu protects setting/getting specBlockCh only + // specBlockMu protects speculative block assembly vars below: specBlockMu deadlock.Mutex + specActive bool + // specBlockDigest ValidatedBlock.Block().Digest() + specBlockDigest crypto.Digest + cancelSpeculativeAssembly context.CancelFunc + speculativePool *TransactionPool + // specBlockCh has an assembled speculative block + //specBlockCh chan *ledgercore.ValidatedBlock // specAsmDone channel is closed when there is no speculative assembly - specAsmDone <-chan struct{} + //specAsmDone <-chan struct{} + // TODO: feed into assemblyMu & assemblyCond above! cfg config.Local // stateproofOverflowed indicates that a stateproof transaction was allowed to @@ -149,10 +158,11 @@ func MakeTransactionPool(ledger *ledger.Ledger, cfg config.Local, log logging.Lo return &pool } -func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Context, vb *ledgercore.ValidatedBlock) (*TransactionPool, chan<- *ledgercore.ValidatedBlock, chan<- struct{}, error) { +// TODO: this needs a careful read for lock/shallow-copy issues +func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Context, vb *ledgercore.ValidatedBlock) (*TransactionPool, context.CancelFunc, error) { specLedger, err := ledger.MakeValidatedBlockAsLFE(vb, pool.ledger) if err != nil { - return nil, nil, make(chan struct{}), err + return nil, nil, err } copyPoolctx, cancel := context.WithCancel(ctx) @@ -174,19 +184,18 @@ func (pool *TransactionPool) copyTransactionPoolOverSpecLedger(ctx context.Conte cfg: pool.cfg, ctx: copyPoolctx, } + // TODO: make an 'assembly context struct' with a subset of TransactionPool fields? copy.cond.L = ©.mu copy.assemblyCond.L = ©.assemblyMu - pool.cancelSpeculativeAssembly = cancel - specDoneCh := make(chan struct{}) - pool.specAsmDone = specDoneCh + //pool.cancelSpeculativeAssembly = cancel - specBlockCh := make(chan *ledgercore.ValidatedBlock, 1) - pool.specBlockMu.Lock() - pool.specBlockCh = specBlockCh - pool.specBlockMu.Unlock() + // specBlockCh := make(chan *ledgercore.ValidatedBlock, 1) + // pool.specBlockMu.Lock() + // pool.specBlockCh = specBlockCh + // pool.specBlockMu.Unlock() - return ©, specBlockCh, specDoneCh, nil + return ©, cancel, nil } // poolAsmResults is used to syncronize the state of the block assembly process. @@ -242,11 +251,11 @@ func (pool *TransactionPool) Reset() { pool.recomputeBlockEvaluator(nil, 0, false) // cancel speculative assembly and clear its result + pool.specBlockMu.Lock() if pool.cancelSpeculativeAssembly != nil { pool.cancelSpeculativeAssembly() - <-pool.specAsmDone } - pool.specBlockCh = nil + pool.specBlockMu.Unlock() } // NumExpired returns the number of transactions that expired at the @@ -562,64 +571,96 @@ func (pool *TransactionPool) Lookup(txid transactions.Txid) (tx transactions.Sig return pool.statusCache.check(txid) } -// OnNewSpeculativeBlock handles creating a speculative block -func (pool *TransactionPool) OnNewSpeculativeBlock(ctx context.Context, vb *ledgercore.ValidatedBlock) { +// StartSpeculativeBlockAssembly handles creating a speculative block +func (pool *TransactionPool) StartSpeculativeBlockAssembly(ctx context.Context, vb *ledgercore.ValidatedBlock, blockHash crypto.Digest, onlyIfStarted bool) { - pool.mu.Lock() - // cancel any pending speculative assembly - if pool.cancelSpeculativeAssembly != nil { + if pool.cfg.SpeculativeAssemblyDisable { + return + } + + if blockHash.IsZero() { + // if we don't already have a block hash, calculate it now + blockHash = vb.Block().Digest() + } + + pool.specBlockMu.Lock() + defer pool.specBlockMu.Unlock() + if pool.specActive { + if blockHash == pool.specBlockDigest { + pool.log.Infof("StartSpeculativeBlockAssembly %s already running", blockHash.String()) + return + } + // cancel prior speculative block assembly based on different block + speculativeAssemblyDiscarded.Add("start", 1) pool.cancelSpeculativeAssembly() - <-pool.specAsmDone + pool.specActive = false + } else if onlyIfStarted { + // not already started, don't start one + return } + pool.log.Infof("StartSpeculativeBlockAssembly %s", blockHash.String()) + pool.specActive = true + pool.specBlockDigest = blockHash + + pool.mu.Lock() // move remembered txns to pending pool.rememberCommit(false) // create shallow pool copy, close the done channel when we're done with // speculative block assembly. - speculativePool, outchan, specAsmDoneCh, err := pool.copyTransactionPoolOverSpecLedger(ctx, vb) - defer close(specAsmDoneCh) + speculativePool, cancel, err := pool.copyTransactionPoolOverSpecLedger(ctx, vb) + pool.cancelSpeculativeAssembly = cancel + pool.speculativePool = speculativePool + pool.mu.Unlock() if err != nil { - pool.log.Warnf("OnNewSpeculativeBlock: %v", err) + pool.specActive = false + pool.log.Warnf("StartSpeculativeBlockAssembly: %v", err) return } // process txns only until one block is full - speculativePool.onNewBlock(vb.Block(), vb.Delta(), true) - - if speculativePool.assemblyResults.err != nil { - return - } - - select { - case outchan <- speculativePool.assemblyResults.blk: - default: - speculativePool.log.Errorf("failed writing speculative block to channel, channel already has a block") - } + // action on subordinate pool continues asynchronously, to be picked up in tryReadSpeculativeBlock + go speculativePool.onNewBlock(vb.Block(), vb.Delta(), true) } -func (pool *TransactionPool) tryReadSpeculativeBlock(branch bookkeeping.BlockHash) (*ledgercore.ValidatedBlock, error) { +func (pool *TransactionPool) tryReadSpeculativeBlock(branch bookkeeping.BlockHash, round basics.Round, deadline time.Time, stats *telemetryspec.AssembleBlockMetrics) (*ledgercore.ValidatedBlock, error) { + // assumes pool.assemblyMu is held pool.specBlockMu.Lock() - defer pool.specBlockMu.Unlock() - select { - case vb := <-pool.specBlockCh: - if vb == nil { - return nil, fmt.Errorf("speculation block is nil") + if pool.specActive { + if pool.specBlockDigest == crypto.Digest(branch) { + pool.log.Infof("update speculative deadline: %s", deadline.String()) + specPool := pool.speculativePool + pool.specBlockMu.Unlock() + // TODO: is continuing to hold outer pool.assemblyMu here a bad thing? + specPool.assemblyMu.Lock() + assembled, err := specPool.waitForBlockAssembly(round, deadline, stats) + specPool.assemblyMu.Unlock() + return assembled, err } - if vb.Block().Branch != branch { - return nil, fmt.Errorf("misspeculated block: branch = %x vs. latest hash = %x", vb.Block().Branch, branch) - } - return vb, nil - default: - return nil, fmt.Errorf("speculation block not ready") + speculativeAssemblyDiscarded.Add("read", 1) + pool.cancelSpeculativeAssembly() + pool.specActive = false } + pool.specBlockMu.Unlock() + // nope, nothing + return nil, nil } // OnNewBlock callback calls the internal implementation, onNewBlock with the ``false'' parameter to process all transactions. func (pool *TransactionPool) OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { + pool.specBlockMu.Lock() + if pool.specActive && block.Digest() != pool.specBlockDigest { + speculativeAssemblyDiscarded.Add("ONB", 1) + pool.cancelSpeculativeAssembly() + pool.specActive = false + // cancel speculative assembly, fall through to starting normal assembly + } + // TODO: make core onNewBlock() _wait_ until speculative block is done, merge state from that result + pool.specBlockMu.Unlock() pool.onNewBlock(block, delta, false) } @@ -646,10 +687,6 @@ func (pool *TransactionPool) onNewBlock(block bookkeeping.Block, delta ledgercor defer pool.mu.Unlock() defer pool.cond.Broadcast() - if !stopReprocessingAtFirstAsmBlock && pool.cancelSpeculativeAssembly != nil { - pool.cancelSpeculativeAssembly() - } - if pool.pendingBlockEvaluator == nil || block.Round() >= pool.pendingBlockEvaluator.Round() { // Adjust the pool fee threshold. The rules are: // - If there was less than one full block in the pool, reduce @@ -797,7 +834,7 @@ func (pool *TransactionPool) addToPendingBlockEvaluator(txgroup []transactions.S // recomputeBlockEvaluator constructs a new BlockEvaluator and feeds all // in-pool transactions to it (removing any transactions that are rejected // by the BlockEvaluator). Expects that the pool.mu mutex would be already taken. -func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transactions.Txid]ledgercore.IncludedTransactions, knownCommitted uint, speculativeAsm bool) (stats telemetryspec.ProcessBlockMetrics) { +func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transactions.Txid]ledgercore.IncludedTransactions, knownCommitted uint, stopAtFirstBlock bool) (stats telemetryspec.ProcessBlockMetrics) { pool.pendingBlockEvaluator = nil latest := pool.ledger.Latest() @@ -881,7 +918,7 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact continue } - hasBlock, err := pool.add(txgroup, &asmStats, speculativeAsm) + hasBlock, err := pool.add(txgroup, &asmStats, stopAtFirstBlock) if err != nil { for _, tx := range txgroup { pool.statusCache.put(tx, err.Error()) @@ -901,11 +938,11 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact case *ledgercore.LeaseInLedgerError: asmStats.LeaseErrorCount++ stats.RemovedInvalidCount++ - pool.log.Infof("Cannot re-add pending transaction to pool: %v", err) + pool.log.Infof("Cannot re-add pending transaction to pool (lease): %v", err) case *transactions.MinFeeError: asmStats.MinFeeErrorCount++ stats.RemovedInvalidCount++ - pool.log.Infof("Cannot re-add pending transaction to pool: %v", err) + pool.log.Infof("Cannot re-add pending transaction to pool (fee): %v", err) default: asmStats.InvalidCount++ stats.RemovedInvalidCount++ @@ -942,7 +979,7 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIds map[transact pool.assemblyMu.Unlock() // remember the changes made to the tx pool if we're not in speculative assembly - if !speculativeAsm { + if !stopAtFirstBlock { pool.rememberCommit(true) } return @@ -1060,15 +1097,22 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim } // Maybe we have a block already assembled + // TODO: this needs to be changed if a speculative block is in flight and if it is valid. If it is not valid, cancel it, if it is valid, wait for it. prev, err := pool.ledger.Block(round.SubSaturate(1)) if err == nil { - validatedBlock, err := pool.tryReadSpeculativeBlock(prev.Hash()) - if err == nil { - assembled = validatedBlock - return assembled, nil + specBlock, specErr := pool.tryReadSpeculativeBlock(prev.Hash(), round, deadline, &stats) + if specBlock != nil || specErr != nil { + pool.log.Infof("got spec block for %s, specErr %v", prev.Hash().String(), specErr) + return specBlock, specErr } } + assembled, err = pool.waitForBlockAssembly(round, deadline, &stats) + return +} + +// should be called with assemblyMu held +func (pool *TransactionPool) waitForBlockAssembly(round basics.Round, deadline time.Time, stats *telemetryspec.AssembleBlockMetrics) (assembled *ledgercore.ValidatedBlock, err error) { pool.assemblyDeadline = deadline pool.assemblyRound = round for time.Now().Before(deadline) && (!pool.assemblyResults.ok || pool.assemblyResults.roundStartedEvaluating != round) { @@ -1086,13 +1130,9 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim pool.assemblyMu.Lock() if pool.assemblyResults.roundStartedEvaluating > round { - // this case is expected to happen only if the transaction pool was - // able to construct *two* rounds during the time we were trying to - // assemble the empty block. while this is extreamly unlikely, we - // need to handle this. the handling it quite straight-forward : - // since the network is already ahead of us, there is no issue here - // in not generating a block ( since the block would get discarded - // anyway ) + // this case is expected to happen only if the transaction pool was able to construct *two* rounds during the time we were trying to assemble the empty block. + // while this is extremely unlikely, we need to handle this. the handling it quite straight-forward : + // since the network is already ahead of us, there is no issue here in not generating a block ( since the block would get discarded anyway ) pool.log.Infof("AssembleBlock: requested round is behind transaction pool round after timing out %d < %d", round, pool.assemblyResults.roundStartedEvaluating) return nil, ErrStaleBlockAssemblyRequest } @@ -1119,10 +1159,9 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim return nil, fmt.Errorf("AssemblyBlock: encountered error for round %d: %v", round, pool.assemblyResults.err) } if pool.assemblyResults.roundStartedEvaluating > round { - // this scenario should not happen unless the txpool is receiving the - // new blocks via OnNewBlock with "jumps" between consecutive blocks ( - // which is why it's a warning ) The "normal" usecase is evaluated on - // the top of the function. + // this scenario should not happen unless the txpool is receiving the new blocks via OnNewBlock + // with "jumps" between consecutive blocks ( which is why it's a warning ) + // The "normal" use case is evaluated on the top of the function. pool.log.Warnf("AssembleBlock: requested round is behind transaction pool round %d < %d", round, pool.assemblyResults.roundStartedEvaluating) return nil, ErrStaleBlockAssemblyRequest } else if pool.assemblyResults.roundStartedEvaluating == round.SubSaturate(1) { @@ -1134,7 +1173,7 @@ func (pool *TransactionPool) AssembleBlock(round basics.Round, deadline time.Tim pool.assemblyResults.roundStartedEvaluating, round) } - stats = pool.assemblyResults.stats + *stats = pool.assemblyResults.stats return pool.assemblyResults.blk, nil } diff --git a/data/pools/transactionPool_test.go b/data/pools/transactionPool_test.go index ee32723501..306af606aa 100644 --- a/data/pools/transactionPool_test.go +++ b/data/pools/transactionPool_test.go @@ -46,6 +46,7 @@ import ( "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/stateproof" "github.com/algorand/go-algorand/stateproof/verify" @@ -1634,7 +1635,8 @@ func TestSpeculativeBlockAssembly(t *testing.T) { cfg := config.GetDefaultLocal() cfg.TxPoolSize = testPoolSize cfg.EnableProcessBlockStats = false - transactionPool := MakeTransactionPool(mockLedger, cfg, logging.Base()) + log := logging.TestingLog(t) + transactionPool := MakeTransactionPool(mockLedger, cfg, log) savedTransactions := 0 for i, sender := range addresses { @@ -1695,8 +1697,11 @@ func TestSpeculativeBlockAssembly(t *testing.T) { block, err := blockEval.GenerateBlock() require.NoError(t, err) - transactionPool.OnNewSpeculativeBlock(context.Background(), block) - <-transactionPool.specAsmDone + t.Logf("prev block digest %s", block.Block().Digest().String()) + t.Logf("prev block hash %s", block.Block().Hash().String()) + + transactionPool.StartSpeculativeBlockAssembly(context.Background(), block, crypto.Digest{}, false) + //<-transactionPool.specAsmDone // add the block mockLedger.AddBlock(block.Block(), agreement.Certificate{}) @@ -1706,9 +1711,9 @@ func TestSpeculativeBlockAssembly(t *testing.T) { transactionPool.pendingTxGroups = nil // check that we still assemble the block - specBlock, err := transactionPool.AssembleBlock(block.Block().Round()+1, time.Now().Add(10*time.Millisecond)) + specBlock, err := transactionPool.AssembleBlock(block.Block().Round()+1, time.Now().Add(time.Second)) require.NoError(t, err) - require.Equal(t, specBlock.Block().Branch, block.Block().Hash()) + require.Equal(t, specBlock.Block().Branch, block.Block().Hash(), "spec.Branch %s", specBlock.Block().Branch.String()) require.NotNil(t, specBlock) require.Len(t, specBlock.Block().Payset, savedTransactions) } @@ -1777,10 +1782,11 @@ func TestSpeculativeBlockAssemblyWithOverlappingBlock(t *testing.T) { block, err := blockEval.GenerateBlock() require.NoError(t, err) - transactionPool.OnNewSpeculativeBlock(context.Background(), block) - <-transactionPool.specAsmDone - specBlock, err := transactionPool.tryReadSpeculativeBlock(block.Block().Hash()) - require.NoError(t, err) + transactionPool.StartSpeculativeBlockAssembly(context.Background(), block, crypto.Digest{}, false) + //<-transactionPool.specAsmDone + var stats telemetryspec.AssembleBlockMetrics + specBlock, specErr := transactionPool.tryReadSpeculativeBlock(block.Block().Hash(), block.Block().Round()+1, time.Now().Add(time.Second), &stats) + require.NoError(t, specErr) require.NotNil(t, specBlock) // assembled block doesn't have txn in the speculated block require.Len(t, specBlock.Block().Payset, savedTransactions-1) @@ -1847,6 +1853,7 @@ func TestSpeculativeBlockAssemblyDataRace(t *testing.T) { } } } + t.Logf("savedTransactions=%d", savedTransactions) pending := transactionPool.PendingTxGroups() require.Len(t, pending, savedTransactions) require.Len(t, pendingTxIDSet, savedTransactions) @@ -1859,6 +1866,7 @@ func TestSpeculativeBlockAssemblyDataRace(t *testing.T) { block, err := blockEval.GenerateBlock() require.NoError(t, err) + transactionPool.StartSpeculativeBlockAssembly(context.Background(), block, crypto.Digest{}, false) newSavedTransactions := 0 var wg sync.WaitGroup wg.Add(1) @@ -1893,18 +1901,19 @@ func TestSpeculativeBlockAssemblyDataRace(t *testing.T) { } } }() - transactionPool.OnNewSpeculativeBlock(context.Background(), block) wg.Wait() - <-transactionPool.specAsmDone - specBlock, err := transactionPool.tryReadSpeculativeBlock(block.Block().Hash()) - require.NoError(t, err) - require.NotNil(t, specBlock) - // assembled block doesn't have txn in the speculated block - require.Len(t, specBlock.Block().Payset, savedTransactions-1) + t.Logf("newSavedTransactions=%d", newSavedTransactions) // tx pool should have old txns and new txns require.Len(t, transactionPool.PendingTxIDs(), savedTransactions+newSavedTransactions) + var stats telemetryspec.AssembleBlockMetrics + specBlock, specErr := transactionPool.tryReadSpeculativeBlock(block.Block().Hash(), block.Block().Round()+1, time.Now().Add(time.Second), &stats) + require.NoError(t, specErr) + require.NotNil(t, specBlock) + // assembled block doesn't have txn in the speculated block + require.Len(t, specBlock.Block().Payset, savedTransactions-1, "len(Payset)=%d, savedTransactions=%d", len(specBlock.Block().Payset), savedTransactions) + for _, txn := range specBlock.Block().Payset { require.NotEqual(t, txn.SignedTxn.Sig, pendingTxn.Sig) require.True(t, pendingTxIDSet[txn.SignedTxn.Sig]) diff --git a/installer/config.json.example b/installer/config.json.example index 0f3c58b027..b595bf7da1 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -91,7 +91,8 @@ "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, - "SpeculativeBlockAssemblyGraceTime": 50000000, + "SpeculativeAsmTimeOffset": 400000000, + "SpeculativeAssemblyDisable": false, "SuggestedFeeBlockHistory": 3, "SuggestedFeeSlidingWindowSize": 50, "TLSCertFile": "", diff --git a/ledger/speculative.go b/ledger/speculative.go index a67a4e5f2e..be76877674 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -17,6 +17,7 @@ package ledger import ( + "errors" "fmt" "github.com/algorand/go-algorand/config" @@ -32,28 +33,21 @@ import ( // LedgerForEvaluator defines the ledger interface needed by the evaluator. type LedgerForEvaluator interface { //nolint:revive //LedgerForEvaluator is a long established but newly leaking-out name, and there really isn't a better name for it despite how lint dislikes ledger.LedgerForEvaluator - // Needed for cow.go - Block(basics.Round) (bookkeeping.Block, error) - BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) - CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error - LookupWithoutRewards(basics.Round, basics.Address) (ledgercore.AccountData, basics.Round, error) - GetCreatorForRound(basics.Round, basics.CreatableIndex, basics.CreatableType) (basics.Address, bool, error) - StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint, maxTxnBytesPerBlock int) (*internal.BlockEvaluator, error) - LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) - LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) + internal.LedgerForEvaluator LookupKv(rnd basics.Round, key string) ([]byte, error) - VerifiedTransactionCache() verify.VerifiedTransactionCache - LatestTotals() (basics.Round, ledgercore.AccountTotals, error) FlushCaches() - // Needed for the evaluator - GenesisHash() crypto.Digest + // and a few more Ledger functions + + Block(basics.Round) (bookkeeping.Block, error) Latest() basics.Round - VotersForStateProof(basics.Round) (*ledgercore.VotersForRound, error) - GenesisProto() config.ConsensusParams - BlockHdrCached(rnd basics.Round) (hdr bookkeeping.BlockHeader, err error) + VerifiedTransactionCache() verify.VerifiedTransactionCache + StartEvaluator(hdr bookkeeping.BlockHeader, paysetHint, maxTxnBytesPerBlock int) (*internal.BlockEvaluator, error) } +// ErrRoundTooHigh "round too high" try to get ledger record in the future +var ErrRoundTooHigh = errors.New("round too high") + // validatedBlockAsLFE presents a LedgerForEvaluator interface on top of // a ValidatedBlock. This makes it possible to construct a BlockEvaluator // on top, which in turn allows speculatively constructing a subsequent @@ -155,8 +149,12 @@ func (v *validatedBlockAsLFE) VotersForStateProof(r basics.Round) (*ledgercore.V } // GetCreatorForRound implements the ledgerForEvaluator interface. -func (v *validatedBlockAsLFE) GetCreatorForRound(r basics.Round, cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) { - if r == v.vb.Block().Round() { +func (v *validatedBlockAsLFE) GetCreatorForRound(rnd basics.Round, cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) { + vbround := v.vb.Block().Round() + if rnd > vbround { + return basics.Address{}, false, ErrRoundTooHigh + } + if rnd == vbround { delta, ok := v.vb.Delta().Creatables[cidx] if ok { if delta.Created && delta.Ctype == ctype { @@ -164,9 +162,12 @@ func (v *validatedBlockAsLFE) GetCreatorForRound(r basics.Round, cidx basics.Cre } return basics.Address{}, false, nil } + + // no change in this block, so lookup in previous + rnd-- } - return v.l.GetCreatorForRound(r, cidx, ctype) + return v.l.GetCreatorForRound(rnd, cidx, ctype) } // GenesisProto returns the initial protocol for this ledger. @@ -176,8 +177,11 @@ func (v *validatedBlockAsLFE) GenesisProto() config.ConsensusParams { // LookupApplication loads an application resource that matches the request parameters from the ledger. func (v *validatedBlockAsLFE) LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) { - - if rnd == v.vb.Block().Round() { + vbround := v.vb.Block().Round() + if rnd > vbround { + return ledgercore.AppResource{}, ErrRoundTooHigh + } + if rnd == vbround { // Intentionally apply (pending) rewards up to rnd. res, ok := v.vb.Delta().Accts.GetResource(addr, basics.CreatableIndex(aidx), basics.AppCreatable) if ok { @@ -185,7 +189,7 @@ func (v *validatedBlockAsLFE) LookupApplication(rnd basics.Round, addr basics.Ad } // fall back to looking up asset in ledger, until previous block - rnd = v.vb.Block().Round() - 1 + rnd-- } return v.l.LookupApplication(rnd, addr, aidx) @@ -193,14 +197,18 @@ func (v *validatedBlockAsLFE) LookupApplication(rnd basics.Round, addr basics.Ad // LookupAsset loads an asset resource that matches the request parameters from the ledger. func (v *validatedBlockAsLFE) LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) { - if rnd == v.vb.Block().Round() { + vbround := v.vb.Block().Round() + if rnd > vbround { + return ledgercore.AssetResource{}, ErrRoundTooHigh + } + if rnd == vbround { // Intentionally apply (pending) rewards up to rnd. - res, ok := v.vb.Delta().Accts.GetResource(addr, basics.CreatableIndex(aidx), basics.AppCreatable) + res, ok := v.vb.Delta().Accts.GetResource(addr, basics.CreatableIndex(aidx), basics.AssetCreatable) if ok { return ledgercore.AssetResource{AssetParams: res.AssetParams, AssetHolding: res.AssetHolding}, nil } // fall back to looking up asset in ledger, until previous block - rnd = v.vb.Block().Round() - 1 + rnd-- } return v.l.LookupAsset(rnd, addr, aidx) @@ -208,13 +216,17 @@ func (v *validatedBlockAsLFE) LookupAsset(rnd basics.Round, addr basics.Address, // LookupWithoutRewards implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) LookupWithoutRewards(rnd basics.Round, a basics.Address) (ledgercore.AccountData, basics.Round, error) { - if rnd == v.vb.Block().Round() { + vbround := v.vb.Block().Round() + if rnd > vbround { + return ledgercore.AccountData{}, rnd, ErrRoundTooHigh + } + if rnd == vbround { data, ok := v.vb.Delta().Accts.GetData(a) if ok { return data, rnd, nil } // fall back to looking up account in ledger, until previous block - rnd = v.vb.Block().Round() - 1 + rnd-- } // account didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger @@ -232,7 +244,11 @@ func (v *validatedBlockAsLFE) VerifiedTransactionCache() verify.VerifiedTransact // VerifiedTransactionCache implements the ledgerForEvaluator interface. func (v *validatedBlockAsLFE) BlockHdrCached(rnd basics.Round) (hdr bookkeeping.BlockHeader, err error) { - if rnd == v.vb.Block().Round() { + vbround := v.vb.Block().Round() + if rnd > vbround { + return bookkeeping.BlockHeader{}, ErrRoundTooHigh + } + if rnd == vbround { return v.vb.Block().BlockHeader, nil } return v.l.BlockHdrCached(rnd) @@ -253,19 +269,23 @@ func (v *validatedBlockAsLFE) StartEvaluator(hdr bookkeeping.BlockHeader, payset }) } -// FlushCaches is noop +// FlushCaches is part of ledger/internal.LedgerForEvaluator interface func (v *validatedBlockAsLFE) FlushCaches() { } // LookupKv implements LookupKv func (v *validatedBlockAsLFE) LookupKv(rnd basics.Round, key string) ([]byte, error) { - if rnd == v.vb.Block().Round() { + vbround := v.vb.Block().Round() + if rnd > vbround { + return nil, ErrRoundTooHigh + } + if rnd == vbround { data, ok := v.vb.Delta().KvMods[key] if ok { return data.Data, nil } // fall back to looking up account in ledger, until previous block - rnd = v.vb.Block().Round() - 1 + rnd-- } // account didn't change in last round. Subtract 1 so we can lookup the most recent change in the ledger diff --git a/logging/logspec/agreement.go b/logging/logspec/agreement.go index 6091891fbc..73af4c8d5a 100644 --- a/logging/logspec/agreement.go +++ b/logging/logspec/agreement.go @@ -57,6 +57,7 @@ const ( // has been reached for a given value during some (round, period, step). ThresholdReached // BlockAssembled is emitted when the source receives all parts of a block. + // BlockAssembled is not when the local node has assembled a block, but when a block has been _received_ and verified from the network, or possibly when a locally assembled block has been added to the proposal cache of blocks validaded and available for voting for the next round. TODO: rename? BlockAssembled // BlockCommittable is emitted when the source observes a // block B and a threshold of soft-votes for H(B). It is diff --git a/node/node.go b/node/node.go index acb5bfd847..7d4d67f4d9 100644 --- a/node/node.go +++ b/node/node.go @@ -1283,13 +1283,13 @@ func (node *AlgorandFullNode) AssembleBlock(round basics.Round) (agreement.Valid return validatedBlock{vb: lvb}, nil } -// OnNewSpeculativeBlock handles creating a speculative block -func (node *AlgorandFullNode) OnNewSpeculativeBlock(ctx context.Context, avb agreement.ValidatedBlock) { +// StartSpeculativeBlockAssembly handles creating a speculative block +func (node *AlgorandFullNode) StartSpeculativeBlockAssembly(ctx context.Context, avb agreement.ValidatedBlock, blockHash crypto.Digest, onlyIfStarted bool) { vb, ok := avb.(validatedBlock) if ok { - node.transactionPool.OnNewSpeculativeBlock(ctx, vb.vb) + node.transactionPool.StartSpeculativeBlockAssembly(ctx, vb.vb, blockHash, onlyIfStarted) } else { - node.log.Errorf("cannot convert agreement ValidatedBlock to ValidateBlock") + node.log.Panicf("cannot convert agreement ValidatedBlock to ValidateBlock, got %T", avb) } } diff --git a/test/scripts/e2e_client_runner.py b/test/scripts/e2e_client_runner.py index 7da46e2a81..75059bbe44 100755 --- a/test/scripts/e2e_client_runner.py +++ b/test/scripts/e2e_client_runner.py @@ -446,11 +446,17 @@ def main(): retcode = 0 capv = args.version.capitalize() xrun(['goal', 'network', 'create', '-r', netdir, '-n', 'tbd', '-t', os.path.join(repodir, f'test/testdata/nettemplates/TwoNodes50Each{capv}.json')], timeout=90) + env['ALGORAND_DATA'] = os.path.join(netdir, 'Node') + env['ALGORAND_DATA2'] = os.path.join(netdir, 'Primary') + cfgpath = os.path.join(netdir, 'Node', 'config.json') + with open(cfgpath, 'rt') as fin: + ncfg = json.load(fin) + ncfg['EnableDeveloperAPI'] = True + with open(cfgpath, 'wt') as fout: + json.dump(ncfg, fout) xrun(['goal', 'network', 'start', '-r', netdir], timeout=90) atexit.register(goal_network_stop, netdir, env) - env['ALGORAND_DATA'] = os.path.join(netdir, 'Node') - env['ALGORAND_DATA2'] = os.path.join(netdir, 'Primary') if args.unsafe_scrypt: create_kmd_config_with_unsafe_scrypt(env['ALGORAND_DATA']) diff --git a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh index 25a543300f..905f236c29 100755 --- a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh +++ b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh @@ -20,10 +20,29 @@ ACCOUNT=$(${gcmd} account list|awk '{ print $3 }') ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1337 ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unitname bogo|grep 'Asset ID'|awk '{ print $3 }') +#${gcmd} account info -a ${ACCOUNT} + # Create app that reads asset balance and checks asset details and checks round ROUND=$(goal node status | grep 'Last committed' | awk '{ print $4 }') -TIMESTAMP=$(goal ledger block --strict ${ROUND} | jq .block.ts) -APP_ID=$(${gcmd} app create --creator ${ACCOUNT} --foreign-asset $ASSET_ID --app-arg "int:$ASSET_ID" --app-arg "int:1337" --app-arg "int:0" --app-arg "int:0" --app-arg "int:1337" --app-arg "str:bogo" --app-arg "int:$ROUND" --app-arg "int:$TIMESTAMP" --approval-prog ${DIR}/tealprogs/assetround.teal --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0 --clear-prog <(printf "#pragma version 2\nint 1") | grep Created | awk '{ print $6 }') +#TIMESTAMP=$(goal ledger block --strict ${ROUND} | jq .block.ts) +TIMESTAMP=$(date +%s) +#${gcmd} app create --dryrun-dump -o ${TEMPDIR}/ac.txn --creator ${ACCOUNT} --foreign-asset $ASSET_ID --app-arg "int:$ASSET_ID" --app-arg "int:1337" --app-arg "int:0" --app-arg "int:0" --app-arg "int:1337" --app-arg "str:bogo" --app-arg "int:$ROUND" --app-arg "int:$TIMESTAMP" --approval-prog ${DIR}/tealprogs/assetround.teal --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0 --clear-prog ${DIR}/tealprogs/approve-all.teal +#${gcmd} clerk dryrun-remote -v -D ${TEMPDIR}/ac.txn + +set +e +date '+before %s' +APP_ID=$(${gcmd} app create --creator ${ACCOUNT} --foreign-asset $ASSET_ID --app-arg "int:$ASSET_ID" --app-arg "int:1337" --app-arg "int:0" --app-arg "int:0" --app-arg "int:1337" --app-arg "str:bogo" --app-arg "int:$ROUND" --app-arg "int:$TIMESTAMP" --approval-prog ${DIR}/tealprogs/assetround.teal --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0 --clear-prog ${DIR}/tealprogs/approve-all.teal | grep Created | awk '{ print $6 }') +date '+after %s' + +# if [ "x${APP_ID}x" = "xx" ]; then +# set -e +# sleep 10 +# date '+before %s' +# ROUND=$(goal node status | grep 'Last committed' | awk '{ print $4 }') +# TIMESTAMP=$(date +%s) +# APP_ID=$(${gcmd} app create --creator ${ACCOUNT} --foreign-asset $ASSET_ID --app-arg "int:$ASSET_ID" --app-arg "int:1337" --app-arg "int:0" --app-arg "int:0" --app-arg "int:1337" --app-arg "str:bogo" --app-arg "int:$ROUND" --app-arg "int:$TIMESTAMP" --approval-prog ${DIR}/tealprogs/assetround.teal --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0 --clear-prog ${DIR}/tealprogs/approve-all.teal | grep Created | awk '{ print $6 }') +# date '+after %s' +# fi # Create another account, fund it, send it some asset ACCOUNTB=$(${gcmd} account new|awk '{ print $6 }') diff --git a/test/scripts/e2e_subs/tealprogs/assetround.teal b/test/scripts/e2e_subs/tealprogs/assetround.teal index e6b3c6144f..f147a8ac0c 100644 --- a/test/scripts/e2e_subs/tealprogs/assetround.teal +++ b/test/scripts/e2e_subs/tealprogs/assetround.teal @@ -69,39 +69,47 @@ bz fail round: -// Check round against arg 6 (arg < global Round, arg + 4 > global Round) +// Check round against arg 6 // arg <= Round <= arg + 4 +// (arg <= global Round, arg + 4 >= global Round) txna ApplicationArgs 6 btoi +dup global Round -< +<= bz fail -txna ApplicationArgs 6 -btoi +//txna ApplicationArgs 6 // from dup +//btoi int 4 + -// Check timestamp against arg 7 (arg < global LatestTimestamp + 60, arg + 60 > global LatestTimestamp) +global Round +>= +bz fail + +// Check timestamp against arg 7 (arg-60 <= timestamp <= arg+60) +// (arg <= global LatestTimestamp + 60, arg + 60 >= global LatestTimestamp) txna ApplicationArgs 7 btoi +dup global LatestTimestamp int 60 + -< +<= bz fail -txna ApplicationArgs 7 -btoi +//txna ApplicationArgs 7 // from dup +//btoi int 60 + global LatestTimestamp -> +>= bz fail success: diff --git a/test/testdata/configs/config-v27.json b/test/testdata/configs/config-v27.json index a1cf1e512f..4c39e41e34 100644 --- a/test/testdata/configs/config-v27.json +++ b/test/testdata/configs/config-v27.json @@ -91,7 +91,7 @@ "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, - "SpeculativeBlockAssemblyGraceTime": 50000000, + "SpeculativeAsmTimeOffset": 40000000, "SuggestedFeeBlockHistory": 3, "SuggestedFeeSlidingWindowSize": 50, "TLSCertFile": "", From 657dc2070903aef3f7790a34a8be7b44146fb10f Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Thu, 26 Jan 2023 11:09:36 -0500 Subject: [PATCH 49/50] license fix --- ledger/speculative.go | 2 +- ledger/speculative_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ledger/speculative.go b/ledger/speculative.go index be76877674..a5ac7de315 100644 --- a/ledger/speculative.go +++ b/ledger/speculative.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2022 Algorand, Inc. +// Copyright (C) 2019-2023 Algorand, Inc. // This file is part of go-algorand // // go-algorand is free software: you can redistribute it and/or modify diff --git a/ledger/speculative_test.go b/ledger/speculative_test.go index 607d1efcf4..5c5d906556 100644 --- a/ledger/speculative_test.go +++ b/ledger/speculative_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2022 Algorand, Inc. +// Copyright (C) 2019-2023 Algorand, Inc. // This file is part of go-algorand // // go-algorand is free software: you can redistribute it and/or modify From 20765633081ae505d0c771503782764603ce4f04 Mon Sep 17 00:00:00 2001 From: Yossi Gilad Date: Thu, 27 Jul 2023 15:53:55 -0400 Subject: [PATCH 50/50] versioning --- config/localTemplate.go | 6 +++--- config/local_defaults.go | 2 +- installer/config.json.example | 2 +- test/testdata/configs/config-v28.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/localTemplate.go b/config/localTemplate.go index c0999cac17..52abd06101 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -41,7 +41,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29"` // environmental (may be overridden) // When enabled, stores blocks indefinitely, otherwise, only the most recent blocks @@ -523,9 +523,9 @@ type Local struct { // SpeculativeAsmTimeOffset defines when speculative block assembly first starts, nanoseconds before consensus AgreementFilterTimeoutPeriod0 or AgreementFilterTimeout // A huge value (greater than either AgreementFilterTimeout) disables this event. - SpeculativeAsmTimeOffset time.Duration `version[28]:"400000000"` + SpeculativeAsmTimeOffset time.Duration `version[29]:"400000000"` - SpeculativeAssemblyDisable bool `version[28]:"false"` + SpeculativeAssemblyDisable bool `version[29]:"false"` } // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers diff --git a/config/local_defaults.go b/config/local_defaults.go index 0568f4a879..b9dcc4bb98 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,7 +20,7 @@ package config var defaultLocal = Local{ - Version: 28, + Version: 29, AccountUpdatesStatsInterval: 5000000000, AccountsRebuildSynchronousMode: 1, AgreementIncomingBundlesQueueLength: 15, diff --git a/installer/config.json.example b/installer/config.json.example index 3a62038fee..21cfeee0ed 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,5 @@ { - "Version": 28, + "Version": 29, "AccountUpdatesStatsInterval": 5000000000, "AccountsRebuildSynchronousMode": 1, "AgreementIncomingBundlesQueueLength": 15, diff --git a/test/testdata/configs/config-v28.json b/test/testdata/configs/config-v28.json index 3a62038fee..b885aa68a7 100644 --- a/test/testdata/configs/config-v28.json +++ b/test/testdata/configs/config-v28.json @@ -98,7 +98,7 @@ "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, - "SpeculativeAsmTimeOffset": 400000000, + "SpeculativeAsmTimeOffset": 0, "SpeculativeAssemblyDisable": false, "StorageEngine": "sqlite", "SuggestedFeeBlockHistory": 3,