incentives: cache top online accounts and use when building AbsentPar…

…ticipationAccounts (#6085)

Co-authored-by: John Jannotti <[email protected]>

Makefile

@@ -293,6 +293,9 @@ $(GOPATH1)/bin/%:
 test: build
 	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
+testc:
+	echo $(UNIT_TEST_SOURCES) | xargs -P8 -n1 go test -c
+
 benchcheck: build
 	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -run ^NOTHING -bench Benchmark -benchtime 1x -timeout 1h
 

cmd/tealdbg/localLedger.go

@@ -359,6 +359,10 @@ func (l *localLedger) LookupAgreement(rnd basics.Round, addr basics.Address) (ba
 	}, nil
 }
 
+func (l *localLedger) GetKnockOfflineCandidates(basics.Round, config.ConsensusParams) (map[basics.Address]basics.OnlineAccountData, error) {
+	return nil, nil
+}
+
 func (l *localLedger) OnlineCirculation(rnd basics.Round, voteRound basics.Round) (basics.MicroAlgos, error) {
 	// A constant is fine for tealdbg
 	return basics.Algos(1_000_000_000), nil // 1B

daemon/algod/api/server/v2/dryrun.go

@@ -329,6 +329,10 @@ func (dl *dryrunLedger) LookupAgreement(rnd basics.Round, addr basics.Address) (
 	}, nil
 }
 
+func (dl *dryrunLedger) GetKnockOfflineCandidates(basics.Round, config.ConsensusParams) (map[basics.Address]basics.OnlineAccountData, error) {
+	return nil, nil
+}
+
 func (dl *dryrunLedger) OnlineCirculation(rnd basics.Round, voteRnd basics.Round) (basics.MicroAlgos, error) {
 	// dryrun doesn't support setting the global online stake, so we'll just return a constant
 	return basics.Algos(1_000_000_000), nil // 1B

data/basics/userBalance.go

@@ -111,7 +111,10 @@ type VotingData struct {
 type OnlineAccountData struct {
 	MicroAlgosWithRewards MicroAlgos
 	VotingData
+
 	IncentiveEligible bool
+	LastProposed      Round
+	LastHeartbeat     Round
 }
 
 // AccountData contains the data associated with a given address.
@@ -561,6 +564,8 @@ func (u AccountData) OnlineAccountData() OnlineAccountData {
 			VoteKeyDilution: u.VoteKeyDilution,
 		},
 		IncentiveEligible: u.IncentiveEligible,
+		LastProposed:      u.LastProposed,
+		LastHeartbeat:     u.LastHeartbeat,
 	}
 }
 

ledger/acctonline.go

@@ -622,11 +622,6 @@ func (ao *onlineAccounts) onlineTotals(rnd basics.Round) (basics.MicroAlgos, pro
 	return basics.MicroAlgos{Raw: onlineRoundParams.OnlineSupply}, onlineRoundParams.CurrentProtocol, nil
 }
 
-// LookupOnlineAccountData returns the online account data for a given address at a given round.
-func (ao *onlineAccounts) LookupOnlineAccountData(rnd basics.Round, addr basics.Address) (data basics.OnlineAccountData, err error) {
-	return ao.lookupOnlineAccountData(rnd, addr)
-}
-
 // roundOffset calculates the offset of the given round compared to the current dbRound. Requires that the lock would be taken.
 func (ao *onlineAccounts) roundOffset(rnd basics.Round) (offset uint64, err error) {
 	if rnd < ao.cachedDBRoundOnline {

ledger/acctonline_expired_test.go

@@ -458,7 +458,7 @@ func TestOnlineAcctModelSimple(t *testing.T) {
 	})
 	// test same scenario on double ledger
 	t.Run("DoubleLedger", func(t *testing.T) {
-		m := newDoubleLedgerAcctModel(t, protocol.ConsensusFuture, true)
+		m := newDoubleLedgerAcctModel(t, protocol.ConsensusV39, true) // TODO simulate heartbeats
 		defer m.teardown()
 		testOnlineAcctModelSimple(t, m)
 	})
@@ -626,7 +626,7 @@ func TestOnlineAcctModelScenario(t *testing.T) {
 			})
 			// test same scenario on double ledger
 			t.Run("DoubleLedger", func(t *testing.T) {
-				m := newDoubleLedgerAcctModel(t, protocol.ConsensusFuture, true)
+				m := newDoubleLedgerAcctModel(t, protocol.ConsensusV39, true) // TODO simulate heartbeats
 				defer m.teardown()
 				runScenario(t, m, tc.scenario)
 			})

ledger/eval/appcow_test.go

@@ -56,6 +56,10 @@ func (ml *emptyLedger) onlineStake() (basics.MicroAlgos, error) {
 	return basics.MicroAlgos{}, nil
 }
 
+func (ml *emptyLedger) knockOfflineCandidates() (map[basics.Address]basics.OnlineAccountData, error) {
+	return nil, nil
+}
+
 func (ml *emptyLedger) lookupAppParams(addr basics.Address, aidx basics.AppIndex, cacheOnly bool) (ledgercore.AppParamsDelta, bool, error) {
 	return ledgercore.AppParamsDelta{}, true, nil
 }

ledger/eval/cow.go

@@ -47,6 +47,7 @@ type roundCowParent interface {
 	// lookup retrieves agreement data about an address, querying the ledger if necessary.
 	lookupAgreement(basics.Address) (basics.OnlineAccountData, error)
 	onlineStake() (basics.MicroAlgos, error)
+	knockOfflineCandidates() (map[basics.Address]basics.OnlineAccountData, error)
 
 	// lookupAppParams, lookupAssetParams, lookupAppLocalState, and lookupAssetHolding retrieve data for a given address and ID.
 	// If cacheOnly is set, the ledger DB will not be queried, and only the cache will be consulted.
@@ -192,6 +193,10 @@ func (cb *roundCowState) lookupAgreement(addr basics.Address) (data basics.Onlin
 	return cb.lookupParent.lookupAgreement(addr)
 }
 
+func (cb *roundCowState) knockOfflineCandidates() (map[basics.Address]basics.OnlineAccountData, error) {
+	return cb.lookupParent.knockOfflineCandidates()
+}
+
 func (cb *roundCowState) lookupAppParams(addr basics.Address, aidx basics.AppIndex, cacheOnly bool) (ledgercore.AppParamsDelta, bool, error) {
 	params, ok := cb.mods.Accts.GetAppParams(addr, aidx)
 	if ok {

ledger/eval/cow_test.go

@@ -73,6 +73,10 @@ func (ml *mockLedger) onlineStake() (basics.MicroAlgos, error) {
 	return basics.Algos(55_555), nil
 }
 
+func (ml *mockLedger) knockOfflineCandidates() (map[basics.Address]basics.OnlineAccountData, error) {
+	return nil, nil
+}
+
 func (ml *mockLedger) lookupAppParams(addr basics.Address, aidx basics.AppIndex, cacheOnly bool) (ledgercore.AppParamsDelta, bool, error) {
 	params, ok := ml.balanceMap[addr].AppParams[aidx]
 	return ledgercore.AppParamsDelta{Params: &params}, ok, nil // XXX make a copy?

ledger/eval/eval.go

@@ -38,6 +38,7 @@ import (
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/go-algorand/util"
 	"github.com/algorand/go-algorand/util/execpool"
 )
 
@@ -48,6 +49,7 @@ type LedgerForCowBase interface {
 	CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error
 	LookupWithoutRewards(basics.Round, basics.Address) (ledgercore.AccountData, basics.Round, error)
 	LookupAgreement(basics.Round, basics.Address) (basics.OnlineAccountData, error)
+	GetKnockOfflineCandidates(basics.Round, config.ConsensusParams) (map[basics.Address]basics.OnlineAccountData, error)
 	LookupAsset(basics.Round, basics.Address, basics.AssetIndex) (ledgercore.AssetResource, error)
 	LookupApplication(basics.Round, basics.Address, basics.AppIndex) (ledgercore.AppResource, error)
 	LookupKv(basics.Round, string) ([]byte, error)
@@ -237,6 +239,10 @@ func (x *roundCowBase) lookupAgreement(addr basics.Address) (basics.OnlineAccoun
 	return ad, err
 }
 
+func (x *roundCowBase) knockOfflineCandidates() (map[basics.Address]basics.OnlineAccountData, error) {
+	return x.l.GetKnockOfflineCandidates(x.rnd, x.proto)
+}
+
 // onlineStake returns the total online stake as of the start of the round. It
 // caches the result to prevent repeated calls to the ledger.
 func (x *roundCowBase) onlineStake() (basics.MicroAlgos, error) {
@@ -1339,7 +1345,13 @@ func (eval *BlockEvaluator) TestingTxnCounter() uint64 {
 }
 
 // Call "endOfBlock" after all the block's rewards and transactions are processed.
-func (eval *BlockEvaluator) endOfBlock() error {
+// When generating a block, participating addresses are passed to prevent a
+// proposer from suspending itself.
+func (eval *BlockEvaluator) endOfBlock(participating ...basics.Address) error {
+	if participating != nil && !eval.generate {
+		panic("logic error: only pass partAddresses to endOfBlock when generating")
+	}
+
 	if eval.generate {
 		var err error
 		eval.block.TxnCommitments, err = eval.block.PaysetCommit()
@@ -1364,7 +1376,7 @@ func (eval *BlockEvaluator) endOfBlock() error {
 			}
 		}
 
-		eval.generateKnockOfflineAccountsList()
+		eval.generateKnockOfflineAccountsList(participating)
 
 		if eval.proto.StateProofInterval > 0 {
 			var basicStateProof bookkeeping.StateProofTrackingData
@@ -1607,25 +1619,94 @@ type challenge struct {
 // deltas and testing if any of them needs to be reset/suspended. Expiration
 // takes precedence - if an account is expired, it should be knocked offline and
 // key material deleted. If it is only suspended, the key material will remain.
-func (eval *BlockEvaluator) generateKnockOfflineAccountsList() {
+//
+// Different ndoes may propose different list of addresses based on node state.
+// Block validators only check whether ExpiredParticipationAccounts or
+// AbsentParticipationAccounts meet the criteria for expiration or suspension,
+// not whether the lists are complete.
+//
+// This function is passed a list of participating addresses so a node will not
+// propose a block that suspends or expires itself.
+func (eval *BlockEvaluator) generateKnockOfflineAccountsList(participating []basics.Address) {
 	if !eval.generate {
 		return
 	}
-	current := eval.Round()
 
+	current := eval.Round()
 	maxExpirations := eval.proto.MaxProposedExpiredOnlineAccounts
 	maxSuspensions := eval.proto.Payouts.MaxMarkAbsent
 
 	updates := &eval.block.ParticipationUpdates
 
-	ch := activeChallenge(&eval.proto, uint64(eval.Round()), eval.state)
+	ch := activeChallenge(&eval.proto, uint64(current), eval.state)
+
+	// Make a set of candidate addresses to check for expired or absentee status.
+	type candidateData struct {
+		VoteLastValid         basics.Round
+		VoteID                crypto.OneTimeSignatureVerifier
+		Status                basics.Status
+		LastProposed          basics.Round
+		LastHeartbeat         basics.Round
+		MicroAlgosWithRewards basics.MicroAlgos
+		IncentiveEligible     bool // currently unused below, but may be needed in the future
+	}
+	candidates := make(map[basics.Address]candidateData)
+	partAddrs := util.MakeSet(participating...)
+
+	// First, ask the ledger for the top N online accounts, with their latest
+	// online account data, current up to the previous round.
+	if maxSuspensions > 0 {
+		knockOfflineCandidates, err := eval.state.knockOfflineCandidates()
+		if err != nil {
+			// Log an error and keep going; generating lists of absent and expired
+			// accounts is not required by block validation rules.
+			logging.Base().Warnf("error fetching knockOfflineCandidates: %v", err)
+			knockOfflineCandidates = nil
+		}
+		for accountAddr, acctData := range knockOfflineCandidates {
+			// acctData is from previous block: doesn't include any updates in mods
+			candidates[accountAddr] = candidateData{
+				VoteLastValid:         acctData.VoteLastValid,
+				VoteID:                acctData.VoteID,
+				Status:                basics.Online, // from lookupOnlineAccountData, which only returns online accounts
+				LastProposed:          acctData.LastProposed,
+				LastHeartbeat:         acctData.LastHeartbeat,
+				MicroAlgosWithRewards: acctData.MicroAlgosWithRewards,
+				IncentiveEligible:     acctData.IncentiveEligible,
+			}
+		}
+	}
 
+	// Then add any accounts modified in this block, with their state at the
+	// end of the round.
 	for _, accountAddr := range eval.state.modifiedAccounts() {
 		acctData, found := eval.state.mods.Accts.GetData(accountAddr)
 		if !found {
 			continue
 		}
+		// This will overwrite data from the knockOfflineCandidates() list, if they were modified in the current block.
+		candidates[accountAddr] = candidateData{
+			VoteLastValid:         acctData.VoteLastValid,
+			VoteID:                acctData.VoteID,
+			Status:                acctData.Status,
+			LastProposed:          acctData.LastProposed,
+			LastHeartbeat:         acctData.LastHeartbeat,
+			MicroAlgosWithRewards: acctData.WithUpdatedRewards(eval.proto, eval.state.rewardsLevel()).MicroAlgos,
+			IncentiveEligible:     acctData.IncentiveEligible,
+		}
+	}
+
+	// Now, check these candidate accounts to see if they are expired or absent.
+	for accountAddr, acctData := range candidates {
+		if acctData.MicroAlgosWithRewards.IsZero() {
+			continue // don't check accounts that are being closed
+		}
+
+		if _, ok := partAddrs[accountAddr]; ok {
+			continue // don't check our own participation accounts
+		}
 
+		// Expired check: are this account's voting keys no longer valid?
 		// Regardless of being online or suspended, if voting data exists, the
 		// account can be expired to remove it.  This means an offline account
 		// can be expired (because it was already suspended).
@@ -1641,13 +1722,15 @@ func (eval *BlockEvaluator) generateKnockOfflineAccountsList() {
 			}
 		}
 
+		// Absent check: has it been too long since the last heartbeat/proposal, or
+		// has this online account failed a challenge?
 		if len(updates.AbsentParticipationAccounts) >= maxSuspensions {
 			continue // no more room (don't break the loop, since we may have more expiries)
 		}
 
 		if acctData.Status == basics.Online {
 			lastSeen := max(acctData.LastProposed, acctData.LastHeartbeat)
-			if isAbsent(eval.state.prevTotals.Online.Money, acctData.MicroAlgos, lastSeen, current) ||
+			if isAbsent(eval.state.prevTotals.Online.Money, acctData.MicroAlgosWithRewards, lastSeen, current) ||
 				failsChallenge(ch, accountAddr, lastSeen) {
 				updates.AbsentParticipationAccounts = append(
 					updates.AbsentParticipationAccounts,
@@ -1658,14 +1741,6 @@ func (eval *BlockEvaluator) generateKnockOfflineAccountsList() {
 	}
 }
 
-// delete me in Go 1.21
-func max(a, b basics.Round) basics.Round {
-	if a > b {
-		return a
-	}
-	return b
-}
-
 // bitsMatch checks if the first n bits of two byte slices match. Written to
 // work on arbitrary slices, but we expect that n is small. Only user today
 // calls with n=5.
@@ -1821,6 +1896,9 @@ func (eval *BlockEvaluator) validateAbsentOnlineAccounts() error {
 		if acctData.Status != basics.Online {
 			return fmt.Errorf("proposed absent account %v was %v, not Online", accountAddr, acctData.Status)
 		}
+		if acctData.MicroAlgos.IsZero() {
+			return fmt.Errorf("proposed absent account %v with zero algos", accountAddr)
+		}
 
 		lastSeen := max(acctData.LastProposed, acctData.LastHeartbeat)
 		if isAbsent(eval.state.prevTotals.Online.Money, acctData.MicroAlgos, lastSeen, eval.Round()) {
@@ -1890,7 +1968,16 @@ func (eval *BlockEvaluator) suspendAbsentAccounts() error {
 // After a call to GenerateBlock, the BlockEvaluator can still be used to
 // accept transactions.  However, to guard against reuse, subsequent calls
 // to GenerateBlock on the same BlockEvaluator will fail.
-func (eval *BlockEvaluator) GenerateBlock(addrs []basics.Address) (*ledgercore.UnfinishedBlock, error) {
+//
+// A list of participating addresses is passed to GenerateBlock. This lets
+// the BlockEvaluator know which of this node's participating addresses might
+// be proposing this block. This information is used when:
+//   - generating lists of absent accounts (don't suspend yourself)
+//   - preparing a ledgercore.UnfinishedBlock, which contains the end-of-block
+//     state of each potential proposer. This allows for a final check in
+//     UnfinishedBlock.FinishBlock to ensure the proposer hasn't closed its
+//     account before setting the ProposerPayout header.
+func (eval *BlockEvaluator) GenerateBlock(participating []basics.Address) (*ledgercore.UnfinishedBlock, error) {
 	if !eval.generate {
 		logging.Base().Panicf("GenerateBlock() called but generate is false")
 	}
@@ -1899,19 +1986,19 @@ func (eval *BlockEvaluator) GenerateBlock(addrs []basics.Address) (*ledgercore.U
 		return nil, fmt.Errorf("GenerateBlock already called on this BlockEvaluator")
 	}
 
-	err := eval.endOfBlock()
+	err := eval.endOfBlock(participating...)
 	if err != nil {
 		return nil, err
 	}
 
-	// look up set of participation accounts passed to GenerateBlock (possible proposers)
-	finalAccounts := make(map[basics.Address]ledgercore.AccountData, len(addrs))
-	for i := range addrs {
-		acct, err := eval.state.lookup(addrs[i])
+	// look up end-of-block state of possible proposers passed to GenerateBlock
+	finalAccounts := make(map[basics.Address]ledgercore.AccountData, len(participating))
+	for i := range participating {
+		acct, err := eval.state.lookup(participating[i])
 		if err != nil {
 			return nil, err
 		}
-		finalAccounts[addrs[i]] = acct
+		finalAccounts[participating[i]] = acct
 	}
 
 	vb := ledgercore.MakeUnfinishedBlock(eval.block, eval.state.deltas(), finalAccounts)