Merge pull request #5601 from Algo-devops-service/relbeta3.17.0

.circleci/config.yml

@@ -47,19 +47,20 @@ executors:
     resource_class: arm.large
   mac_amd64_medium:
     macos:
-      xcode: 13.2.1
-    resource_class: medium
+      xcode: 14.2.0
+    resource_class: macos.x86.medium.gen2
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   mac_amd64_large:
     macos:
-      xcode: 13.2.1
-    resource_class: large
+      xcode: 14.2.0
+    # Since they removed the large class for amd64, we will use medium here too.
+    resource_class: macos.x86.medium.gen2
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   mac_arm64: &executor-mac-arm64
     machine: true
-    resource_class: algorand/macstadium-m1-macos11
+    resource_class: algorand/macstadium-m1
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
   # these are required b/c jobs explicitly assign sizes to the executors

catchup/catchpointService.go

@@ -38,10 +38,6 @@ const (
 	// noPeersAvailableSleepInterval is the sleep interval that the node would wait if no peers are available to download the next block from.
 	// this delay is intended to ensure to give the network package some time to download the list of relays.
 	noPeersAvailableSleepInterval = 50 * time.Millisecond
-
-	// checkLedgerDownloadRetries is the number of times the catchpoint service will attempt to HEAD request the
-	// ledger from peers when `Start`ing catchpoint catchup
-	checkLedgerDownloadRetries = 10
 )
 
 // CatchpointCatchupNodeServices defines the external node support needed
@@ -826,7 +822,7 @@ func (cs *CatchpointCatchupService) checkLedgerDownload() error {
 	}
 	peerSelector := makePeerSelector(cs.net, []peerClass{{initialRank: peerRankInitialFirstPriority, peerClass: network.PeersPhonebookRelays}})
 	ledgerFetcher := makeLedgerFetcher(cs.net, cs.ledgerAccessor, cs.log, cs, cs.config)
-	for i := 0; i < checkLedgerDownloadRetries; i++ {
+	for i := 0; i < cs.config.CatchupLedgerDownloadRetryAttempts; i++ {
 		psp, peerError := peerSelector.getNextPeer()
 		if peerError != nil {
 			return err

catchup/catchpointService_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/test/partitiontest"
 )
@@ -76,6 +77,11 @@ func (m *catchpointCatchupAccessorMock) Ledger() (l ledger.CatchupAccessorClient
 	return m.l
 }
 
+// GetVerifyData returns the balances hash, spver hash and totals used by VerifyCatchpoint
+func (m *catchpointCatchupAccessorMock) GetVerifyData(ctx context.Context) (balancesHash crypto.Digest, spverHash crypto.Digest, totals ledgercore.AccountTotals, err error) {
+	return crypto.Digest{}, crypto.Digest{}, ledgercore.AccountTotals{}, nil
+}
+
 // TestCatchpointServicePeerRank ensures CatchpointService does not crash when a block fetched
 // from the local ledger and not from network when ranking a peer
 func TestCatchpointServicePeerRank(t *testing.T) {

catchup/service.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -744,17 +745,19 @@ func (s *Service) fetchRound(cert agreement.Certificate, verifier *agreement.Asy
 		if cert.Round == fetchedCert.Round &&
 			cert.Proposal.BlockDigest != fetchedCert.Proposal.BlockDigest &&
 			fetchedCert.Authenticate(*block, s.ledger, verifier) == nil {
-			s := "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "fetchRound called with a cert authenticating block with hash %v.\n"
-			s += "We fetched a valid cert authenticating a different block, %v. This indicates a fork.\n\n"
-			s += "Cert from our agreement service:\n%#v\n\n"
-			s += "Cert from the fetcher:\n%#v\n\n"
-			s += "Block from the fetcher:\n%#v\n\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-			s += "!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n"
-			s += "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+			var builder strings.Builder
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("fetchRound called with a cert authenticating block with hash %v.\n")
+			builder.WriteString("We fetched a valid cert authenticating a different block, %v. This indicates a fork.\n\n")
+			builder.WriteString("Cert from our agreement service:\n%#v\n\n")
+			builder.WriteString("Cert from the fetcher:\n%#v\n\n")
+			builder.WriteString("Block from the fetcher:\n%#v\n\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!! FORK DETECTED !!!!!!!!!!!\n")
+			builder.WriteString("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n")
+			s := builder.String()
 			s = fmt.Sprintf(s, cert.Proposal.BlockDigest, fetchedCert.Proposal.BlockDigest, cert, fetchedCert, block)
 			fmt.Println(s)
 			logging.Base().Error(s)

cmd/catchpointdump/file.go

@@ -35,6 +35,7 @@ import (
 	"github.com/algorand/avm-abi/apps"
 	cmdutil "github.com/algorand/go-algorand/cmd/util"
 	"github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
@@ -48,11 +49,13 @@ import (
 var catchpointFile string
 var outFileName string
 var excludedFields = cmdutil.MakeCobraStringSliceValue(nil, []string{"version", "catchpoint"})
+var printDigests bool
 
 func init() {
 	fileCmd.Flags().StringVarP(&catchpointFile, "tar", "t", "", "Specify the catchpoint file (either .tar or .tar.gz) to process")
 	fileCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. tracker.dump.txt )")
 	fileCmd.Flags().BoolVarP(&loadOnly, "load", "l", false, "Load only, do not dump")
+	fileCmd.Flags().BoolVarP(&printDigests, "digest", "d", false, "Print balances and spver digests")
 	fileCmd.Flags().VarP(excludedFields, "exclude-fields", "e", "List of fields to exclude from the dump: ["+excludedFields.AllowedString()+"]")
 }
 
@@ -206,6 +209,18 @@ func loadCatchpointIntoDatabase(ctx context.Context, catchupAccessor ledger.Catc
 		header, err := tarReader.Next()
 		if err != nil {
 			if err == io.EOF {
+				if printDigests {
+					err = catchupAccessor.BuildMerkleTrie(ctx, func(uint64, uint64) {})
+					if err != nil {
+						return fileHeader, err
+					}
+					var balanceHash, spverHash crypto.Digest
+					balanceHash, spverHash, _, err = catchupAccessor.GetVerifyData(ctx)
+					if err != nil {
+						return fileHeader, err
+					}
+					fmt.Printf("accounts digest=%s, spver digest=%s\n\n", balanceHash, spverHash)
+				}
 				return fileHeader, nil
 			}
 			return fileHeader, err

cmd/goal/clerk.go

@@ -74,6 +74,8 @@ var (
 	simulateAllowMoreOpcodeBudget bool
 	simulateExtraOpcodeBudget     uint64
 	simulateEnableRequestTrace    bool
+	simulateStackChange           bool
+	simulateScratchChange         bool
 )
 
 func init() {
@@ -97,7 +99,7 @@ func init() {
 	sendCmd.Flags().Uint64VarP(&amount, "amount", "a", 0, "The amount to be transferred (required), in microAlgos")
 	sendCmd.Flags().StringVarP(&closeToAddress, "close-to", "c", "", "Close account and send remainder to this address")
 	sendCmd.Flags().StringVar(&rekeyToAddress, "rekey-to", "", "Rekey account to the given spending key/address. (Future transactions from this account will need to be signed with the new key.)")
-	sendCmd.Flags().StringVarP(&programSource, "from-program", "F", "", "Program source to use as account logic")
+	sendCmd.Flags().StringVarP(&programSource, "from-program", "F", "", "Program source file to use as account logic")
 	sendCmd.Flags().StringVarP(&progByteFile, "from-program-bytes", "P", "", "Program binary to use as account logic")
 	sendCmd.Flags().StringSliceVar(&argB64Strings, "argb64", nil, "Base64 encoded args to pass to transaction logic")
 	sendCmd.Flags().StringVarP(&logicSigFile, "logic-sig", "L", "", "LogicSig to apply to transaction")
@@ -117,7 +119,7 @@ func init() {
 	signCmd.Flags().StringVarP(&txFilename, "infile", "i", "", "Partially-signed transaction file to add signature to")
 	signCmd.Flags().StringVarP(&outFilename, "outfile", "o", "", "Filename for writing the signed transaction")
 	signCmd.Flags().StringVarP(&signerAddress, "signer", "S", "", "Address of key to sign with, if different from transaction \"from\" address due to rekeying")
-	signCmd.Flags().StringVarP(&programSource, "program", "p", "", "Program source to use as account logic")
+	signCmd.Flags().StringVarP(&programSource, "program", "p", "", "Program source file to use as account logic")
 	signCmd.Flags().StringVarP(&logicSigFile, "logic-sig", "L", "", "LogicSig to apply to transaction")
 	signCmd.Flags().StringSliceVar(&argB64Strings, "argb64", nil, "Base64 encoded args to pass to transaction logic")
 	signCmd.Flags().StringVarP(&protoVersion, "proto", "P", "", "Consensus protocol version id string")
@@ -163,6 +165,8 @@ func init() {
 	simulateCmd.Flags().BoolVar(&simulateAllowMoreOpcodeBudget, "allow-more-opcode-budget", false, "Apply max extra opcode budget for apps per transaction group (default 320000) during simulation")
 	simulateCmd.Flags().Uint64Var(&simulateExtraOpcodeBudget, "extra-opcode-budget", 0, "Apply extra opcode budget for apps per transaction group during simulation")
 	simulateCmd.Flags().BoolVar(&simulateEnableRequestTrace, "trace", false, "Enable simulation time execution trace of app calls")
+	simulateCmd.Flags().BoolVar(&simulateStackChange, "stack", false, "Report stack change during simulation time")
+	simulateCmd.Flags().BoolVar(&simulateScratchChange, "scratch", false, "Report scratch change during simulation time")
 }
 
 var clerkCmd = &cobra.Command{
@@ -1365,6 +1369,8 @@ func decodeTxnsFromFile(file string) []transactions.SignedTxn {
 
 func traceCmdOptionToSimulateTraceConfigModel() simulation.ExecTraceConfig {
 	return simulation.ExecTraceConfig{
-		Enable: simulateEnableRequestTrace,
+		Enable:  simulateEnableRequestTrace,
+		Stack:   simulateStackChange,
+		Scratch: simulateScratchChange,
 	}
 }

components/mocks/mockCatchpointCatchupAccessor.go

@@ -19,9 +19,11 @@ package mocks
 import (
 	"context"
 
+	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
 )
 
 // MockCatchpointCatchupAccessor is a dummy CatchpointCatchupAccessor implementation which doesn't do anything.
@@ -67,6 +69,11 @@ func (m *MockCatchpointCatchupAccessor) GetCatchupBlockRound(ctx context.Context
 	return basics.Round(0), nil
 }
 
+// GetVerifyData returns the balances hash, spver hash and totals used by VerifyCatchpoint
+func (m *MockCatchpointCatchupAccessor) GetVerifyData(ctx context.Context) (balancesHash crypto.Digest, spverHash crypto.Digest, totals ledgercore.AccountTotals, err error) {
+	return crypto.Digest{}, crypto.Digest{}, ledgercore.AccountTotals{}, nil
+}
+
 // VerifyCatchpoint verifies that the catchpoint is valid by reconstructing the label.
 func (m *MockCatchpointCatchupAccessor) VerifyCatchpoint(ctx context.Context, blk *bookkeeping.Block) (err error) {
 	return nil

daemon/algod/api/algod.oas2.json

@@ -3108,6 +3108,29 @@
         }
       }
     },
+    "AvmValue": {
+      "description": "Represents an AVM value.",
+      "type": "object",
+      "required": [
+        "type"
+      ],
+      "properties": {
+        "type": {
+          "description": "value type. Value `1` refers to **bytes**, value `2` refers to **uint64**",
+          "type": "integer"
+        },
+        "bytes": {
+          "description": "bytes value.",
+          "type": "string",
+          "format": "byte"
+        },
+        "uint": {
+          "description": "uint value.",
+          "type": "integer",
+          "x-algorand-format": "uint64"
+        }
+      }
+    },
     "StateDelta": {
       "description": "Application state delta.",
       "type": "array",
@@ -3491,6 +3514,14 @@
         "enable": {
           "description": "A boolean option for opting in execution trace features simulation endpoint.",
           "type": "boolean"
+        },
+        "stack-change": {
+          "description": "A boolean option enabling returning stack changes together with execution trace during simulation.",
+          "type": "boolean"
+        },
+        "scratch-change": {
+          "description": "A boolean option enabling returning scratch slot changes together with execution trace during simulation.",
+          "type": "boolean"
         }
       }
     },
@@ -3853,6 +3884,23 @@
         }
       }
     },
+    "ScratchChange": {
+      "description": "A write operation into a scratch slot.",
+      "type": "object",
+      "required": [
+        "slot",
+        "new-value"
+      ],
+      "properties": {
+        "slot": {
+          "description": "The scratch slot written.",
+          "type": "integer"
+        },
+        "new-value": {
+          "$ref": "#/definitions/AvmValue"
+        }
+      }
+    },
     "SimulationOpcodeTraceUnit": {
       "description": "The set of trace information and effect from evaluating a single opcode.",
       "type": "object",
@@ -3864,12 +3912,30 @@
           "description": "The program counter of the current opcode being evaluated.",
           "type": "integer"
         },
+        "scratch-changes": {
+          "description": "The writes into scratch slots.",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/ScratchChange"
+          }
+        },
         "spawned-inners": {
           "description": "The indexes of the traces for inner transactions spawned by this opcode, if any.",
           "type": "array",
           "items": {
             "type": "integer"
           }
+        },
+        "stack-pop-count": {
+          "description": "The number of deleted stack values by this opcode.",
+          "type": "integer"
+        },
+        "stack-additions": {
+          "description": "The values added by this opcode to the stack.",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/AvmValue"
+          }
         }
       }
     },