From dc6e17f7746326551f3e01caec091992af65299d Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 21 Nov 2024 19:59:36 -0500 Subject: [PATCH 01/46] chain: Handle txpool conflicts properly --- chain/manager.go | 129 +++++++++++++++++++++--------------------- wallet/wallet_test.go | 16 ++++-- 2 files changed, 75 insertions(+), 70 deletions(-) diff --git a/chain/manager.go b/chain/manager.go index d1b6288..679b281 100644 --- a/chain/manager.go +++ b/chain/manager.go @@ -473,7 +473,7 @@ func (m *Manager) revalidatePool() { // transactions, but that's expensive; this approach should work fine in // practice. if m.txpool.weight >= txpoolMaxWeight { - // sort txns fee without modifying the actual pool slice + // sort txns by fee without modifying the actual pool slice type feeTxn struct { index int fees types.Currency @@ -507,10 +507,16 @@ func (m *Manager) revalidatePool() { return txnFees[i].index < txnFees[j].index }) rem := m.txpool.txns[:0] + v2rem := m.txpool.v2txns[:0] for _, ft := range txnFees { - rem = append(rem, m.txpool.txns[ft.index]) + if !ft.v2 { + rem = append(rem, m.txpool.txns[ft.index]) + } else { + v2rem = append(v2rem, m.txpool.v2txns[ft.index]) + } } m.txpool.txns = rem + m.txpool.v2txns = v2rem } // remove and re-add all transactions @@ -1001,44 +1007,56 @@ func (m *Manager) V2TransactionSet(basis types.ChainIndex, txn types.V2Transacti return m.tipState.Index, append(parents, txn), nil } -func (m *Manager) checkDupTxnSet(txns []types.Transaction, v2txns []types.V2Transaction) (types.Hash256, bool) { +func (m *Manager) checkTxnSet(txns []types.Transaction, v2txns []types.V2Transaction) (bool, error) { allInPool := true - checkPool := func(txid types.TransactionID) { + checkPool := func(txid types.TransactionID) types.TransactionID { if allInPool { if _, ok := m.txpool.indices[txid]; !ok { allInPool = false } } + return txid } h := types.NewHasher() - for i, txn := range txns { - txid := txn.ID() - checkPool(txid) - h.E.WriteUint64(uint64(i)) - txid.EncodeTo(h.E) + for _, txn := range txns { + checkPool(txn.ID()).EncodeTo(h.E) } - for i, txn := range v2txns { - txid := txn.ID() - checkPool(txid) - h.E.WriteUint64(uint64(i)) - txid.EncodeTo(h.E) + for _, txn := range v2txns { + checkPool(txn.ID()).EncodeTo(h.E) } setID := h.Sum() - _, invalid := m.txpool.invalidTxnSets[setID] - return setID, allInPool || invalid -} + if err := m.txpool.invalidTxnSets[setID]; allInPool || err != nil { + return true, err + } -func (m *Manager) markBadTxnSet(setID types.Hash256, err error) error { - const maxInvalidTxnSets = 1000 - if len(m.txpool.invalidTxnSets) >= maxInvalidTxnSets { - // forget a random entry - for id := range m.txpool.invalidTxnSets { - delete(m.txpool.invalidTxnSets, id) - break + // validate + markBadTxnSet := func(err error) error { + const maxInvalidTxnSets = 1000 + if len(m.txpool.invalidTxnSets) >= maxInvalidTxnSets { + // forget a random entry + for id := range m.txpool.invalidTxnSets { + delete(m.txpool.invalidTxnSets, id) + break + } } + m.txpool.invalidTxnSets[setID] = err + return err } - m.txpool.invalidTxnSets[setID] = err - return err + ms := consensus.NewMidState(m.tipState) + for _, txn := range txns { + ts := m.store.SupplementTipTransaction(txn) + if err := consensus.ValidateTransaction(ms, txn, ts); err != nil { + return false, markBadTxnSet(fmt.Errorf("transaction %v is invalid: %w", txn.ID(), err)) + } + ms.ApplyTransaction(txn, ts) + } + for _, txn := range v2txns { + if err := consensus.ValidateV2Transaction(ms, txn); err != nil { + return false, markBadTxnSet(fmt.Errorf("v2 transaction %v is invalid: %w", txn.ID(), err)) + } + ms.ApplyV2Transaction(txn) + } + return false, nil } func (m *Manager) updateV2TransactionProofs(txns []types.V2Transaction, from, to types.ChainIndex) ([]types.V2Transaction, error) { @@ -1145,32 +1163,25 @@ func (m *Manager) AddPoolTransactions(txns []types.Transaction) (known bool, err defer m.mu.Unlock() m.revalidatePool() - setID, known := m.checkDupTxnSet(txns, nil) - if known { - return true, m.txpool.invalidTxnSets[setID] - } - - // validate as a standalone set - ms := consensus.NewMidState(m.tipState) - for _, txn := range txns { - ts := m.store.SupplementTipTransaction(txn) - if err := consensus.ValidateTransaction(ms, txn, ts); err != nil { - return false, m.markBadTxnSet(setID, fmt.Errorf("transaction %v is invalid: %w", txn.ID(), err)) - } - ms.ApplyTransaction(txn, ts) + if known, err := m.checkTxnSet(txns, nil); known || err != nil { + return known, err } for _, txn := range txns { txid := txn.ID() if _, ok := m.txpool.indices[txid]; ok { - continue // skip transactions already in pool + continue // skip transactions already in the pool } - m.txpool.ms.ApplyTransaction(txn, m.store.SupplementTipTransaction(txn)) + ts := m.store.SupplementTipTransaction(txn) + if err := consensus.ValidateTransaction(m.txpool.ms, txn, ts); err != nil { + m.txpool.ms = nil // force revalidation next time the pool is queried + return false, fmt.Errorf("transaction %v conflicts with pool: %w", txid, err) + } + m.txpool.ms.ApplyTransaction(txn, ts) m.txpool.indices[txid] = len(m.txpool.txns) m.txpool.txns = append(m.txpool.txns, txn) m.txpool.weight += m.tipState.TransactionWeight(txn) } - // invalidate caches m.txpool.medianFee = nil m.txpool.parentMap = nil @@ -1188,7 +1199,6 @@ func (m *Manager) UpdateV2TransactionSet(txns []types.V2Transaction, from, to ty if from == to { return txns, nil } - m.mu.Lock() defer m.mu.Unlock() return m.updateV2TransactionProofs(txns, from, to) @@ -1213,49 +1223,38 @@ func (m *Manager) AddV2PoolTransactions(basis types.ChainIndex, txns []types.V2T defer m.mu.Unlock() m.revalidatePool() - setID, known := m.checkDupTxnSet(nil, txns) - if known { - return true, m.txpool.invalidTxnSets[setID] - } - - // take ownership + // take ownership of Merkle proofs, and update them to the current tip txns = append([]types.V2Transaction(nil), txns...) for i := range txns { txns[i] = txns[i].DeepCopy() } - - // update the transaction set to the current tip txns, err := m.updateV2TransactionProofs(txns, basis, m.tipState.Index) if err != nil { - return false, m.markBadTxnSet(setID, fmt.Errorf("failed to update set basis: %w", err)) - } else if len(txns) == 0 { - return true, nil + return false, fmt.Errorf("failed to update set basis: %w", err) } - // validate as a standalone set - ms := consensus.NewMidState(m.tipState) - for _, txn := range txns { - if err := consensus.ValidateV2Transaction(ms, txn); err != nil { - return false, m.markBadTxnSet(setID, fmt.Errorf("transaction %v is invalid: %w", txn.ID(), err)) - } - ms.ApplyV2Transaction(txn) + if known, err := m.checkTxnSet(nil, txns); known || err != nil { + return known, err } for _, txn := range txns { txid := txn.ID() if _, ok := m.txpool.indices[txid]; ok { - continue // skip transactions already in pool + continue // skip transactions already in the pool + } + if err := consensus.ValidateV2Transaction(m.txpool.ms, txn); err != nil { + m.txpool.ms = nil // force revalidation next time the pool is queried + return false, fmt.Errorf("transaction %v conflicts with pool: %w", txid, err) } m.txpool.ms.ApplyV2Transaction(txn) - m.txpool.indices[txid] = len(m.txpool.v2txns) + m.txpool.indices[txid] = len(m.txpool.txns) m.txpool.v2txns = append(m.txpool.v2txns, txn) m.txpool.weight += m.tipState.V2TransactionWeight(txn) } - // invalidate caches m.txpool.medianFee = nil m.txpool.parentMap = nil - return + return false, nil } // NewManager returns a Manager initialized with the provided Store and State. diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 152d185..4948d89 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1879,7 +1879,7 @@ func TestSingleAddressWalletEventTypes(t *testing.T) { }) } -func TestV2TPoolRace(t *testing.T) { +func TestV2TxPoolRace(t *testing.T) { // create wallet store pk := types.GeneratePrivateKey() ws := testutil.NewEphemeralWalletStore() @@ -1946,11 +1946,17 @@ func TestV2TPoolRace(t *testing.T) { // output in the spend transaction invalid unless it is updated. mineAndSync(t, cm, ws, w, types.VoidAddress, 1) - // broadcast the transaction set including the already confirmed setup - // transaction. This seems unnecessary, but it's a fairly common occurrence - // when passing transaction sets using unconfirmed outputs between a renter - // and host. If the transaction set is not updated correctly, it will fail. + // even though the setup transaction has been confirmed, and the spend + // transaction is outdated, we can still add them without error: internally, + // AddV2PoolTransactions will remove any confirmed transactions, replace any + // ephemeral outputs, and update the Merkle proofs of all elements. if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{setupTxn, spendTxn}); err != nil { t.Fatal(err) } + // updating the transaction shouldn't change its ID + if spendTxn, ok := cm.V2PoolTransaction(spendTxn.ID()); !ok { + t.Fatal("expected spend transaction to be in pool") + } else if spendTxn.SiacoinInputs[0].Parent.StateElement.LeafIndex == types.UnassignedLeafIndex { + t.Fatal("expected ephemeral output to be replaced") + } } From e086953a449a261ed284a50a572a33ee286cfb8c Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Dec 2024 20:42:31 -0800 Subject: [PATCH 02/46] all: update core --- .gitignore | 2 + go.mod | 2 +- go.sum | 6 +-- miner.go | 15 +------ rhp/v4/rpc.go | 27 ++++++++---- rhp/v4/rpc_test.go | 28 ------------- rhp/v4/server.go | 44 +++++++++++-------- syncer/peer.go | 12 +++--- syncer/syncer.go | 8 ++-- syncer/syncer_test.go | 7 +--- testutil/host.go | 2 +- wallet/wallet_test.go | 98 +++---------------------------------------- 12 files changed, 69 insertions(+), 182 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..94f1119 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +.vscode diff --git a/go.mod b/go.mod index a1ff3a1..82bfc26 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.6.2 + go.sia.tech/core v0.7.1-0.20241203043244-c435a355b1da go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.29.0 diff --git a/go.sum b/go.sum index 8bcd91f..085e6d6 100644 --- a/go.sum +++ b/go.sum @@ -6,10 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.6.1 h1:eaExM2E2eNr43su2XDkY5J24E3F54YGS7hcC3WtVjVk= -go.sia.tech/core v0.6.1/go.mod h1:P3C1BWa/7J4XgdzWuaYHBvLo2RzZ0UBaJM4TG1GWB2g= -go.sia.tech/core v0.6.2 h1:8NEjxyD93A+EhZopsBy/LvuHH+zUSjRNKnf9rXgtIwU= -go.sia.tech/core v0.6.2/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= +go.sia.tech/core v0.7.1-0.20241203043244-c435a355b1da h1:taO86czGly5SIb8UswVI2W7rmxhmv9G4C93zoAwtfxk= +go.sia.tech/core v0.7.1-0.20241203043244-c435a355b1da/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/miner.go b/miner.go index 85ad705..db4358a 100644 --- a/miner.go +++ b/miner.go @@ -1,7 +1,6 @@ package coreutils import ( - "encoding/binary" "time" "go.sia.tech/core/consensus" @@ -12,22 +11,10 @@ import ( // FindBlockNonce attempts to find a nonce for b that meets the PoW target. func FindBlockNonce(cs consensus.State, b *types.Block, timeout time.Duration) bool { b.Nonce = 0 - buf := make([]byte, 32+8+8+32) - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) - if b.V2 != nil { - copy(buf[:32], "sia/id/block|") - copy(buf[48:], b.V2.Commitment[:]) - } else { - root := b.MerkleRoot() - copy(buf[:32], b.ParentID[:]) - copy(buf[48:], root[:]) - } factor := cs.NonceFactor() startBlock := time.Now() - for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { + for b.ID().CmpWork(cs.ChildTarget) < 0 { b.Nonce += factor - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) if time.Since(startBlock) > timeout { return false } diff --git a/rhp/v4/rpc.go b/rhp/v4/rpc.go index 93d5973..88fcecc 100644 --- a/rhp/v4/rpc.go +++ b/rhp/v4/rpc.go @@ -246,11 +246,10 @@ func RPCWriteSector(ctx context.Context, t TransportClient, prices rhp4.HostPric req := rhp4.RPCWriteSectorRequest{ Prices: prices, Token: token, - Duration: duration, DataLength: length, } - if err := req.Validate(t.PeerKey(), req.Duration); err != nil { + if err := req.Validate(t.PeerKey()); err != nil { return RPCWriteSectorResult{}, fmt.Errorf("invalid request: %w", err) } @@ -287,7 +286,7 @@ func RPCWriteSector(ctx context.Context, t TransportClient, prices rhp4.HostPric return RPCWriteSectorResult{ Root: resp.Root, - Usage: prices.RPCWriteSectorCost(uint64(length), duration), + Usage: prices.RPCWriteSectorCost(uint64(length)), }, nil } @@ -498,7 +497,7 @@ func RPCSectorRoots(ctx context.Context, t TransportClient, cs consensus.State, RenterSignature: revision.RenterSignature, } - if err := req.Validate(contract.Revision.HostPublicKey, revision, length); err != nil { + if err := req.Validate(contract.Revision.HostPublicKey, revision); err != nil { return RPCSectorRootsResult{}, fmt.Errorf("invalid request: %w", err) } @@ -727,10 +726,14 @@ func RPCRenewContract(ctx context.Context, t TransportClient, tp TxPool, signer // sign the renewal renewalSigHash := cs.RenewalSigHash(renewal) renewal.RenterSignature = signer.SignHash(renewalSigHash) + // sign the contract + contractSigHash := cs.ContractSigHash(renewal.NewContract) + renewal.NewContract.RenterSignature = signer.SignHash(contractSigHash) // send the renter signatures renterPolicyResp := rhp4.RPCRenewContractSecondResponse{ - RenterRenewalSignature: renewal.RenterSignature, + RenterRenewalSignature: renewal.RenterSignature, + RenterContractSignature: renewal.NewContract.RenterSignature, } for _, si := range renewalTxn.SiacoinInputs[:len(req.RenterInputs)] { renterPolicyResp.RenterSatisfiedPolicies = append(renterPolicyResp.RenterSatisfiedPolicies, si.SatisfiedPolicy) @@ -760,7 +763,9 @@ func RPCRenewContract(ctx context.Context, t TransportClient, tp TxPool, signer // validate the host signature if !existing.HostPublicKey.VerifyHash(renewalSigHash, hostRenewal.HostSignature) { - return RPCRenewContractResult{}, errors.New("invalid host signature") + return RPCRenewContractResult{}, errors.New("invalid host renewal signature") + } else if !existing.HostPublicKey.VerifyHash(contractSigHash, hostRenewal.NewContract.HostSignature) { + return RPCRenewContractResult{}, errors.New("invalid host contract signature") } return RPCRenewContractResult{ Contract: ContractRevision{ @@ -852,10 +857,14 @@ func RPCRefreshContract(ctx context.Context, t TransportClient, tp TxPool, signe // sign the renewal renewalSigHash := cs.RenewalSigHash(renewal) renewal.RenterSignature = signer.SignHash(renewalSigHash) + // sign the new contract + contractSigHash := cs.ContractSigHash(renewal.NewContract) + renewal.NewContract.RenterSignature = signer.SignHash(contractSigHash) // send the renter signatures renterPolicyResp := rhp4.RPCRefreshContractSecondResponse{ - RenterRenewalSignature: renewal.RenterSignature, + RenterRenewalSignature: renewal.RenterSignature, + RenterContractSignature: renewal.NewContract.RenterSignature, } for _, si := range renewalTxn.SiacoinInputs[:len(req.RenterInputs)] { renterPolicyResp.RenterSatisfiedPolicies = append(renterPolicyResp.RenterSatisfiedPolicies, si.SatisfiedPolicy) @@ -885,7 +894,9 @@ func RPCRefreshContract(ctx context.Context, t TransportClient, tp TxPool, signe // validate the host signature if !existing.HostPublicKey.VerifyHash(renewalSigHash, hostRenewal.HostSignature) { - return RPCRefreshContractResult{}, errors.New("invalid host signature") + return RPCRefreshContractResult{}, errors.New("invalid host renewal signature") + } else if !existing.HostPublicKey.VerifyHash(contractSigHash, hostRenewal.NewContract.HostSignature) { + return RPCRefreshContractResult{}, errors.New("invalid host contract signature") } return RPCRefreshContractResult{ Contract: ContractRevision{ diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index 2f75957..e08e553 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -156,8 +156,6 @@ func TestSettings(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -216,8 +214,6 @@ func TestFormContract(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -275,8 +271,6 @@ func TestFormContractBasis(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -333,8 +327,6 @@ func TestRPCRefresh(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -446,8 +438,6 @@ func TestRPCRenew(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -607,8 +597,6 @@ func TestAccounts(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -708,8 +696,6 @@ func TestReadWriteSector(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -804,8 +790,6 @@ func TestAppendSectors(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -921,8 +905,6 @@ func TestVerifySector(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -1014,8 +996,6 @@ func TestRPCFreeSectors(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -1137,8 +1117,6 @@ func TestRPCSectorRoots(t *testing.T) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -1247,8 +1225,6 @@ func BenchmarkWrite(b *testing.B) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -1337,8 +1313,6 @@ func BenchmarkRead(b *testing.B) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ @@ -1439,8 +1413,6 @@ func BenchmarkContractUpload(b *testing.B) { WalletAddress: w.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, - MaxSectorDuration: 3 * 144, - MaxSectorBatchSize: 100, RemainingStorage: 100 * proto4.SectorSize, TotalStorage: 100 * proto4.SectorSize, Prices: proto4.HostPrices{ diff --git a/rhp/v4/server.go b/rhp/v4/server.go index af45916..cfd32b7 100644 --- a/rhp/v4/server.go +++ b/rhp/v4/server.go @@ -232,9 +232,7 @@ func (s *Server) handleRPCWriteSector(stream net.Conn) error { var req rhp4.RPCWriteSectorRequest if err := rhp4.ReadRequest(stream, &req); err != nil { return errorDecodingError("failed to read request: %v", err) - } - settings := s.settings.RHP4Settings() - if err := req.Validate(s.hostKey.PublicKey(), settings.MaxSectorDuration); err != nil { + } else if err := req.Validate(s.hostKey.PublicKey()); err != nil { return errorBadRequest("request invalid: %v", err) } prices := req.Prices @@ -253,12 +251,12 @@ func (s *Server) handleRPCWriteSector(stream net.Conn) error { return errorDecodingError("failed to read sector data: %v", err) } - usage := prices.RPCWriteSectorCost(req.DataLength, req.Duration) + usage := prices.RPCWriteSectorCost(req.DataLength) if err = s.contractor.DebitAccount(req.Token.Account, usage); err != nil { return fmt.Errorf("failed to debit account: %w", err) } - if err := s.sectors.StoreSector(root, §or, req.Duration); err != nil { + if err := s.sectors.StoreSector(root, §or, prices.TipHeight+rhp4.TempSectorDuration); err != nil { return fmt.Errorf("failed to store sector: %w", err) } return rhp4.WriteResponse(stream, &rhp4.RPCWriteSectorResponse{ @@ -283,9 +281,7 @@ func (s *Server) handleRPCFreeSectors(stream net.Conn) error { } fc := state.Revision - - settings := s.settings.RHP4Settings() - if err := req.Validate(s.hostKey.PublicKey(), fc, settings.MaxSectorBatchSize); err != nil { + if err := req.Validate(s.hostKey.PublicKey(), fc); err != nil { return errorBadRequest("request invalid: %v", err) } prices := req.Prices @@ -347,8 +343,7 @@ func (s *Server) handleRPCAppendSectors(stream net.Conn) error { return errorDecodingError("failed to read request: %v", err) } - settings := s.settings.RHP4Settings() - if err := req.Validate(s.hostKey.PublicKey(), settings.MaxSectorBatchSize); err != nil { + if err := req.Validate(s.hostKey.PublicKey()); err != nil { return errorBadRequest("request invalid: %v", err) } @@ -483,8 +478,7 @@ func (s *Server) handleRPCSectorRoots(stream net.Conn) error { defer unlock() // validate the request fields - settings := s.settings.RHP4Settings() - if err := req.Validate(s.hostKey.PublicKey(), state.Revision, settings.MaxSectorBatchSize); err != nil { + if err := req.Validate(s.hostKey.PublicKey(), state.Revision); err != nil { return rhp4.NewRPCError(rhp4.ErrorCodeBadRequest, err.Error()) } prices := req.Prices @@ -693,7 +687,7 @@ func (s *Server) handleRPCRefreshContract(stream net.Conn) error { // validate the request settings := s.settings.RHP4Settings() - if err := req.Validate(s.hostKey.PublicKey(), state.Revision.ExpirationHeight, settings.MaxCollateral); err != nil { + if err := req.Validate(s.hostKey.PublicKey(), state.Revision.TotalCollateral, state.Revision.ExpirationHeight, settings.MaxCollateral); err != nil { return rhp4.NewRPCError(rhp4.ErrorCodeBadRequest, err.Error()) } @@ -782,15 +776,22 @@ func (s *Server) handleRPCRefreshContract(stream net.Conn) error { // validate the renter's signature renewalSigHash := cs.RenewalSigHash(renewal) if !existing.RenterPublicKey.VerifyHash(renewalSigHash, renterSigResp.RenterRenewalSignature) { - return rhp4.ErrInvalidSignature + return fmt.Errorf("failed to validate renter renewal signature: %w", rhp4.ErrInvalidSignature) } renewal.RenterSignature = renterSigResp.RenterRenewalSignature + renewal.HostSignature = s.hostKey.SignHash(renewalSigHash) + + contractSigHash := cs.ContractSigHash(renewal.NewContract) + if !existing.RenterPublicKey.VerifyHash(contractSigHash, renterSigResp.RenterContractSignature) { + return fmt.Errorf("failed to validate renter contract signature: %w", rhp4.ErrInvalidSignature) + } + renewal.NewContract.RenterSignature = renterSigResp.RenterContractSignature + renewal.NewContract.HostSignature = s.hostKey.SignHash(contractSigHash) // apply the renter's signatures for i, policy := range renterSigResp.RenterSatisfiedPolicies { renewalTxn.SiacoinInputs[i].SatisfiedPolicy = policy } - renewal.HostSignature = s.hostKey.SignHash(renewalSigHash) // add the renter's parents to our transaction pool to ensure they are valid // and update the proofs. @@ -849,7 +850,7 @@ func (s *Server) handleRPCRenewContract(stream net.Conn) error { tip := s.chain.Tip() // validate the request - if err := req.Validate(s.hostKey.PublicKey(), tip, state.Revision.ProofHeight, settings.MaxCollateral, settings.MaxContractDuration); err != nil { + if err := req.Validate(s.hostKey.PublicKey(), tip, state.Revision.Filesize, state.Revision.ProofHeight, settings.MaxCollateral, settings.MaxContractDuration); err != nil { return rhp4.NewRPCError(rhp4.ErrorCodeBadRequest, err.Error()) } @@ -944,15 +945,22 @@ func (s *Server) handleRPCRenewContract(stream net.Conn) error { // validate the renter's signature renewalSigHash := cs.RenewalSigHash(renewal) if !existing.RenterPublicKey.VerifyHash(renewalSigHash, renterSigResp.RenterRenewalSignature) { - return rhp4.ErrInvalidSignature + return fmt.Errorf("failed to validate renewal signature: %w", rhp4.ErrInvalidSignature) } renewal.RenterSignature = renterSigResp.RenterRenewalSignature + renewal.HostSignature = s.hostKey.SignHash(renewalSigHash) + + contractSighash := cs.ContractSigHash(renewal.NewContract) + if !existing.RenterPublicKey.VerifyHash(contractSighash, renterSigResp.RenterContractSignature) { + return fmt.Errorf("failed to validate contract signature: %w", rhp4.ErrInvalidSignature) + } + renewal.NewContract.RenterSignature = renterSigResp.RenterContractSignature + renewal.NewContract.HostSignature = s.hostKey.SignHash(contractSighash) // apply the renter's signatures for i, policy := range renterSigResp.RenterSatisfiedPolicies { renewalTxn.SiacoinInputs[i].SatisfiedPolicy = policy } - renewal.HostSignature = s.hostKey.SignHash(renewalSigHash) // add the renter's parents to our transaction pool to ensure they are valid // and update the proofs. diff --git a/syncer/peer.go b/syncer/peer.go index 9c5c938..45a6d81 100644 --- a/syncer/peer.go +++ b/syncer/peer.go @@ -117,7 +117,7 @@ func (p *Peer) SendBlock(id types.BlockID, timeout time.Duration) (types.Block, } // RelayHeader relays a header to the peer. -func (p *Peer) RelayHeader(h gateway.BlockHeader, timeout time.Duration) error { +func (p *Peer) RelayHeader(h types.BlockHeader, timeout time.Duration) error { return p.callRPC(&gateway.RPCRelayHeader{Header: h}, timeout) } @@ -182,7 +182,7 @@ func (p *Peer) SendCheckpoint(index types.ChainIndex, timeout time.Duration) (ty } // RelayV2Header relays a v2 block header to the peer. -func (p *Peer) RelayV2Header(h gateway.V2BlockHeader, timeout time.Duration) error { +func (p *Peer) RelayV2Header(h types.BlockHeader, timeout time.Duration) error { return p.callRPC(&gateway.RPCRelayV2Header{Header: h}, timeout) } @@ -381,17 +381,17 @@ func (s *Syncer) handleRPC(id types.Specifier, stream *gateway.Stream, origin *P if err := stream.ReadRequest(r); err != nil { return err } - cs, ok := s.cm.State(r.Header.Parent.ID) + cs, ok := s.cm.State(r.Header.ParentID) if !ok { - s.resync(origin, fmt.Sprintf("peer relayed a v2 header with unknown parent (%v)", r.Header.Parent.ID)) + s.resync(origin, fmt.Sprintf("peer relayed a v2 header with unknown parent (%v)", r.Header.ParentID)) return nil } - bid := r.Header.ID(cs) + bid := r.Header.ID() if _, ok := s.cm.State(bid); ok { return nil // already seen } else if bid.CmpWork(cs.ChildTarget) < 0 { return s.ban(origin, errors.New("peer sent v2 header with insufficient work")) - } else if r.Header.Parent != s.cm.Tip() { + } else if r.Header.ParentID != s.cm.Tip().ID { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain s.resync(origin, "peer relayed a v2 header that does not attach to our tip") diff --git a/syncer/syncer.go b/syncer/syncer.go index 83f9f64..f1ac29b 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -301,7 +301,7 @@ func (s *Syncer) runPeer(p *Peer) error { } } -func (s *Syncer) relayHeader(h gateway.BlockHeader, origin *Peer) { +func (s *Syncer) relayHeader(h types.BlockHeader, origin *Peer) { s.mu.Lock() defer s.mu.Unlock() for _, p := range s.peers { @@ -323,7 +323,7 @@ func (s *Syncer) relayTransactionSet(txns []types.Transaction, origin *Peer) { } } -func (s *Syncer) relayV2Header(bh gateway.V2BlockHeader, origin *Peer) { +func (s *Syncer) relayV2Header(bh types.BlockHeader, origin *Peer) { s.mu.Lock() defer s.mu.Unlock() for _, p := range s.peers { @@ -723,10 +723,10 @@ func (s *Syncer) Connect(ctx context.Context, addr string) (*Peer, error) { } // BroadcastHeader broadcasts a header to all peers. -func (s *Syncer) BroadcastHeader(h gateway.BlockHeader) { s.relayHeader(h, nil) } +func (s *Syncer) BroadcastHeader(h types.BlockHeader) { s.relayHeader(h, nil) } // BroadcastV2Header broadcasts a v2 header to all peers. -func (s *Syncer) BroadcastV2Header(h gateway.V2BlockHeader) { s.relayV2Header(h, nil) } +func (s *Syncer) BroadcastV2Header(h types.BlockHeader) { s.relayV2Header(h, nil) } // BroadcastV2BlockOutline broadcasts a v2 block outline to all peers. func (s *Syncer) BroadcastV2BlockOutline(b gateway.V2BlockOutline) { s.relayV2BlockOutline(b, nil) } diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index 89d8502..d3551a0 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -80,12 +80,7 @@ func TestSyncer(t *testing.T) { } // broadcast the tip from s1 to s2 - s1.BroadcastHeader(gateway.BlockHeader{ - ParentID: b.ParentID, - Nonce: b.Nonce, - Timestamp: b.Timestamp, - MerkleRoot: b.MerkleRoot(), - }) + s1.BroadcastHeader(b.Header()) for i := 0; i < 100; i++ { if cm1.Tip() == cm2.Tip() { diff --git a/testutil/host.go b/testutil/host.go index 87cda42..a1703fe 100644 --- a/testutil/host.go +++ b/testutil/host.go @@ -160,7 +160,7 @@ func (ec *EphemeralContractor) RenewV2Contract(renewalSet rhp4.TransactionSet, _ return errors.New("contract already exists") } - ec.contracts[existingID] = renewal.FinalRevision + delete(ec.contracts, existingID) // remove the existing contract ec.contracts[contractID] = renewal.NewContract ec.roots[contractID] = append([]types.Hash256(nil), ec.roots[existingID]...) return nil diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 152d185..abb82ed 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -3,7 +3,6 @@ package wallet_test import ( "errors" "fmt" - "math" "math/bits" "path/filepath" "testing" @@ -1728,14 +1727,10 @@ func TestSingleAddressWalletEventTypes(t *testing.T) { cau.UpdateElementProof(&fce.StateElement) } - // finalize the contract - finalRevision := fce.V2FileContract - finalRevision.RevisionNumber = math.MaxUint64 - finalRevision.RenterSignature = types.Signature{} - finalRevision.HostSignature = types.Signature{} // create a renewal renewal := types.V2FileContractRenewal{ - FinalRevision: finalRevision, + FinalRenterOutput: fce.V2FileContract.RenterOutput, + FinalHostOutput: fce.V2FileContract.HostOutput, NewContract: types.V2FileContract{ RenterOutput: fc.RenterOutput, ProofHeight: fc.ProofHeight + 10, @@ -1750,6 +1745,10 @@ func TestSingleAddressWalletEventTypes(t *testing.T) { renewalSig := pk.SignHash(renewalSigHash) renewal.RenterSignature = renewalSig renewal.HostSignature = renewalSig + contractSigHash := cm.TipState().ContractSigHash(renewal.NewContract) + contractSig := pk.SignHash(contractSigHash) + renewal.NewContract.RenterSignature = contractSig + renewal.NewContract.HostSignature = contractSig newContractValue := renterPayout.Add(cm.TipState().V2FileContractTax(renewal.NewContract)) @@ -1792,91 +1791,6 @@ func TestSingleAddressWalletEventTypes(t *testing.T) { mineAndSync(t, cm, ws, wm, types.VoidAddress, 1) assertEvent(t, wm, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+network.MaturityDelay) }) - - t.Run("v2 contract resolution - finalization", func(t *testing.T) { - // create a storage contract - renterPayout := types.Siacoins(10000) - fc := types.V2FileContract{ - RenterOutput: types.SiacoinOutput{ - Address: addr, - Value: renterPayout, - }, - HostOutput: types.SiacoinOutput{ - Address: types.VoidAddress, - Value: types.ZeroCurrency, - }, - ProofHeight: cm.TipState().Index.Height + 10, - ExpirationHeight: cm.TipState().Index.Height + 20, - - RenterPublicKey: pk.PublicKey(), - HostPublicKey: pk.PublicKey(), - } - contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) - sigHash := cm.TipState().ContractSigHash(fc) - sig := pk.SignHash(sigHash) - fc.RenterSignature = sig - fc.HostSignature = sig - - // create a transaction with the contract - txn := types.V2Transaction{ - FileContracts: []types.V2FileContract{fc}, - } - basis, toSign, err := wm.FundV2Transaction(&txn, contractValue, false) - if err != nil { - t.Fatal(err) - } - wm.SignV2Inputs(&txn, toSign) - - // broadcast the transaction - if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { - t.Fatal(err) - } - // current tip - tip := cm.Tip() - // mine a block to confirm the contract formation - mineAndSync(t, cm, ws, wm, types.VoidAddress, 1) - - // this is annoying because we have to keep the file contract - // proof - _, applied, err := cm.UpdatesSince(tip, 1000) - if err != nil { - t.Fatal(err) - } - - // get the confirmed file contract element - var fce types.V2FileContractElement - applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { - fce = ele - }) - for _, cau := range applied { - cau.UpdateElementProof(&fce.StateElement) - } - - // finalize the contract - finalRevision := fce.V2FileContract - finalRevision.RevisionNumber = math.MaxUint64 - finalRevisionSigHash := cm.TipState().ContractSigHash(finalRevision) - // create a renewal - finalization := types.V2FileContractFinalization(pk.SignHash(finalRevisionSigHash)) - - // create the renewal transaction - resolutionTxn := types.V2Transaction{ - FileContractResolutions: []types.V2FileContractResolution{ - { - Parent: fce, - Resolution: &finalization, - }, - }, - } - - // broadcast the renewal - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolutionTxn}); err != nil { - t.Fatal(err) - } - // mine a block to confirm the renewal - mineAndSync(t, cm, ws, wm, types.VoidAddress, 1) - assertEvent(t, wm, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+network.MaturityDelay) - }) } func TestV2TPoolRace(t *testing.T) { From 2bcb90e5f2df2c1ca56bc5d312835988946de8b3 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Dec 2024 20:57:24 -0800 Subject: [PATCH 03/46] add renter and host signatures to rpc form, renew, and refresh --- rhp/v4/rpc.go | 8 ++++---- rhp/v4/rpc_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/rhp/v4/rpc.go b/rhp/v4/rpc.go index 88fcecc..8a71126 100644 --- a/rhp/v4/rpc.go +++ b/rhp/v4/rpc.go @@ -596,10 +596,10 @@ func RPCFormContract(ctx context.Context, t TransportClient, tp TxPool, signer F // sign the renter inputs after the host inputs have been added signer.SignV2Inputs(&formationTxn, toSign) formationSigHash := cs.ContractSigHash(fc) - formationTxn.FileContracts[0].RenterSignature = signer.SignHash(formationSigHash) + fc.RenterSignature = signer.SignHash(formationSigHash) renterPolicyResp := rhp4.RPCFormContractSecondResponse{ - RenterContractSignature: formationTxn.FileContracts[0].RenterSignature, + RenterContractSignature: fc.RenterSignature, } for _, si := range formationTxn.SiacoinInputs[:len(renterSiacoinElements)] { renterPolicyResp.RenterSatisfiedPolicies = append(renterPolicyResp.RenterSatisfiedPolicies, si.SatisfiedPolicy) @@ -770,7 +770,7 @@ func RPCRenewContract(ctx context.Context, t TransportClient, tp TxPool, signer return RPCRenewContractResult{ Contract: ContractRevision{ ID: params.ContractID.V2RenewalID(), - Revision: renewal.NewContract, + Revision: hostRenewal.NewContract, }, RenewalSet: TransactionSet{ Basis: hostTransactionSetResp.Basis, @@ -901,7 +901,7 @@ func RPCRefreshContract(ctx context.Context, t TransportClient, tp TxPool, signe return RPCRefreshContractResult{ Contract: ContractRevision{ ID: params.ContractID.V2RenewalID(), - Revision: renewal.NewContract, + Revision: hostRenewal.NewContract, }, RenewalSet: TransactionSet{ Basis: hostTransactionSetResp.Basis, diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index e08e553..bd1fbaf 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -253,6 +253,13 @@ func TestFormContract(t *testing.T) { } else if !known { t.Fatal("expected transaction set to be known") } + + sigHash := cm.TipState().ContractSigHash(result.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, result.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, result.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } } func TestFormContractBasis(t *testing.T) { @@ -420,6 +427,13 @@ func TestRPCRefresh(t *testing.T) { } else if !known { t.Fatal("expected transaction set to be known") } + + sigHash := cm.TipState().ContractSigHash(refreshResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, refreshResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, refreshResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } }) } @@ -473,6 +487,12 @@ func TestRPCRenew(t *testing.T) { t.Fatal(err) } revision := result.Contract + sigHash := cm.TipState().ContractSigHash(revision.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, revision.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, revision.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } // verify the transaction set is valid if known, err := cm.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { @@ -534,6 +554,13 @@ func TestRPCRenew(t *testing.T) { } else if !known { t.Fatal("expected transaction set to be known") } + + sigHash := cm.TipState().ContractSigHash(renewResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } }) t.Run("full rollover", func(t *testing.T) { @@ -556,6 +583,13 @@ func TestRPCRenew(t *testing.T) { } else if !known { t.Fatal("expected transaction set to be known") } + + sigHash := cm.TipState().ContractSigHash(renewResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } }) t.Run("no rollover", func(t *testing.T) { @@ -578,6 +612,13 @@ func TestRPCRenew(t *testing.T) { } else if !known { t.Fatal("expected transaction set to be known") } + + sigHash := cm.TipState().ContractSigHash(renewResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } }) } From 83ba3e02a6862a18db49f0e0df0a2b1ec3dec214 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Dec 2024 22:05:19 -0800 Subject: [PATCH 04/46] rhp4: fix missing signatures, add regression checks --- rhp/v4/rpc.go | 1 + rhp/v4/rpc_test.go | 24 ++++++++++++++++++++++++ rhp/v4/server.go | 13 ++----------- testutil/host.go | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/rhp/v4/rpc.go b/rhp/v4/rpc.go index 8a71126..34a8fd7 100644 --- a/rhp/v4/rpc.go +++ b/rhp/v4/rpc.go @@ -418,6 +418,7 @@ func RPCAppendSectors(ctx context.Context, t TransportClient, cs consensus.State } else if !contract.Revision.HostPublicKey.VerifyHash(sigHash, hostSignature.HostSignature) { return RPCAppendSectorsResult{}, rhp4.ErrInvalidSignature } + revision.HostSignature = hostSignature.HostSignature return RPCAppendSectorsResult{ Revision: revision, Usage: usage, diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index bd1fbaf..4e939a0 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -865,6 +865,27 @@ func TestAppendSectors(t *testing.T) { } revision := formResult.Contract + assertLastRevision := func(t *testing.T) { + t.Helper() + + lastRev, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(lastRev, revision.Revision) { + t.Log(lastRev) + t.Log(revision.Revision) + t.Fatalf("expected last revision to match") + } + + sigHash := cm.TipState().ContractSigHash(revision.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, lastRev.RenterSignature) { + t.Fatal("renter signature invalid") + } else if !hostKey.PublicKey().VerifyHash(sigHash, lastRev.HostSignature) { + t.Fatal("host signature invalid") + } + } + assertLastRevision(t) + cs := cm.TipState() account := proto4.Account(renterKey.PublicKey()) @@ -876,6 +897,7 @@ func TestAppendSectors(t *testing.T) { t.Fatal(err) } revision.Revision = fundResult.Revision + assertLastRevision(t) token := proto4.AccountToken{ Account: account, @@ -915,6 +937,8 @@ func TestAppendSectors(t *testing.T) { if appendResult.Revision.FileMerkleRoot != proto4.MetaRoot(roots) { t.Fatal("root mismatch") } + revision.Revision = appendResult.Revision + assertLastRevision(t) // read the sectors back buf := bytes.NewBuffer(make([]byte, 0, proto4.SectorSize)) diff --git a/rhp/v4/server.go b/rhp/v4/server.go index cfd32b7..16f6ec2 100644 --- a/rhp/v4/server.go +++ b/rhp/v4/server.go @@ -17,17 +17,6 @@ import ( "lukechampine.com/frand" ) -const ( - sectorsPerTiB = (1 << 40) / (1 << 22) - memoryPer1TiB = sectorsPerTiB * 32 - - sectorsPer10TiB = 10 * sectorsPerTiB - memoryPer10TiB = sectorsPer10TiB * 32 - - sectorsPer100TiB = 100 * sectorsPerTiB - memoryPer100TiB = sectorsPer100TiB * 32 -) - var protocolVersion = [3]byte{4, 0, 0} type ( @@ -435,6 +424,7 @@ func (s *Server) handleRPCFundAccounts(stream net.Conn) error { if !revision.RenterPublicKey.VerifyHash(sigHash, req.RenterSignature) { return rhp4.ErrInvalidSignature } + revision.RenterSignature = req.RenterSignature revision.HostSignature = s.hostKey.SignHash(sigHash) balances, err := s.contractor.CreditAccountsWithContract(req.Deposits, req.ContractID, revision, usage) @@ -498,6 +488,7 @@ func (s *Server) handleRPCSectorRoots(stream net.Conn) error { // sign the revision revision.HostSignature = s.hostKey.SignHash(sigHash) + revision.RenterSignature = req.RenterSignature // update the contract err = s.contractor.ReviseV2Contract(req.ContractID, revision, state.Roots, usage) diff --git a/testutil/host.go b/testutil/host.go index a1703fe..c0af3f9 100644 --- a/testutil/host.go +++ b/testutil/host.go @@ -7,6 +7,7 @@ import ( "sync" "testing" + "go.sia.tech/core/consensus" proto4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" @@ -119,6 +120,13 @@ func (ec *EphemeralContractor) AddV2Contract(formationSet rhp4.TransactionSet, _ } fc := formationTxn.FileContracts[0] + sigHash := consensus.State{}.ContractSigHash(fc) + if !fc.RenterPublicKey.VerifyHash(sigHash, fc.RenterSignature) { + return errors.New("invalid renter signature") + } else if !fc.HostPublicKey.VerifyHash(sigHash, fc.HostSignature) { + return errors.New("invalid host signature") + } + contractID := formationTxn.V2FileContractID(formationTxn.ID(), 0) if _, ok := ec.contracts[contractID]; ok { return errors.New("contract already exists") @@ -160,6 +168,13 @@ func (ec *EphemeralContractor) RenewV2Contract(renewalSet rhp4.TransactionSet, _ return errors.New("contract already exists") } + sigHash := consensus.State{}.ContractSigHash(renewal.NewContract) + if !existing.RenterPublicKey.VerifyHash(sigHash, renewal.NewContract.RenterSignature) { + return errors.New("invalid renter signature") + } else if !existing.HostPublicKey.VerifyHash(sigHash, renewal.NewContract.HostSignature) { + return errors.New("invalid host signature") + } + delete(ec.contracts, existingID) // remove the existing contract ec.contracts[contractID] = renewal.NewContract ec.roots[contractID] = append([]types.Hash256(nil), ec.roots[existingID]...) @@ -179,6 +194,13 @@ func (ec *EphemeralContractor) ReviseV2Contract(contractID types.FileContractID, return errors.New("revision number must be greater than existing") } + sigHash := consensus.State{}.ContractSigHash(revision) + if !existing.RenterPublicKey.VerifyHash(sigHash, revision.RenterSignature) { + return errors.New("invalid renter signature") + } else if !existing.HostPublicKey.VerifyHash(sigHash, revision.HostSignature) { + return errors.New("invalid host signature") + } + ec.contracts[contractID] = revision ec.roots[contractID] = append([]types.Hash256(nil), roots...) return nil @@ -202,11 +224,26 @@ func (ec *EphemeralContractor) CreditAccountsWithContract(deposits []proto4.Acco ec.mu.Lock() defer ec.mu.Unlock() + existing, ok := ec.contracts[contractID] + if !ok { + return nil, errors.New("contract not found") + } else if revision.RevisionNumber <= existing.RevisionNumber { + return nil, errors.New("revision number must be greater than existing") + } + + sigHash := consensus.State{}.ContractSigHash(revision) + if !existing.RenterPublicKey.VerifyHash(sigHash, revision.RenterSignature) { + return nil, errors.New("invalid renter signature") + } else if !existing.HostPublicKey.VerifyHash(sigHash, revision.HostSignature) { + return nil, errors.New("invalid host signature") + } + var balance = make([]types.Currency, 0, len(deposits)) for _, deposit := range deposits { ec.accounts[deposit.Account] = ec.accounts[deposit.Account].Add(deposit.Amount) balance = append(balance, ec.accounts[deposit.Account]) } + ec.contracts[contractID] = revision return balance, nil } From 7bf95dd18f31d97c5e03377a3027d88d86944f49 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 3 Dec 2024 09:25:14 -0800 Subject: [PATCH 05/46] fix(rhp4): remove duration param from RPCWrite impl --- rhp/v4/rpc.go | 2 +- rhp/v4/rpc_test.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rhp/v4/rpc.go b/rhp/v4/rpc.go index 34a8fd7..38d6320 100644 --- a/rhp/v4/rpc.go +++ b/rhp/v4/rpc.go @@ -237,7 +237,7 @@ func RPCReadSector(ctx context.Context, t TransportClient, prices rhp4.HostPrice } // RPCWriteSector writes a sector to the host. -func RPCWriteSector(ctx context.Context, t TransportClient, prices rhp4.HostPrices, token rhp4.AccountToken, data io.Reader, length uint64, duration uint64) (RPCWriteSectorResult, error) { +func RPCWriteSector(ctx context.Context, t TransportClient, prices rhp4.HostPrices, token rhp4.AccountToken, data io.Reader, length uint64) (RPCWriteSectorResult, error) { if length == 0 { return RPCWriteSectorResult{}, errors.New("cannot write zero-length sector") } else if length > rhp4.SectorSize { diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index 4e939a0..f09be58 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -793,7 +793,7 @@ func TestReadWriteSector(t *testing.T) { data := frand.Bytes(1024) // store the sector - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data)), 5) + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) if err != nil { t.Fatal(err) } @@ -913,7 +913,7 @@ func TestAppendSectors(t *testing.T) { frand.Read(sector[:]) root := proto4.SectorRoot(§or) - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sector[:]), proto4.SectorSize, 5) + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sector[:]), proto4.SectorSize) if err != nil { t.Fatal(err) } else if writeResult.Root != root { @@ -1026,7 +1026,7 @@ func TestVerifySector(t *testing.T) { data := frand.Bytes(1024) // store the sector - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data)), 5) + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) if err != nil { t.Fatal(err) } @@ -1120,7 +1120,7 @@ func TestRPCFreeSectors(t *testing.T) { data := frand.Bytes(1024) // store the sector - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data)), 5) + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) if err != nil { t.Fatal(err) } @@ -1259,7 +1259,7 @@ func TestRPCSectorRoots(t *testing.T) { data := frand.Bytes(1024) // store the sector - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data)), 5) + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) if err != nil { t.Fatal(err) } @@ -1355,7 +1355,7 @@ func BenchmarkWrite(b *testing.B) { for i := 0; i < b.N; i++ { // store the sector - _, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize, 5) + _, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize) if err != nil { b.Fatal(err) } @@ -1438,7 +1438,7 @@ func BenchmarkRead(b *testing.B) { sectors = append(sectors, sector) // store the sector - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize, 5) + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize) if err != nil { b.Fatal(err) } @@ -1548,7 +1548,7 @@ func BenchmarkContractUpload(b *testing.B) { wg.Add(1) go func(i int) { defer wg.Done() - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize, 5) + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize) if err != nil { b.Error(err) } else if writeResult.Root != roots[i] { From 0025b13da7111e2e643d9f76392aa7047254089e Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 3 Dec 2024 09:48:11 -0800 Subject: [PATCH 06/46] chore: automate releases with knope --- .github/workflows/prepare-release.yml | 26 +++++++++++++ .github/workflows/release.yml | 27 ++++++++++++++ go.mod | 2 +- knope.toml | 54 +++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/prepare-release.yml create mode 100644 .github/workflows/release.yml create mode 100644 knope.toml diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 0000000..7b6ca01 --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,26 @@ +on: + push: + branches: [master] + +permissions: + contents: write + pull-requests: write + +name: Create Release PR +jobs: + prepare-release: + if: "!contains(github.event.head_commit.message, 'chore: prepare release')" # Skip merges from releases + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config --global user.name github-actions[bot] + git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 + - run: knope prepare-release --verbose + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..bd5b98b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,27 @@ +on: + pull_request: + types: + - closed + branches: + - master + +permissions: + contents: write + +name: Create Release PR +jobs: + prepare-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config --global user.name github-actions[bot] + git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 + - run: knope release --verbose + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true diff --git a/go.mod b/go.mod index 82bfc26..5d1df8f 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/coreutils +module go.sia.tech/coreutils // v0.7.0 go 1.23.1 diff --git a/knope.toml b/knope.toml new file mode 100644 index 0000000..3aea5f1 --- /dev/null +++ b/knope.toml @@ -0,0 +1,54 @@ +[package] +changelog = "CHANGELOG.md" +versioned_files = ["go.mod"] +ignore_go_major_versioning = true + +[[workflows]] +name = "document-change" + +[[workflows.steps]] +type = "CreateChangeFile" + +[[workflows]] +name = "prepare-release" + +[[workflows.steps]] +type = "Command" +command = "git switch -c release" + +[[workflows.steps]] +type = "PrepareRelease" + +[[workflows.steps]] +type = "Command" +command = "git commit -m \"chore: prepare release $version\"" +variables = { "$version" = "Version" } + +[[workflows.steps]] +type = "Command" +command = "git push --force --set-upstream origin release" + +[workflows.steps.variables] +"$version" = "Version" + +[[workflows.steps]] +type = "CreatePullRequest" +base = "master" + +[workflows.steps.title] +template = "chore: prepare release $version" +variables = { "$version" = "Version" } + +[workflows.steps.body] +template = "This PR was created automatically. Merging it will create a new release for $version\n\n$changelog" +variables = { "$changelog" = "ChangelogEntry", "$version" = "Version" } + +[[workflows]] +name = "release" + +[[workflows.steps]] +type = "Release" + +[github] +owner = "SiaFoundation" +repo = "hostd" From 4aa3ef13d304cb691fcbf5b2774b5682a88fe7f8 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 3 Dec 2024 09:56:50 -0800 Subject: [PATCH 07/46] add first changeset --- .changeset/automate_releases.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/automate_releases.md diff --git a/.changeset/automate_releases.md b/.changeset/automate_releases.md new file mode 100644 index 0000000..d5fea6e --- /dev/null +++ b/.changeset/automate_releases.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Automate releases From 35640a58afcf2e2948495a643f88bf1774d90bb8 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 4 Dec 2024 07:05:10 -0800 Subject: [PATCH 08/46] fix knope repo --- knope.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knope.toml b/knope.toml index 3aea5f1..7f5f214 100644 --- a/knope.toml +++ b/knope.toml @@ -51,4 +51,4 @@ type = "Release" [github] owner = "SiaFoundation" -repo = "hostd" +repo = "coreutils" From b73c11c939939e40aef99bb2590cea4eda486f47 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 4 Dec 2024 09:40:31 -0800 Subject: [PATCH 09/46] ci: fix workflow names --- .github/workflows/prepare-release.yml | 2 +- .github/workflows/release.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 7b6ca01..0a719bd 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -1,3 +1,4 @@ +name: Prepare Release on: push: branches: [master] @@ -6,7 +7,6 @@ permissions: contents: write pull-requests: write -name: Create Release PR jobs: prepare-release: if: "!contains(github.event.head_commit.message, 'chore: prepare release')" # Skip merges from releases diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd5b98b..47dae24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,14 +1,14 @@ +name: Release on: pull_request: - types: + types: - closed - branches: + branches: - master permissions: contents: write -name: Create Release PR jobs: prepare-release: runs-on: ubuntu-latest From 12dcfc50949381200055d1ac85ffd298605117ef Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 4 Dec 2024 08:21:39 -0800 Subject: [PATCH 10/46] fix(rhp4): Return 0 for nonexistent accounts --- rhp/v4/rpc_test.go | 11 +++++++++-- testutil/host.go | 5 +---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index f09be58..a869635 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -625,6 +625,7 @@ func TestRPCRenew(t *testing.T) { func TestAccounts(t *testing.T) { n, genesis := testutil.V2Network() hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + account := proto4.Account(renterKey.PublicKey()) cm, s, w := startTestNode(t, n, genesis) @@ -673,7 +674,13 @@ func TestAccounts(t *testing.T) { revision := formResult.Contract cs := cm.TipState() - account := proto4.Account(renterKey.PublicKey()) + + balance, err := rhp4.RPCAccountBalance(context.Background(), transport, account) + if err != nil { + t.Fatal(err) + } else if !balance.IsZero() { + t.Fatal("expected zero balance") + } accountFundAmount := types.Siacoins(25) fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ @@ -713,7 +720,7 @@ func TestAccounts(t *testing.T) { } // verify the account balance - balance, err := rhp4.RPCAccountBalance(context.Background(), transport, account) + balance, err = rhp4.RPCAccountBalance(context.Background(), transport, account) if err != nil { t.Fatal(err) } else if !balance.Equals(accountFundAmount) { diff --git a/testutil/host.go b/testutil/host.go index c0af3f9..b5b4407 100644 --- a/testutil/host.go +++ b/testutil/host.go @@ -210,10 +210,7 @@ func (ec *EphemeralContractor) ReviseV2Contract(contractID types.FileContractID, func (ec *EphemeralContractor) AccountBalance(account proto4.Account) (types.Currency, error) { ec.mu.Lock() defer ec.mu.Unlock() - balance, ok := ec.accounts[account] - if !ok { - return types.Currency{}, errors.New("account not found") - } + balance, _ := ec.accounts[account] return balance, nil } From 7ea0739cfd5e4a308fa05f93a33caf14fc90f2ee Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 4 Dec 2024 09:43:29 -0800 Subject: [PATCH 11/46] chore: update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5d1df8f..dfbeb29 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.7.1-0.20241203043244-c435a355b1da + go.sia.tech/core v0.7.1 go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.29.0 diff --git a/go.sum b/go.sum index 085e6d6..50c98dc 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.1-0.20241203043244-c435a355b1da h1:taO86czGly5SIb8UswVI2W7rmxhmv9G4C93zoAwtfxk= -go.sia.tech/core v0.7.1-0.20241203043244-c435a355b1da/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= +go.sia.tech/core v0.7.1 h1:PrKh19Ql5vJbQbB5YGtTHQ8W3fRF8hhYnR4kPOIOIME= +go.sia.tech/core v0.7.1/go.mod h1:gB8iXFJFSV8XIHRaL00CL6Be+hyykB+SYnvRPHCCc/E= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= From 237a1adbaad02a13ef54ff3bef9962b13f2c2dc3 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Fri, 8 Nov 2024 21:35:19 -0500 Subject: [PATCH 12/46] chain,syncer: Add support for block pruning --- chain/db.go | 125 +++++++++++++++++++++++++++++++---------------- chain/manager.go | 12 ++++- chain/options.go | 7 +++ miner.go | 8 +-- syncer/peer.go | 4 +- syncer/syncer.go | 10 ++-- 6 files changed, 114 insertions(+), 52 deletions(-) diff --git a/chain/db.go b/chain/db.go index 9f1eda0..466acfa 100644 --- a/chain/db.go +++ b/chain/db.go @@ -13,43 +13,32 @@ import ( ) type supplementedBlock struct { - Block types.Block + Header *types.BlockHeader + Block *types.Block Supplement *consensus.V1BlockSupplement } func (sb supplementedBlock) EncodeTo(e *types.Encoder) { - e.WriteUint8(2) - (types.V2Block)(sb.Block).EncodeTo(e) - e.WriteBool(sb.Supplement != nil) - if sb.Supplement != nil { - sb.Supplement.EncodeTo(e) - } + e.WriteUint8(3) + types.EncodePtr(e, sb.Header) + types.EncodePtr(e, (*types.V2Block)(sb.Block)) + types.EncodePtr(e, sb.Supplement) } func (sb *supplementedBlock) DecodeFrom(d *types.Decoder) { - if v := d.ReadUint8(); v != 2 { - d.SetErr(fmt.Errorf("incompatible version (%d)", v)) - } - (*types.V2Block)(&sb.Block).DecodeFrom(d) - if d.ReadBool() { - sb.Supplement = new(consensus.V1BlockSupplement) - sb.Supplement.DecodeFrom(d) - } -} - -// helper type for decoding just the header information from a block -type supplementedHeader struct { - ParentID types.BlockID - Timestamp time.Time -} - -func (sh *supplementedHeader) DecodeFrom(d *types.Decoder) { - if v := d.ReadUint8(); v != 2 { + switch v := d.ReadUint8(); v { + case 2: + sb.Header = nil + sb.Block = new(types.Block) + (*types.V2Block)(sb.Block).DecodeFrom(d) + types.DecodePtr(d, &sb.Supplement) + case 3: + types.DecodePtr(d, &sb.Header) + types.DecodePtrCast[types.V2Block](d, &sb.Block) + types.DecodePtr(d, &sb.Supplement) + default: d.SetErr(fmt.Errorf("incompatible version (%d)", v)) } - sh.ParentID.DecodeFrom(d) - _ = d.ReadUint64() // nonce - sh.Timestamp = d.ReadTime() } type versionedState struct { @@ -304,21 +293,62 @@ func (db *DBStore) putState(cs consensus.State) { db.bucket(bStates).put(cs.Index.ID[:], versionedState{cs}) } -func (db *DBStore) getBlock(id types.BlockID) (b types.Block, bs *consensus.V1BlockSupplement, _ bool) { +func (db *DBStore) getBlock(id types.BlockID) (bh types.BlockHeader, b *types.Block, bs *consensus.V1BlockSupplement, _ bool) { var sb supplementedBlock ok := db.bucket(bBlocks).get(id[:], &sb) - return sb.Block, sb.Supplement, ok + if sb.Header == nil { + sb.Header = new(types.BlockHeader) + *sb.Header = sb.Block.Header() + } + return *sb.Header, sb.Block, sb.Supplement, ok } -func (db *DBStore) putBlock(b types.Block, bs *consensus.V1BlockSupplement) { - id := b.ID() - db.bucket(bBlocks).put(id[:], supplementedBlock{b, bs}) +func (db *DBStore) putBlock(bh types.BlockHeader, b *types.Block, bs *consensus.V1BlockSupplement) { + id := bh.ID() + db.bucket(bBlocks).put(id[:], supplementedBlock{&bh, b, bs}) } -func (db *DBStore) getBlockHeader(id types.BlockID) (parentID types.BlockID, timestamp time.Time, _ bool) { - var sh supplementedHeader - ok := db.bucket(bBlocks).get(id[:], &sh) - return sh.ParentID, sh.Timestamp, ok +func (db *DBStore) getAncestorInfo(id types.BlockID) (parentID types.BlockID, timestamp time.Time, ok bool) { + ok = db.bucket(bBlocks).get(id[:], types.DecoderFunc(func(d *types.Decoder) { + v := d.ReadUint8() + if v != 2 && v != 3 { + d.SetErr(fmt.Errorf("incompatible version (%d)", v)) + } + // kinda cursed; don't worry about it + if v == 3 { + if !d.ReadBool() { + d.ReadBool() + } + } + parentID.DecodeFrom(d) + _ = d.ReadUint64() // nonce + timestamp = d.ReadTime() + })) + return +} + +func (db *DBStore) getBlockHeader(id types.BlockID) (bh types.BlockHeader, ok bool) { + ok = db.bucket(bBlocks).get(id[:], types.DecoderFunc(func(d *types.Decoder) { + v := d.ReadUint8() + if v != 2 && v != 3 { + d.SetErr(fmt.Errorf("incompatible version (%d)", v)) + return + } + if v == 3 { + bhp := &bh + types.DecodePtr(d, &bhp) + if bhp != nil { + return + } else if !d.ReadBool() { + d.SetErr(errors.New("neither header nor block present")) + return + } + } + var b types.Block + (*types.V2Block)(&b).DecodeFrom(d) + bh = b.Header() + })) + return } func (db *DBStore) treeKey(row, col uint64) []byte { @@ -628,9 +658,9 @@ func (db *DBStore) AncestorTimestamp(id types.BlockID) (t time.Time, ok bool) { } break } - ancestorID, _, _ = db.getBlockHeader(ancestorID) + ancestorID, _, _ = db.getAncestorInfo(ancestorID) } - _, t, ok = db.getBlockHeader(ancestorID) + _, t, ok = db.getAncestorInfo(ancestorID) return } @@ -646,12 +676,23 @@ func (db *DBStore) AddState(cs consensus.State) { // Block implements Store. func (db *DBStore) Block(id types.BlockID) (types.Block, *consensus.V1BlockSupplement, bool) { - return db.getBlock(id) + _, b, bs, ok := db.getBlock(id) + if !ok || b == nil { + return types.Block{}, nil, false + } + return *b, bs, ok } // AddBlock implements Store. func (db *DBStore) AddBlock(b types.Block, bs *consensus.V1BlockSupplement) { - db.putBlock(b, bs) + db.putBlock(b.Header(), &b, bs) +} + +// PruneBlock implements Store. +func (db *DBStore) PruneBlock(id types.BlockID) { + if bh, _, _, ok := db.getBlock(id); ok { + db.putBlock(bh, nil, nil) + } } func (db *DBStore) shouldFlush() bool { @@ -743,7 +784,7 @@ func NewDBStore(db DB, n *consensus.Network, genesisBlock types.Block) (_ *DBSto dbs.putState(genesisState) bs := consensus.V1BlockSupplement{Transactions: make([]consensus.V1TransactionSupplement, len(genesisBlock.Transactions))} cs, cau := consensus.ApplyBlock(genesisState, genesisBlock, bs, time.Time{}) - dbs.putBlock(genesisBlock, &bs) + dbs.putBlock(genesisBlock.Header(), &genesisBlock, &bs) dbs.putState(cs) dbs.ApplyBlock(cs, cau) if err := dbs.Flush(); err != nil { diff --git a/chain/manager.go b/chain/manager.go index 679b281..53e47da 100644 --- a/chain/manager.go +++ b/chain/manager.go @@ -45,6 +45,7 @@ type Store interface { Block(id types.BlockID) (types.Block, *consensus.V1BlockSupplement, bool) AddBlock(b types.Block, bs *consensus.V1BlockSupplement) + PruneBlock(id types.BlockID) State(id types.BlockID) (consensus.State, bool) AddState(cs consensus.State) AncestorTimestamp(id types.BlockID) (time.Time, bool) @@ -74,12 +75,15 @@ func blockAndChild(s Store, id types.BlockID) (types.Block, *consensus.V1BlockSu // A Manager tracks multiple blockchains and identifies the best valid // chain. type Manager struct { - log *zap.Logger store Store tipState consensus.State onReorg map[[16]byte]func(types.ChainIndex) invalidBlocks map[types.BlockID]error + // configuration options + log *zap.Logger + pruneTarget uint64 + txpool struct { txns []types.Transaction v2txns []types.V2Transaction @@ -314,6 +318,12 @@ func (m *Manager) applyTip(index types.ChainIndex) error { m.store.ApplyBlock(cs, cau) m.applyPoolUpdate(cau, cs) m.tipState = cs + + if m.pruneTarget != 0 && cs.Index.Height > m.pruneTarget { + if index, ok := m.store.BestIndex(cs.Index.Height - m.pruneTarget); ok { + m.store.PruneBlock(index.ID) + } + } return nil } diff --git a/chain/options.go b/chain/options.go index f7fa10a..05d5f6d 100644 --- a/chain/options.go +++ b/chain/options.go @@ -11,3 +11,10 @@ func WithLog(l *zap.Logger) ManagerOption { m.log = l } } + +// WithPruneTarget sets the target number of blocks to store. +func WithPruneTarget(n uint64) ManagerOption { + return func(m *Manager) { + m.pruneTarget = n + } +} diff --git a/miner.go b/miner.go index db4358a..e028d9d 100644 --- a/miner.go +++ b/miner.go @@ -10,15 +10,17 @@ import ( // FindBlockNonce attempts to find a nonce for b that meets the PoW target. func FindBlockNonce(cs consensus.State, b *types.Block, timeout time.Duration) bool { - b.Nonce = 0 + bh := b.Header() + bh.Nonce = 0 factor := cs.NonceFactor() startBlock := time.Now() - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += factor + for bh.ID().CmpWork(cs.ChildTarget) < 0 { + bh.Nonce += factor if time.Since(startBlock) > timeout { return false } } + b.Nonce = bh.Nonce return true } diff --git a/syncer/peer.go b/syncer/peer.go index 45a6d81..de98c86 100644 --- a/syncer/peer.go +++ b/syncer/peer.go @@ -182,8 +182,8 @@ func (p *Peer) SendCheckpoint(index types.ChainIndex, timeout time.Duration) (ty } // RelayV2Header relays a v2 block header to the peer. -func (p *Peer) RelayV2Header(h types.BlockHeader, timeout time.Duration) error { - return p.callRPC(&gateway.RPCRelayV2Header{Header: h}, timeout) +func (p *Peer) RelayV2Header(bh types.BlockHeader, timeout time.Duration) error { + return p.callRPC(&gateway.RPCRelayV2Header{Header: bh}, timeout) } // RelayV2BlockOutline relays a v2 block outline to the peer. diff --git a/syncer/syncer.go b/syncer/syncer.go index f1ac29b..6778d79 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -533,9 +533,9 @@ func (s *Syncer) peerLoop(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, s.config.ConnectTimeout) if _, err := s.Connect(ctx, p); err != nil { - log.Debug("connected to peer", zap.String("peer", p)) - } else { log.Debug("failed to connect to peer", zap.String("peer", p), zap.Error(err)) + } else { + log.Debug("connected to peer", zap.String("peer", p)) } cancel() lastTried[p] = time.Now() @@ -723,10 +723,12 @@ func (s *Syncer) Connect(ctx context.Context, addr string) (*Peer, error) { } // BroadcastHeader broadcasts a header to all peers. -func (s *Syncer) BroadcastHeader(h types.BlockHeader) { s.relayHeader(h, nil) } +func (s *Syncer) BroadcastHeader(bh types.BlockHeader) { s.relayHeader(bh, nil) } // BroadcastV2Header broadcasts a v2 header to all peers. -func (s *Syncer) BroadcastV2Header(h types.BlockHeader) { s.relayV2Header(h, nil) } +func (s *Syncer) BroadcastV2Header(bh types.BlockHeader) { + s.relayV2Header(bh, nil) +} // BroadcastV2BlockOutline broadcasts a v2 block outline to all peers. func (s *Syncer) BroadcastV2BlockOutline(b gateway.V2BlockOutline) { s.relayV2BlockOutline(b, nil) } From 1eba8522f72929d1c621e544fd4dfccf08fb6c79 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Fri, 8 Nov 2024 21:35:48 -0500 Subject: [PATCH 13/46] testutil: Rename MemPeerStore to EphemeralPeerStore --- rhp/v4/rpc_test.go | 2 +- syncer/syncer_test.go | 4 ++-- testutil/syncer.go | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index a869635..e264b1b 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -74,7 +74,7 @@ func startTestNode(tb testing.TB, n *consensus.Network, genesis types.Block) (*c } tb.Cleanup(func() { syncerListener.Close() }) - s := syncer.New(syncerListener, cm, testutil.NewMemPeerStore(), gateway.Header{ + s := syncer.New(syncerListener, cm, testutil.NewEphemeralPeerStore(), gateway.Header{ GenesisID: genesis.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: "localhost:1234", diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index d3551a0..eb5d673 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -42,7 +42,7 @@ func TestSyncer(t *testing.T) { } defer l2.Close() - s1 := syncer.New(l1, cm1, testutil.NewMemPeerStore(), gateway.Header{ + s1 := syncer.New(l1, cm1, testutil.NewEphemeralPeerStore(), gateway.Header{ GenesisID: genesis.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: l1.Addr().String(), @@ -50,7 +50,7 @@ func TestSyncer(t *testing.T) { defer s1.Close() go s1.Run(context.Background()) - s2 := syncer.New(l2, cm2, testutil.NewMemPeerStore(), gateway.Header{ + s2 := syncer.New(l2, cm2, testutil.NewEphemeralPeerStore(), gateway.Header{ GenesisID: genesis.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: l2.Addr().String(), diff --git a/testutil/syncer.go b/testutil/syncer.go index 1aba4f6..caa621a 100644 --- a/testutil/syncer.go +++ b/testutil/syncer.go @@ -7,15 +7,15 @@ import ( "go.sia.tech/coreutils/syncer" ) -// A MemPeerStore is an in-memory implementation of a PeerStore. -type MemPeerStore struct { +// A EphemeralPeerStore is an in-memory implementation of a PeerStore. +type EphemeralPeerStore struct { mu sync.Mutex peers map[string]syncer.PeerInfo } // AddPeer adds a peer to the store. If the peer already exists, nil should // be returned. -func (ps *MemPeerStore) AddPeer(addr string) error { +func (ps *EphemeralPeerStore) AddPeer(addr string) error { ps.mu.Lock() defer ps.mu.Unlock() if _, ok := ps.peers[addr]; ok { @@ -26,7 +26,7 @@ func (ps *MemPeerStore) AddPeer(addr string) error { } // Peers returns the set of known peers. -func (ps *MemPeerStore) Peers() ([]syncer.PeerInfo, error) { +func (ps *EphemeralPeerStore) Peers() ([]syncer.PeerInfo, error) { ps.mu.Lock() defer ps.mu.Unlock() var peers []syncer.PeerInfo @@ -38,7 +38,7 @@ func (ps *MemPeerStore) Peers() ([]syncer.PeerInfo, error) { // PeerInfo returns the metadata for the specified peer or ErrPeerNotFound // if the peer wasn't found in the store. -func (ps *MemPeerStore) PeerInfo(addr string) (syncer.PeerInfo, error) { +func (ps *EphemeralPeerStore) PeerInfo(addr string) (syncer.PeerInfo, error) { ps.mu.Lock() defer ps.mu.Unlock() p, ok := ps.peers[addr] @@ -50,7 +50,7 @@ func (ps *MemPeerStore) PeerInfo(addr string) (syncer.PeerInfo, error) { // UpdatePeerInfo updates the metadata for the specified peer. If the peer // is not found, the error should be ErrPeerNotFound. -func (ps *MemPeerStore) UpdatePeerInfo(addr string, fn func(*syncer.PeerInfo)) error { +func (ps *EphemeralPeerStore) UpdatePeerInfo(addr string, fn func(*syncer.PeerInfo)) error { ps.mu.Lock() defer ps.mu.Unlock() p := ps.peers[addr] @@ -61,18 +61,18 @@ func (ps *MemPeerStore) UpdatePeerInfo(addr string, fn func(*syncer.PeerInfo)) e // Ban temporarily bans one or more IPs. The addr should either be a single // IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. 1.2.3.4/16). -func (ps *MemPeerStore) Ban(addr string, duration time.Duration, reason string) error { +func (ps *EphemeralPeerStore) Ban(addr string, duration time.Duration, reason string) error { return nil } // Banned returns false -func (ps *MemPeerStore) Banned(addr string) (bool, error) { return false, nil } +func (ps *EphemeralPeerStore) Banned(addr string) (bool, error) { return false, nil } -var _ syncer.PeerStore = (*MemPeerStore)(nil) +var _ syncer.PeerStore = (*EphemeralPeerStore)(nil) -// NewMemPeerStore returns a new MemPeerStore. -func NewMemPeerStore() *MemPeerStore { - return &MemPeerStore{ +// NewEphemeralPeerStore returns a new EphemeralPeerStore. +func NewEphemeralPeerStore() *EphemeralPeerStore { + return &EphemeralPeerStore{ peers: make(map[string]syncer.PeerInfo), } } From 50054723a0ee23c73831f484315eaa2905a2f1ec Mon Sep 17 00:00:00 2001 From: "knope-bot[bot]" <152252888+knope-bot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 01:47:02 +0000 Subject: [PATCH 14/46] Auto generate changeset --- .changeset/add_support_for_block_pruning.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .changeset/add_support_for_block_pruning.md diff --git a/.changeset/add_support_for_block_pruning.md b/.changeset/add_support_for_block_pruning.md new file mode 100644 index 0000000..9f37597 --- /dev/null +++ b/.changeset/add_support_for_block_pruning.md @@ -0,0 +1,17 @@ +--- +default: minor +--- + +# Add support for block pruning + +#116 by @lukechampine + +Been a long time coming. 😅 + +The strategy here is quite naive, but I think it will be serviceable. Basically, when we apply a block `N`, we delete block `N-P`. `P` is therefore the "prune target," i.e. the maximum number of blocks you want to store. + +In practice, this isn't exhaustive: it only deletes blocks from the best chain. It also won't dramatically shrink the size of an existing database. I think this is acceptable, because pruning is most important during the initial sync, and during the initial sync, you'll only be receiving blocks from one chain at a time. Also, we don't want to make pruning *too* easy; after all, we need a good percentage of nodes to be storing the full chain, so that others can sync to them. + +I tested this out locally with a prune target of 1000, and after syncing 400,000 blocks, my `consensus.db` was around 18 GB. This is disappointing; it should be much smaller. With some investigation, I found that the Bolt database was only storing ~5 GB of data (most of which was the accumulator tree, which we can't prune until after v2). I think this is a combination of a) Bolt grows the DB capacity aggressively in response to writes, and b) Bolt never shrinks the DB capacity. So it's possible that we could reduce this number by tweaking our DB batching parameters. Alternatively, we could provide a tool that copies the DB to a new file. Not the most user-friendly, but again, I think I'm okay with that for now. + +Depends on https://github.com/SiaFoundation/core/pull/228 From a349c353bcb65244244b3615664f1219643e889e Mon Sep 17 00:00:00 2001 From: Luke Champine Date: Sat, 7 Dec 2024 09:10:26 -0500 Subject: [PATCH 15/46] Update add_support_for_block_pruning.md --- .changeset/add_support_for_block_pruning.md | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.changeset/add_support_for_block_pruning.md b/.changeset/add_support_for_block_pruning.md index 9f37597..95fe1ad 100644 --- a/.changeset/add_support_for_block_pruning.md +++ b/.changeset/add_support_for_block_pruning.md @@ -1,17 +1,7 @@ ---- -default: minor ---- - -# Add support for block pruning - -#116 by @lukechampine - -Been a long time coming. 😅 +--- +default: minor +--- -The strategy here is quite naive, but I think it will be serviceable. Basically, when we apply a block `N`, we delete block `N-P`. `P` is therefore the "prune target," i.e. the maximum number of blocks you want to store. +# Add support for block pruning -In practice, this isn't exhaustive: it only deletes blocks from the best chain. It also won't dramatically shrink the size of an existing database. I think this is acceptable, because pruning is most important during the initial sync, and during the initial sync, you'll only be receiving blocks from one chain at a time. Also, we don't want to make pruning *too* easy; after all, we need a good percentage of nodes to be storing the full chain, so that others can sync to them. - -I tested this out locally with a prune target of 1000, and after syncing 400,000 blocks, my `consensus.db` was around 18 GB. This is disappointing; it should be much smaller. With some investigation, I found that the Bolt database was only storing ~5 GB of data (most of which was the accumulator tree, which we can't prune until after v2). I think this is a combination of a) Bolt grows the DB capacity aggressively in response to writes, and b) Bolt never shrinks the DB capacity. So it's possible that we could reduce this number by tweaking our DB batching parameters. Alternatively, we could provide a tool that copies the DB to a new file. Not the most user-friendly, but again, I think I'm okay with that for now. - -Depends on https://github.com/SiaFoundation/core/pull/228 +The chain manager can now automatically delete blocks after a configurable number of confirmations. Note that this does not apply retroactively. From a28e868d07995cc1a917b3bd75e6645653c75901 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:27:20 +0000 Subject: [PATCH 16/46] build(deps): bump golang.org/x/crypto from 0.29.0 to 0.30.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.29.0 to 0.30.0. - [Commits](https://github.com/golang/crypto/compare/v0.29.0...v0.30.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index dfbeb29..dcbed28 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,11 @@ require ( go.sia.tech/core v0.7.1 go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.29.0 + golang.org/x/crypto v0.30.0 lukechampine.com/frand v1.5.1 ) require ( go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sys v0.27.0 // indirect + golang.org/x/sys v0.28.0 // indirect ) diff --git a/go.sum b/go.sum index 50c98dc..b822a3c 100644 --- a/go.sum +++ b/go.sum @@ -16,12 +16,12 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/frand v1.5.1 h1:fg0eRtdmGFIxhP5zQJzM1lFDbD6CUfu/f+7WgAZd5/w= From 6881993d88064a814b8856d584e185b62e7bccfc Mon Sep 17 00:00:00 2001 From: Christopher Schinnerl Date: Wed, 11 Dec 2024 05:55:14 +0100 Subject: [PATCH 17/46] Fix panic when fetching block with empty block id from ChainManager (#133) --- ...k_with_empty_block_id_from_chainmanager.md | 5 +++++ chain/db.go | 7 ++++--- chain/db_test.go | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 .changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md create mode 100644 chain/db_test.go diff --git a/.changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md b/.changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md new file mode 100644 index 0000000..ba2f304 --- /dev/null +++ b/.changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Fix panic when fetching block with empty block id from ChainManager diff --git a/chain/db.go b/chain/db.go index 466acfa..eb78fa9 100644 --- a/chain/db.go +++ b/chain/db.go @@ -295,12 +295,13 @@ func (db *DBStore) putState(cs consensus.State) { func (db *DBStore) getBlock(id types.BlockID) (bh types.BlockHeader, b *types.Block, bs *consensus.V1BlockSupplement, _ bool) { var sb supplementedBlock - ok := db.bucket(bBlocks).get(id[:], &sb) - if sb.Header == nil { + if ok := db.bucket(bBlocks).get(id[:], &sb); !ok { + return types.BlockHeader{}, nil, nil, false + } else if sb.Header == nil { sb.Header = new(types.BlockHeader) *sb.Header = sb.Block.Header() } - return *sb.Header, sb.Block, sb.Supplement, ok + return *sb.Header, sb.Block, sb.Supplement, true } func (db *DBStore) putBlock(bh types.BlockHeader, b *types.Block, bs *consensus.V1BlockSupplement) { diff --git a/chain/db_test.go b/chain/db_test.go new file mode 100644 index 0000000..f30f6de --- /dev/null +++ b/chain/db_test.go @@ -0,0 +1,19 @@ +package chain_test + +import ( + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/testutil" +) + +func TestGetEmptyBlockID(t *testing.T) { + n, genesisBlock := testutil.V2Network() + store, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, tipState) + _, _ = cm.Block(types.BlockID{}) +} From 571d72e958da9a460f2927a6ef7e50a127a239aa Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 11 Dec 2024 09:10:45 -0500 Subject: [PATCH 18/46] chain: Return deep copies of v2 pool transactions --- chain/manager.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/chain/manager.go b/chain/manager.go index 53e47da..dabdae3 100644 --- a/chain/manager.go +++ b/chain/manager.go @@ -826,7 +826,7 @@ func (m *Manager) V2PoolTransaction(id types.TransactionID) (types.V2Transaction if !ok { return types.V2Transaction{}, false } - return m.txpool.v2txns[i], ok + return m.txpool.v2txns[i].DeepCopy(), ok } // V2PoolTransactions returns the v2 transactions currently in the txpool. Any @@ -835,7 +835,11 @@ func (m *Manager) V2PoolTransactions() []types.V2Transaction { m.mu.Lock() defer m.mu.Unlock() m.revalidatePool() - return append([]types.V2Transaction(nil), m.txpool.v2txns...) + v2txns := make([]types.V2Transaction, len(m.txpool.v2txns)) + for i, txn := range m.txpool.v2txns { + v2txns[i] = txn.DeepCopy() + } + return v2txns } // TransactionsForPartialBlock returns the transactions in the txpool with the From ee169830e1e678d213c1b2dd9d7519ed1c814516 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 10 Dec 2024 15:14:52 +0100 Subject: [PATCH 19/46] v4: update TestRPCRenew to upload data before renewing --- rhp/v4/rpc_test.go | 48 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index e264b1b..a54ea37 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -355,7 +355,7 @@ func TestRPCRefresh(t *testing.T) { } fundAndSign := &fundAndSign{w, renterKey} - formContractFundAccount := func(t *testing.T, renterAllowance, hostCollateral, accountBalance types.Currency) rhp4.ContractRevision { + formContractUploadSector := func(t *testing.T, renterAllowance, hostCollateral, accountBalance types.Currency) rhp4.ContractRevision { t.Helper() result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ @@ -390,11 +390,27 @@ func TestRPCRefresh(t *testing.T) { t.Fatal(err) } revision.Revision = fundResult.Revision + + // upload data + at := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(5 * time.Minute), + } + at.Signature = renterKey.SignHash(at.SigHash()) + wRes, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, at, bytes.NewReader(bytes.Repeat([]byte{1}, proto4.LeafSize)), proto4.LeafSize) + if err != nil { + t.Fatal(err) + } + aRes, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, []types.Hash256{wRes.Root}) + if err != nil { + t.Fatal(err) + } + revision.Revision = aRes.Revision return revision } t.Run("no allowance or collateral", func(t *testing.T) { - revision := formContractFundAccount(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) // refresh the contract _, err = rhp4.RPCRefreshContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRefreshContractParams{ @@ -410,7 +426,7 @@ func TestRPCRefresh(t *testing.T) { }) t.Run("valid refresh", func(t *testing.T) { - revision := formContractFundAccount(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) // refresh the contract refreshResult, err := rhp4.RPCRefreshContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRefreshContractParams{ ContractID: revision.ID, @@ -473,7 +489,7 @@ func TestRPCRenew(t *testing.T) { } fundAndSign := &fundAndSign{w, renterKey} - formContractFundAccount := func(t *testing.T, renterAllowance, hostCollateral, accountBalance types.Currency) rhp4.ContractRevision { + formContractUploadSector := func(t *testing.T, renterAllowance, hostCollateral, accountBalance types.Currency) rhp4.ContractRevision { t.Helper() result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ @@ -514,11 +530,27 @@ func TestRPCRenew(t *testing.T) { t.Fatal(err) } revision.Revision = fundResult.Revision + + // upload data + at := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(5 * time.Minute), + } + at.Signature = renterKey.SignHash(at.SigHash()) + wRes, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, at, bytes.NewReader(bytes.Repeat([]byte{1}, proto4.LeafSize)), proto4.LeafSize) + if err != nil { + t.Fatal(err) + } + aRes, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, []types.Hash256{wRes.Root}) + if err != nil { + t.Fatal(err) + } + revision.Revision = aRes.Revision return revision } t.Run("same duration", func(t *testing.T) { - revision := formContractFundAccount(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) // renew the contract _, err = rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ @@ -535,7 +567,7 @@ func TestRPCRenew(t *testing.T) { }) t.Run("partial rollover", func(t *testing.T) { - revision := formContractFundAccount(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) // renew the contract renewResult, err := rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ @@ -564,7 +596,7 @@ func TestRPCRenew(t *testing.T) { }) t.Run("full rollover", func(t *testing.T) { - revision := formContractFundAccount(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) // renew the contract renewResult, err := rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ @@ -593,7 +625,7 @@ func TestRPCRenew(t *testing.T) { }) t.Run("no rollover", func(t *testing.T) { - revision := formContractFundAccount(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) // renew the contract renewResult, err := rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ From 244ce91508c887f5e442d12e37c2aaed2ee8326a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 11 Dec 2024 09:05:22 +0100 Subject: [PATCH 20/46] update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dcbed28..2b13f03 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.7.1 + go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.30.0 diff --git a/go.sum b/go.sum index b822a3c..ef2af08 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.1 h1:PrKh19Ql5vJbQbB5YGtTHQ8W3fRF8hhYnR4kPOIOIME= -go.sia.tech/core v0.7.1/go.mod h1:gB8iXFJFSV8XIHRaL00CL6Be+hyykB+SYnvRPHCCc/E= +go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb h1:JHX+qWKS9sAXmEroICAu2jPQkr3CYUF7iWd/zlATsBM= +go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb/go.mod h1:tM9tPD+1jp8d+dqVpX6idTyv5RpI0GEh5L5SyuOpqlc= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= From 7a783051a4bdfccc87c7d92cef92b7dda67b2bd0 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 11 Dec 2024 09:57:55 +0100 Subject: [PATCH 21/46] add changeset --- ...renew_and_testrpcrefresh_with_an_initial_sector_upload.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md diff --git a/.changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md b/.changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md new file mode 100644 index 0000000..1211c12 --- /dev/null +++ b/.changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Extended TestRPCRenew and TestRPCRefresh with an initial sector upload From 85dd0252d9ad90bc23caf48e6febd381027a7a4b Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 09:22:36 +0100 Subject: [PATCH 22/46] update core dependency to tagged version --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2b13f03..0a0dde7 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb + go.sia.tech/core v0.7.2 go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.30.0 + golang.org/x/crypto v0.31.0 lukechampine.com/frand v1.5.1 ) diff --git a/go.sum b/go.sum index ef2af08..8dfe940 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb h1:JHX+qWKS9sAXmEroICAu2jPQkr3CYUF7iWd/zlATsBM= -go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb/go.mod h1:tM9tPD+1jp8d+dqVpX6idTyv5RpI0GEh5L5SyuOpqlc= +go.sia.tech/core v0.7.2 h1:GAsZ77LE592VEBGNdKeXLV4old/zjLjH11RblHhYbP4= +go.sia.tech/core v0.7.2/go.mod h1:pRlqaLm8amh3b/OBTSqJMEXmhPT14RxjntlKPySRNpA= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -16,8 +16,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= From ca211b2c910bb3358322f29046e972ecf7c45e06 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 12 Dec 2024 21:40:51 -0800 Subject: [PATCH 23/46] feat(rhp4): Add revised and renewed fields to RPCLatestRevision --- go.mod | 2 +- go.sum | 4 ++-- rhp/v4/options.go | 8 ------- rhp/v4/rpc.go | 7 +++--- rhp/v4/rpc_test.go | 53 +++++++++++++++++++++++++++++++++++++++++++--- rhp/v4/server.go | 33 ++++++++++++++--------------- testutil/host.go | 13 ++++++------ 7 files changed, 79 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index 0a0dde7..ee539a6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.7.2 + go.sia.tech/core v0.7.4-0.20241212191304-8383d656cbed go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 diff --git a/go.sum b/go.sum index 8dfe940..5c3ba44 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.2 h1:GAsZ77LE592VEBGNdKeXLV4old/zjLjH11RblHhYbP4= -go.sia.tech/core v0.7.2/go.mod h1:pRlqaLm8amh3b/OBTSqJMEXmhPT14RxjntlKPySRNpA= +go.sia.tech/core v0.7.4-0.20241212191304-8383d656cbed h1:L3HgQFyW34ZBlhtUjPzdzeBCLt+v2CAmDy/6BN1PXl0= +go.sia.tech/core v0.7.4-0.20241212191304-8383d656cbed/go.mod h1:nKaUneURNBl2ATp3dWOd1VS7Oe6HiXzUGMIBA83uUGM= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/rhp/v4/options.go b/rhp/v4/options.go index d668025..d265995 100644 --- a/rhp/v4/options.go +++ b/rhp/v4/options.go @@ -13,11 +13,3 @@ func WithPriceTableValidity(validity time.Duration) ServerOption { s.priceTableValidity = validity } } - -// WithContractProofWindowBuffer sets the buffer for revising a contract before -// its proof window starts. -func WithContractProofWindowBuffer(buffer uint64) ServerOption { - return func(s *Server) { - s.contractProofWindowBuffer = buffer - } -} diff --git a/rhp/v4/rpc.go b/rhp/v4/rpc.go index 38d6320..1600453 100644 --- a/rhp/v4/rpc.go +++ b/rhp/v4/rpc.go @@ -474,11 +474,10 @@ func RPCFundAccounts(ctx context.Context, t TransportClient, cs consensus.State, } // RPCLatestRevision returns the latest revision of a contract. -func RPCLatestRevision(ctx context.Context, t TransportClient, contractID types.FileContractID) (types.V2FileContract, error) { +func RPCLatestRevision(ctx context.Context, t TransportClient, contractID types.FileContractID) (resp rhp4.RPCLatestRevisionResponse, err error) { req := rhp4.RPCLatestRevisionRequest{ContractID: contractID} - var resp rhp4.RPCLatestRevisionResponse - err := callSingleRoundtripRPC(ctx, t, rhp4.RPCLatestRevisionID, &req, &resp) - return resp.Contract, err + err = callSingleRoundtripRPC(ctx, t, rhp4.RPCLatestRevisionID, &req, &resp) + return } // RPCSectorRoots returns the sector roots for a contract. diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index a54ea37..266b06c 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -49,7 +49,7 @@ func (fs *fundAndSign) Address() types.Address { } func testRenterHostPair(tb testing.TB, hostKey types.PrivateKey, cm rhp4.ChainManager, s rhp4.Syncer, w rhp4.Wallet, c rhp4.Contractor, sr rhp4.Settings, ss rhp4.Sectors, log *zap.Logger) rhp4.TransportClient { - rs := rhp4.NewServer(hostKey, cm, s, c, w, sr, ss, rhp4.WithContractProofWindowBuffer(10), rhp4.WithPriceTableValidity(2*time.Minute)) + rs := rhp4.NewServer(hostKey, cm, s, c, w, sr, ss, rhp4.WithPriceTableValidity(2*time.Minute)) hostAddr := testutil.ServeSiaMux(tb, rs, log.Named("siamux")) transport, err := rhp4.DialSiaMux(context.Background(), hostAddr, hostKey.PublicKey()) @@ -406,6 +406,15 @@ func TestRPCRefresh(t *testing.T) { t.Fatal(err) } revision.Revision = aRes.Revision + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if rs.Renewed { + t.Fatal("expected contract to not be renewed") + } else if !rs.Revisable { + t.Fatal("expected contract to be revisable") + } return revision } @@ -450,6 +459,15 @@ func TestRPCRefresh(t *testing.T) { } else if !hostKey.PublicKey().VerifyHash(sigHash, refreshResult.Contract.Revision.HostSignature) { t.Fatal("host signature verification failed") } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } }) } @@ -593,6 +611,15 @@ func TestRPCRenew(t *testing.T) { } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { t.Fatal("host signature verification failed") } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } }) t.Run("full rollover", func(t *testing.T) { @@ -622,6 +649,15 @@ func TestRPCRenew(t *testing.T) { } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { t.Fatal("host signature verification failed") } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } }) t.Run("no rollover", func(t *testing.T) { @@ -651,6 +687,15 @@ func TestRPCRenew(t *testing.T) { } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { t.Fatal("host signature verification failed") } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } }) } @@ -907,10 +952,12 @@ func TestAppendSectors(t *testing.T) { assertLastRevision := func(t *testing.T) { t.Helper() - lastRev, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) if err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(lastRev, revision.Revision) { + } + lastRev := rs.Contract + if !reflect.DeepEqual(lastRev, revision.Revision) { t.Log(lastRev) t.Log(revision.Revision) t.Fatalf("expected last revision to match") diff --git a/rhp/v4/server.go b/rhp/v4/server.go index 16f6ec2..3936878 100644 --- a/rhp/v4/server.go +++ b/rhp/v4/server.go @@ -88,8 +88,10 @@ type ( // A RevisionState pairs a contract revision with its sector roots. RevisionState struct { - Revision types.V2FileContract - Roots []types.Hash256 + Revision types.V2FileContract + Renewed bool + Revisable bool + Roots []types.Hash256 } // Contractor is an interface for managing a host's contracts. @@ -123,9 +125,8 @@ type ( // A Server handles incoming RHP4 RPC. Server struct { - hostKey types.PrivateKey - priceTableValidity time.Duration - contractProofWindowBuffer uint64 + hostKey types.PrivateKey + priceTableValidity time.Duration chain ChainManager syncer Syncer @@ -136,18 +137,15 @@ type ( } ) -func (s *Server) lockContractForRevision(contractID types.FileContractID) (rev RevisionState, unlock func(), _ error) { - rev, unlock, err := s.contractor.LockV2Contract(contractID) +func (s *Server) lockContractForRevision(contractID types.FileContractID) (RevisionState, func(), error) { + rs, unlock, err := s.contractor.LockV2Contract(contractID) if err != nil { return RevisionState{}, nil, fmt.Errorf("failed to lock contract: %w", err) - } else if rev.Revision.ProofHeight <= s.chain.Tip().Height+s.contractProofWindowBuffer { + } else if !rs.Revisable { unlock() - return RevisionState{}, nil, errorBadRequest("contract too close to proof window") - } else if rev.Revision.RevisionNumber >= types.MaxRevisionNumber { - unlock() - return RevisionState{}, nil, errorBadRequest("contract is locked for revision") + return RevisionState{}, nil, errorBadRequest("contract is not revisable") } - return rev, unlock, nil + return rs, unlock, nil } func (s *Server) handleRPCSettings(stream net.Conn) error { @@ -451,7 +449,9 @@ func (s *Server) handleRPCLatestRevision(stream net.Conn) error { unlock() return rhp4.WriteResponse(stream, &rhp4.RPCLatestRevisionResponse{ - Contract: state.Revision, + Contract: state.Revision, + Revisable: state.Revisable, + Renewed: state.Renewed, }) } @@ -1118,9 +1118,8 @@ func errorDecodingError(f string, p ...any) error { // NewServer creates a new RHP4 server func NewServer(pk types.PrivateKey, cm ChainManager, syncer Syncer, contracts Contractor, wallet Wallet, settings Settings, sectors Sectors, opts ...ServerOption) *Server { s := &Server{ - hostKey: pk, - priceTableValidity: 30 * time.Minute, - contractProofWindowBuffer: 10, + hostKey: pk, + priceTableValidity: 30 * time.Minute, chain: cm, syncer: syncer, diff --git a/testutil/host.go b/testutil/host.go index b5b4407..5fb8bc5 100644 --- a/testutil/host.go +++ b/testutil/host.go @@ -93,10 +93,14 @@ func (ec *EphemeralContractor) LockV2Contract(contractID types.FileContractID) ( return rhp4.RevisionState{}, nil, errors.New("contract not found") } + _, renewed := ec.contracts[contractID.V2RenewalID()] + var once sync.Once return rhp4.RevisionState{ - Revision: rev, - Roots: ec.roots[contractID], + Revision: rev, + Revisable: !renewed && ec.tip.Height < rev.ProofHeight, + Renewed: renewed, + Roots: ec.roots[contractID], }, func() { once.Do(func() { ec.mu.Lock() @@ -159,8 +163,6 @@ func (ec *EphemeralContractor) RenewV2Contract(renewalSet rhp4.TransactionSet, _ existing, ok := ec.contracts[existingID] if !ok { return errors.New("contract not found") - } else if existing.RevisionNumber == types.MaxRevisionNumber { - return errors.New("contract already at max revision") } contractID := existingID.V2RenewalID() @@ -175,7 +177,6 @@ func (ec *EphemeralContractor) RenewV2Contract(renewalSet rhp4.TransactionSet, _ return errors.New("invalid host signature") } - delete(ec.contracts, existingID) // remove the existing contract ec.contracts[contractID] = renewal.NewContract ec.roots[contractID] = append([]types.Hash256(nil), ec.roots[existingID]...) return nil @@ -210,7 +211,7 @@ func (ec *EphemeralContractor) ReviseV2Contract(contractID types.FileContractID, func (ec *EphemeralContractor) AccountBalance(account proto4.Account) (types.Currency, error) { ec.mu.Lock() defer ec.mu.Unlock() - balance, _ := ec.accounts[account] + balance := ec.accounts[account] return balance, nil } From 1a942629f4c24c174238ade98f76f12af3d94b16 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 13 Dec 2024 08:13:19 -0800 Subject: [PATCH 24/46] update core --- .../add_revised_and_renewed_fields_to_rpclatestrevision.md | 7 +++++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 .changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md diff --git a/.changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md b/.changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md new file mode 100644 index 0000000..c70a89b --- /dev/null +++ b/.changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md @@ -0,0 +1,7 @@ +--- +default: major +--- + +# Add revised and renewed fields to RPCLatestRevision + +Adds two additional fields to the RPCLatestRevision response. The Revisable field indicates whether the host will accept further revisions to the contract. A host will not accept revisions too close to the proof window or revisions on contracts that have already been resolved. The Renewed field indicates whether the contract was renewed. If the contract was renewed, the renter can use FileContractID.V2RenewalID to get the ID of the new contract. diff --git a/go.mod b/go.mod index ee539a6..bf49f04 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.7.4-0.20241212191304-8383d656cbed + go.sia.tech/core v0.8.0 go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 diff --git a/go.sum b/go.sum index 5c3ba44..25118a3 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.4-0.20241212191304-8383d656cbed h1:L3HgQFyW34ZBlhtUjPzdzeBCLt+v2CAmDy/6BN1PXl0= -go.sia.tech/core v0.7.4-0.20241212191304-8383d656cbed/go.mod h1:nKaUneURNBl2ATp3dWOd1VS7Oe6HiXzUGMIBA83uUGM= +go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg= +go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= From 885eff9914ab9da577d74161ce478f42c4f3241f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:16:04 +0000 Subject: [PATCH 25/46] chore: prepare release 0.8.0 --- ...and_renewed_fields_to_rpclatestrevision.md | 7 ------ .changeset/add_support_for_block_pruning.md | 7 ------ .changeset/automate_releases.md | 5 ---- ...pcrefresh_with_an_initial_sector_upload.md | 5 ---- ...k_with_empty_block_id_from_chainmanager.md | 5 ---- CHANGELOG.md | 23 +++++++++++++++++++ go.mod | 2 +- 7 files changed, 24 insertions(+), 30 deletions(-) delete mode 100644 .changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md delete mode 100644 .changeset/add_support_for_block_pruning.md delete mode 100644 .changeset/automate_releases.md delete mode 100644 .changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md delete mode 100644 .changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md create mode 100644 CHANGELOG.md diff --git a/.changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md b/.changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md deleted file mode 100644 index c70a89b..0000000 --- a/.changeset/add_revised_and_renewed_fields_to_rpclatestrevision.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: major ---- - -# Add revised and renewed fields to RPCLatestRevision - -Adds two additional fields to the RPCLatestRevision response. The Revisable field indicates whether the host will accept further revisions to the contract. A host will not accept revisions too close to the proof window or revisions on contracts that have already been resolved. The Renewed field indicates whether the contract was renewed. If the contract was renewed, the renter can use FileContractID.V2RenewalID to get the ID of the new contract. diff --git a/.changeset/add_support_for_block_pruning.md b/.changeset/add_support_for_block_pruning.md deleted file mode 100644 index 95fe1ad..0000000 --- a/.changeset/add_support_for_block_pruning.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: minor ---- - -# Add support for block pruning - -The chain manager can now automatically delete blocks after a configurable number of confirmations. Note that this does not apply retroactively. diff --git a/.changeset/automate_releases.md b/.changeset/automate_releases.md deleted file mode 100644 index d5fea6e..0000000 --- a/.changeset/automate_releases.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Automate releases diff --git a/.changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md b/.changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md deleted file mode 100644 index 1211c12..0000000 --- a/.changeset/extended_testrpcrenew_and_testrpcrefresh_with_an_initial_sector_upload.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Extended TestRPCRenew and TestRPCRefresh with an initial sector upload diff --git a/.changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md b/.changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md deleted file mode 100644 index ba2f304..0000000 --- a/.changeset/fix_panic_when_fetching_block_with_empty_block_id_from_chainmanager.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Fix panic when fetching block with empty block id from ChainManager diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dbcc397 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +## 0.8.0 (2024-12-13) + +### Breaking Changes + +#### Add revised and renewed fields to RPCLatestRevision + +Adds two additional fields to the RPCLatestRevision response. The Revisable field indicates whether the host will accept further revisions to the contract. A host will not accept revisions too close to the proof window or revisions on contracts that have already been resolved. The Renewed field indicates whether the contract was renewed. If the contract was renewed, the renter can use FileContractID.V2RenewalID to get the ID of the new contract. + +### Features + +- Add revised and renewed fields to RPCLatestRevision + +#### Add support for block pruning + +The chain manager can now automatically delete blocks after a configurable number of confirmations. Note that this does not apply retroactively. + +### Fixes + +- remove duration param from RPCWrite impl +- Return 0 for nonexistent accounts +- Automate releases +- Extended TestRPCRenew and TestRPCRefresh with an initial sector upload +- Fix panic when fetching block with empty block id from ChainManager diff --git a/go.mod b/go.mod index bf49f04..19475c9 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/coreutils // v0.7.0 +module go.sia.tech/coreutils // v0.8.0 go 1.23.1 From 2b2624c92f4227acb8c425834c4e298b6b099830 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 13 Dec 2024 08:18:15 -0800 Subject: [PATCH 26/46] Update CHANGELOG.md --- CHANGELOG.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbcc397..603c4c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,18 +6,16 @@ Adds two additional fields to the RPCLatestRevision response. The Revisable field indicates whether the host will accept further revisions to the contract. A host will not accept revisions too close to the proof window or revisions on contracts that have already been resolved. The Renewed field indicates whether the contract was renewed. If the contract was renewed, the renter can use FileContractID.V2RenewalID to get the ID of the new contract. -### Features +- Remove unused duration param from `rhp4.RPCWrite` -- Add revised and renewed fields to RPCLatestRevision +### Features -#### Add support for block pruning +#### Add support for block pruning in v2 The chain manager can now automatically delete blocks after a configurable number of confirmations. Note that this does not apply retroactively. ### Fixes -- remove duration param from RPCWrite impl -- Return 0 for nonexistent accounts -- Automate releases +- Return 0 balance for nonexistent accounts instead of an error - Extended TestRPCRenew and TestRPCRefresh with an initial sector upload - Fix panic when fetching block with empty block id from ChainManager From 5d6fc37cbb944926f0509ef3e37916a21e7721ee Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 11:12:24 +0100 Subject: [PATCH 27/46] fix(rhp): Return ErrNotEnoughFunds when account doesn't exist or has insufficient funds in it --- ...s_when_account_has_insufficient_balance.md | 5 +++++ rhp/v4/rpc_test.go | 20 +++++++++++++++++++ testutil/host.go | 6 ++---- 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 .changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md diff --git a/.changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md b/.changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md new file mode 100644 index 0000000..1792e57 --- /dev/null +++ b/.changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Fix rhp4 server not returning ErrNotEnoughFunds when account has insufficient balance diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index 266b06c..38a52d5 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -752,6 +752,19 @@ func TestAccounts(t *testing.T) { cs := cm.TipState() + // test operations against unknown account + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + + tokenSigHash := token.SigHash() + token.Signature = renterKey.SignHash(tokenSigHash) + _, err = rhp4.RPCVerifySector(context.Background(), transport, settings.Prices, token, types.Hash256{1}) + if err == nil || !strings.Contains(err.Error(), proto4.ErrNotEnoughFunds.Error()) { + t.Fatal(err) + } + balance, err := rhp4.RPCAccountBalance(context.Background(), transport, account) if err != nil { t.Fatal(err) @@ -803,6 +816,13 @@ func TestAccounts(t *testing.T) { } else if !balance.Equals(accountFundAmount) { t.Fatalf("expected %v, got %v", accountFundAmount, balance) } + + // drain account and try using it + _ = c.DebitAccount(account, proto4.Usage{RPC: accountFundAmount}) + _, err = rhp4.RPCVerifySector(context.Background(), transport, settings.Prices, token, types.Hash256{1}) + if err == nil || !strings.Contains(err.Error(), proto4.ErrNotEnoughFunds.Error()) { + t.Fatal(err) + } } func TestReadWriteSector(t *testing.T) { diff --git a/testutil/host.go b/testutil/host.go index 5fb8bc5..f22b617 100644 --- a/testutil/host.go +++ b/testutil/host.go @@ -252,10 +252,8 @@ func (ec *EphemeralContractor) DebitAccount(account proto4.Account, usage proto4 defer ec.mu.Unlock() balance, ok := ec.accounts[account] - if !ok { - return errors.New("account not found") - } else if balance.Cmp(usage.RenterCost()) < 0 { - return errors.New("insufficient funds") + if !ok || balance.Cmp(usage.RenterCost()) < 0 { + return proto4.ErrNotEnoughFunds } ec.accounts[account] = balance.Sub(usage.RenterCost()) return nil From 5839b3142e1a0bcc1de4cbaabd9e2f2551b52bec Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 16 Dec 2024 17:54:07 -0500 Subject: [PATCH 28/46] chain: Finalize v2 hardfork heights --- chain/network.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/network.go b/chain/network.go index 3f301f5..68d60df 100644 --- a/chain/network.go +++ b/chain/network.go @@ -47,8 +47,8 @@ func Mainnet() (*consensus.Network, types.Block) { n.HardforkFoundation.PrimaryAddress = parseAddr("053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807") n.HardforkFoundation.FailsafeAddress = parseAddr("27c22a6c6e6645802a3b8fa0e5374657438ef12716d2205d3e866272de1b644dbabd53d6d560") - n.HardforkV2.AllowHeight = 1000000 // TBD - n.HardforkV2.RequireHeight = 1025000 // ~six months later + n.HardforkV2.AllowHeight = 513400 // March 10th, 2025 @ 6:00pm UTC + n.HardforkV2.RequireHeight = 526000 // June 6th, 2025 @ 6:00am UTC b := types.Block{ Timestamp: n.HardforkOak.GenesisTimestamp, From 9800765d778562d6ca204de6579c55ff04961de4 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 17 Dec 2024 07:29:35 -0800 Subject: [PATCH 29/46] refactor(rhp4): use AccountToken helper --- go.mod | 2 +- go.sum | 4 +-- rhp/v4/rpc_test.go | 82 +++++++--------------------------------------- 3 files changed, 14 insertions(+), 74 deletions(-) diff --git a/go.mod b/go.mod index 19475c9..e9bda9f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.8.0 + go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 diff --git a/go.sum b/go.sum index 25118a3..bf2c74b 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg= -go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= +go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b h1:VRkb6OOX1KawLQwuqOEHLcjha8gxVX0tAyu2Dyoq8Ek= +go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index 38a52d5..3911dd1 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -392,11 +392,7 @@ func TestRPCRefresh(t *testing.T) { revision.Revision = fundResult.Revision // upload data - at := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(5 * time.Minute), - } - at.Signature = renterKey.SignHash(at.SigHash()) + at := account.Token(renterKey, hostKey.PublicKey()) wRes, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, at, bytes.NewReader(bytes.Repeat([]byte{1}, proto4.LeafSize)), proto4.LeafSize) if err != nil { t.Fatal(err) @@ -550,11 +546,7 @@ func TestRPCRenew(t *testing.T) { revision.Revision = fundResult.Revision // upload data - at := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(5 * time.Minute), - } - at.Signature = renterKey.SignHash(at.SigHash()) + at := account.Token(renterKey, hostKey.PublicKey()) wRes, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, at, bytes.NewReader(bytes.Repeat([]byte{1}, proto4.LeafSize)), proto4.LeafSize) if err != nil { t.Fatal(err) @@ -753,13 +745,7 @@ func TestAccounts(t *testing.T) { cs := cm.TipState() // test operations against unknown account - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - - tokenSigHash := token.SigHash() - token.Signature = renterKey.SignHash(tokenSigHash) + token := account.Token(renterKey, hostKey.PublicKey()) _, err = rhp4.RPCVerifySector(context.Background(), transport, settings.Prices, token, types.Hash256{1}) if err == nil || !strings.Contains(err.Error(), proto4.ErrNotEnoughFunds.Error()) { t.Fatal(err) @@ -886,14 +872,7 @@ func TestReadWriteSector(t *testing.T) { t.Fatal(err) } revision.Revision = fundResult.Revision - - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - tokenSigHash := token.SigHash() - token.Signature = renterKey.SignHash(tokenSigHash) - + token := account.Token(renterKey, hostKey.PublicKey()) data := frand.Bytes(1024) // store the sector @@ -1005,12 +984,7 @@ func TestAppendSectors(t *testing.T) { revision.Revision = fundResult.Revision assertLastRevision(t) - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - tokenSigHash := token.SigHash() - token.Signature = renterKey.SignHash(tokenSigHash) + token := account.Token(renterKey, hostKey.PublicKey()) // store random sectors roots := make([]types.Hash256, 0, 10) @@ -1121,14 +1095,7 @@ func TestVerifySector(t *testing.T) { t.Fatal(err) } revision.Revision = fundResult.Revision - - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - tokenSigHash := token.SigHash() - token.Signature = renterKey.SignHash(tokenSigHash) - + token := account.Token(renterKey, hostKey.PublicKey()) data := frand.Bytes(1024) // store the sector @@ -1212,13 +1179,7 @@ func TestRPCFreeSectors(t *testing.T) { t.Fatal(err) } revision.Revision = fundResult.Revision - - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - tokenSigHash := token.SigHash() - token.Signature = renterKey.SignHash(tokenSigHash) + token := account.Token(renterKey, hostKey.PublicKey()) roots := make([]types.Hash256, 10) for i := range roots { @@ -1333,13 +1294,7 @@ func TestRPCSectorRoots(t *testing.T) { t.Fatal(err) } revision.Revision = fundResult.Revision - - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - tokenSigHash := token.SigHash() - token.Signature = renterKey.SignHash(tokenSigHash) + token := account.Token(renterKey, hostKey.PublicKey()) roots := make([]types.Hash256, 0, 50) @@ -1441,12 +1396,7 @@ func BenchmarkWrite(b *testing.B) { b.Fatal(err) } revision.Revision = fundResult.Revision - - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - token.Signature = renterKey.SignHash(token.SigHash()) + token := account.Token(renterKey, hostKey.PublicKey()) var sectors [][proto4.SectorSize]byte for i := 0; i < b.N; i++ { @@ -1529,12 +1479,7 @@ func BenchmarkRead(b *testing.B) { b.Fatal(err) } revision.Revision = fundResult.Revision - - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - token.Signature = renterKey.SignHash(token.SigHash()) + token := account.Token(renterKey, hostKey.PublicKey()) var sectors [][proto4.SectorSize]byte roots := make([]types.Hash256, 0, b.N) @@ -1629,12 +1574,7 @@ func BenchmarkContractUpload(b *testing.B) { b.Fatal(err) } revision.Revision = fundResult.Revision - - token := proto4.AccountToken{ - Account: account, - ValidUntil: time.Now().Add(time.Hour), - } - token.Signature = renterKey.SignHash(token.SigHash()) + token := account.Token(renterKey, hostKey.PublicKey()) var sectors [][proto4.SectorSize]byte roots := make([]types.Hash256, 0, b.N) From af7c48b09fd741238edcec6c5393e0c39fd2cc06 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 18 Dec 2024 14:33:52 +0100 Subject: [PATCH 30/46] fix(rhp): fix panic in handleRPCFormContract --- rhp/v4/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rhp/v4/server.go b/rhp/v4/server.go index 3936878..c05a04a 100644 --- a/rhp/v4/server.go +++ b/rhp/v4/server.go @@ -611,8 +611,8 @@ func (s *Server) handleRPCFormContract(stream net.Conn) error { formationTxn.FileContracts[0].RenterSignature = renterSigResp.RenterContractSignature // add the renter signatures to the transaction - for i := range formationTxn.SiacoinInputs[:len(req.RenterInputs)] { - formationTxn.SiacoinInputs[i].SatisfiedPolicy = renterSigResp.RenterSatisfiedPolicies[i] + for i, policy := range renterSigResp.RenterSatisfiedPolicies { + formationTxn.SiacoinInputs[i].SatisfiedPolicy = policy } // add our signature to the contract From b3acfca374c48a99ba38bf6a8b1d1d28b3118563 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 18 Dec 2024 15:01:28 +0100 Subject: [PATCH 31/46] fix handleRPCFormContract --- rhp/v4/rpc_test.go | 44 +++++++++++++++++++++++++++++++++++++++----- rhp/v4/server.go | 6 +++--- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index 3911dd1..0f72ad4 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -3,6 +3,7 @@ package rhp_test import ( "bytes" "context" + "math" "net" "reflect" "strings" @@ -266,10 +267,44 @@ func TestFormContractBasis(t *testing.T) { n, genesis := testutil.V2Network() hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() - cm, s, w := startTestNode(t, n, genesis) + cm, s, _ := startTestNode(t, n, genesis) + + // create a separate wallet that doesn't automatically sync with the test node + wCS, wState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis) + if err != nil { + t.Fatal(err) + } + wCM := chain.NewManager(wCS, wState) + wStore := testutil.NewEphemeralWalletStore() + w, err := wallet.NewSingleAddressWallet(renterKey, wCM, wStore) + if err != nil { + t.Fatal(err) + } // fund the wallet - mineAndSync(t, cm, w.Address(), int(n.MaturityDelay+20), w) + mineAndSync(t, cm, w.Address(), int(n.MaturityDelay+20)) + + // manually sync wallet's chain manager until one block before the tip + blocks, _, err := cm.BlocksForHistory([]types.BlockID{genesis.ID()}, cm.Tip().Height-1) + if err != nil { + t.Fatal(err) + } else if err := wCM.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + rus, aus, err := wCM.UpdatesSince(types.ChainIndex{}, math.MaxInt32) + if err != nil { + t.Fatal(err) + } + err = wStore.UpdateChainState(func(tx wallet.UpdateTx) error { + return w.UpdateChainState(tx, rus, aus) + }) + if err != nil { + t.Fatal(err) + } + balance, err := w.Balance() + if err != nil { + t.Fatal(err) + } sr := testutil.NewEphemeralSettingsReporter() sr.Update(proto4.HostSettings{ @@ -299,12 +334,11 @@ func TestFormContractBasis(t *testing.T) { } fundAndSign := &fundAndSign{w, renterKey} - renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ RenterPublicKey: renterKey.PublicKey(), RenterAddress: w.Address(), - Allowance: renterAllowance, - Collateral: hostCollateral, + Allowance: balance.Confirmed.Mul64(96).Div64(100), // almost the whole balance to force as many inputs as possible + Collateral: types.ZeroCurrency, ProofHeight: cm.Tip().Height + 50, }) if err != nil { diff --git a/rhp/v4/server.go b/rhp/v4/server.go index c05a04a..25e99f8 100644 --- a/rhp/v4/server.go +++ b/rhp/v4/server.go @@ -585,14 +585,14 @@ func (s *Server) handleRPCFormContract(stream net.Conn) error { // update renter input basis to reflect our funding basis if basis != req.Basis { - hostInputs := formationTxn.SiacoinInputs[len(formationTxn.SiacoinInputs)-len(req.RenterInputs)] - formationTxn.SiacoinInputs = formationTxn.SiacoinInputs[:len(formationTxn.SiacoinInputs)-len(req.RenterInputs)] + hostInputs := formationTxn.SiacoinInputs[len(req.RenterInputs):] + formationTxn.SiacoinInputs = formationTxn.SiacoinInputs[:len(req.RenterInputs)] txnset, err := s.chain.UpdateV2TransactionSet([]types.V2Transaction{formationTxn}, req.Basis, basis) if err != nil { return errorBadRequest("failed to update renter inputs from %q to %q: %v", req.Basis, basis, err) } formationTxn = txnset[0] - formationTxn.SiacoinInputs = append(formationTxn.SiacoinInputs, hostInputs) + formationTxn.SiacoinInputs = append(formationTxn.SiacoinInputs, hostInputs...) } // read the renter's signatures From e30377336603b4ce8eefbb27c2b9a824890e1d95 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 18 Dec 2024 17:18:01 +0100 Subject: [PATCH 32/46] clean up TestFormContractBasis --- rhp/v4/rpc_test.go | 66 ++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/rhp/v4/rpc_test.go b/rhp/v4/rpc_test.go index 0f72ad4..e3c9890 100644 --- a/rhp/v4/rpc_test.go +++ b/rhp/v4/rpc_test.go @@ -3,7 +3,6 @@ package rhp_test import ( "bytes" "context" - "math" "net" "reflect" "strings" @@ -267,50 +266,36 @@ func TestFormContractBasis(t *testing.T) { n, genesis := testutil.V2Network() hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() - cm, s, _ := startTestNode(t, n, genesis) + cm1, s, w1 := startTestNode(t, n, genesis) + cm2, _, w2 := startTestNode(t, n, genesis) - // create a separate wallet that doesn't automatically sync with the test node - wCS, wState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis) - if err != nil { - t.Fatal(err) - } - wCM := chain.NewManager(wCS, wState) - wStore := testutil.NewEphemeralWalletStore() - w, err := wallet.NewSingleAddressWallet(renterKey, wCM, wStore) - if err != nil { - t.Fatal(err) - } - - // fund the wallet - mineAndSync(t, cm, w.Address(), int(n.MaturityDelay+20)) + // fund both wallets + mineAndSync(t, cm1, w2.Address(), int(n.MaturityDelay+20)) + mineAndSync(t, cm1, w1.Address(), int(n.MaturityDelay+20)) - // manually sync wallet's chain manager until one block before the tip - blocks, _, err := cm.BlocksForHistory([]types.BlockID{genesis.ID()}, cm.Tip().Height-1) + // manually sync the second wallet just before tip + _, applied, err := cm1.UpdatesSince(types.ChainIndex{}, int(w1.Tip().Height-5)) if err != nil { t.Fatal(err) - } else if err := wCM.AddBlocks(blocks); err != nil { - t.Fatal(err) } - rus, aus, err := wCM.UpdatesSince(types.ChainIndex{}, math.MaxInt32) - if err != nil { - t.Fatal(err) + var blocks []types.Block + for _, cau := range applied { + blocks = append(blocks, cau.Block) } - err = wStore.UpdateChainState(func(tx wallet.UpdateTx) error { - return w.UpdateChainState(tx, rus, aus) - }) - if err != nil { + if err := cm2.AddBlocks(blocks); err != nil { t.Fatal(err) } - balance, err := w.Balance() - if err != nil { - t.Fatal(err) + + // wait for the second wallet to sync + for cm2.Tip() != w2.Tip() { + time.Sleep(time.Millisecond) } sr := testutil.NewEphemeralSettingsReporter() sr.Update(proto4.HostSettings{ Release: "test", AcceptingContracts: true, - WalletAddress: w.Address(), + WalletAddress: w1.Address(), MaxCollateral: types.Siacoins(10000), MaxContractDuration: 1000, RemainingStorage: 100 * proto4.SectorSize, @@ -324,29 +309,34 @@ func TestFormContractBasis(t *testing.T) { }, }) ss := testutil.NewEphemeralSectorStore() - c := testutil.NewEphemeralContractor(cm) + c := testutil.NewEphemeralContractor(cm1) - transport := testRenterHostPair(t, hostKey, cm, s, w, c, sr, ss, zap.NewNop()) + transport := testRenterHostPair(t, hostKey, cm1, s, w1, c, sr, ss, zap.NewNop()) settings, err := rhp4.RPCSettings(context.Background(), transport) if err != nil { t.Fatal(err) } - fundAndSign := &fundAndSign{w, renterKey} - result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + balance, err := w2.Balance() + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w2, renterKey} + result, err := rhp4.RPCFormContract(context.Background(), transport, cm2, fundAndSign, cm1.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ RenterPublicKey: renterKey.PublicKey(), - RenterAddress: w.Address(), + RenterAddress: w2.Address(), Allowance: balance.Confirmed.Mul64(96).Div64(100), // almost the whole balance to force as many inputs as possible Collateral: types.ZeroCurrency, - ProofHeight: cm.Tip().Height + 50, + ProofHeight: cm1.Tip().Height + 50, }) if err != nil { t.Fatal(err) } // verify the transaction set is valid - if known, err := cm.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { + if known, err := cm1.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { t.Fatal(err) } else if !known { t.Fatal("expected transaction set to be known") From 9f94d7c4bfba7175726ac7bd03d6db1d58ba2441 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 18 Dec 2024 17:19:00 +0100 Subject: [PATCH 33/46] fix handleRPCRefreshContract and handleRPCRenewContract --- rhp/v4/server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rhp/v4/server.go b/rhp/v4/server.go index 25e99f8..a64b51d 100644 --- a/rhp/v4/server.go +++ b/rhp/v4/server.go @@ -722,8 +722,8 @@ func (s *Server) handleRPCRefreshContract(stream net.Conn) error { // update renter inputs to reflect our chain state if basis != req.Basis { - hostInputs := renewalTxn.SiacoinInputs[len(renewalTxn.SiacoinInputs)-len(req.RenterInputs):] - renewalTxn.SiacoinInputs = renewalTxn.SiacoinInputs[:len(renewalTxn.SiacoinInputs)-len(req.RenterInputs)] + hostInputs := renewalTxn.SiacoinInputs[len(req.RenterInputs):] + renewalTxn.SiacoinInputs = renewalTxn.SiacoinInputs[:len(req.RenterInputs)] updated, err := s.chain.UpdateV2TransactionSet([]types.V2Transaction{renewalTxn}, req.Basis, basis) if err != nil { return errorBadRequest("failed to update renter inputs from %q to %q: %v", req.Basis, basis, err) @@ -891,8 +891,8 @@ func (s *Server) handleRPCRenewContract(stream net.Conn) error { // update renter inputs to reflect our chain state if basis != req.Basis { - hostInputs := renewalTxn.SiacoinInputs[len(renewalTxn.SiacoinInputs)-len(req.RenterInputs):] - renewalTxn.SiacoinInputs = renewalTxn.SiacoinInputs[:len(renewalTxn.SiacoinInputs)-len(req.RenterInputs)] + hostInputs := renewalTxn.SiacoinInputs[len(req.RenterInputs):] + renewalTxn.SiacoinInputs = renewalTxn.SiacoinInputs[:len(req.RenterInputs)] updated, err := s.chain.UpdateV2TransactionSet([]types.V2Transaction{renewalTxn}, req.Basis, basis) if err != nil { return errorBadRequest("failed to update renter inputs from %q to %q: %v", req.Basis, basis, err) From 569bd2691d2abf5ef99426c1ea706ab94dfe5663 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 18 Dec 2024 07:20:31 -0800 Subject: [PATCH 34/46] add v2 changeset --- .changeset/finalize_v2_hardfork_dates.md | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .changeset/finalize_v2_hardfork_dates.md diff --git a/.changeset/finalize_v2_hardfork_dates.md b/.changeset/finalize_v2_hardfork_dates.md new file mode 100644 index 0000000..e0b313e --- /dev/null +++ b/.changeset/finalize_v2_hardfork_dates.md @@ -0,0 +1,26 @@ +--- +default: major +--- + +# Finalize V2 Hardfork Dates + +The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has been untouched since Sia's mainnet launch back in 2014, and improve accessibility of the storage network. To ensure a smooth transition from V1, it will be executed in two phases. Additional documentation on upgrading will be released in the near future. + +### V2 Highlights +- Drastically reduces blockchain size on disk +- Improves UTXO spend policies - including HTLC support for Atomic Swaps +- More efficient contract renewals - reducing lock up requirements for hosts and renters +- Improved transfer speeds - enables hot storage + +### Phase 1 - Allow Height +- **Activation Height:** `513400` (March 10th, 2025) +- **New Features:** V2 transactions, contracts, and RHP4 +- **V1 Support:** Both V1 and V2 will be supported during this phase +- **Purpose:** This period gives time for integrators to transition from V1 to V2 +- **Requirements:** Users will need to update to support the hardfork before this block height + +### Phase 2 - Require Height +- **Activation Height:** `526000` (June 6th, 2025) +- **New Features:** The consensus database can be trimmed to only store the Merkle proofs +- **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted +- **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height \ No newline at end of file From 738f2d24b7aa8b9a54bffc141964e05c9a251030 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 18 Dec 2024 21:59:46 -0800 Subject: [PATCH 35/46] build: update core --- go.mod | 2 +- go.sum | 4 ++-- knope.toml | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e9bda9f..0fae397 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( go.etcd.io/bbolt v1.3.11 - go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b + go.sia.tech/core v0.9.0 go.sia.tech/mux v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.31.0 diff --git a/go.sum b/go.sum index bf2c74b..69cb58d 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b h1:VRkb6OOX1KawLQwuqOEHLcjha8gxVX0tAyu2Dyoq8Ek= -go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= +go.sia.tech/core v0.9.0 h1:qV7V8nkNaPvBEhkbwgrETTkb7JCMcAnKUQt9nUumP4k= +go.sia.tech/core v0.9.0/go.mod h1:3NAvYHuzAZg9vP6pyIMOxjTkgHBQ3vx9cXTqRF6oEa4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/knope.toml b/knope.toml index 7f5f214..fbbd57f 100644 --- a/knope.toml +++ b/knope.toml @@ -18,6 +18,7 @@ command = "git switch -c release" [[workflows.steps]] type = "PrepareRelease" +ignore_conventional_commits = true [[workflows.steps]] type = "Command" From 0d929edf753307499558629daf1671dd4d9bb321 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 07:48:24 +0000 Subject: [PATCH 36/46] chore: prepare release 0.9.0 --- .changeset/finalize_v2_hardfork_dates.md | 26 ---------------- ...s_when_account_has_insufficient_balance.md | 5 --- CHANGELOG.md | 31 +++++++++++++++++++ go.mod | 2 +- 4 files changed, 32 insertions(+), 32 deletions(-) delete mode 100644 .changeset/finalize_v2_hardfork_dates.md delete mode 100644 .changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md diff --git a/.changeset/finalize_v2_hardfork_dates.md b/.changeset/finalize_v2_hardfork_dates.md deleted file mode 100644 index e0b313e..0000000 --- a/.changeset/finalize_v2_hardfork_dates.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -default: major ---- - -# Finalize V2 Hardfork Dates - -The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has been untouched since Sia's mainnet launch back in 2014, and improve accessibility of the storage network. To ensure a smooth transition from V1, it will be executed in two phases. Additional documentation on upgrading will be released in the near future. - -### V2 Highlights -- Drastically reduces blockchain size on disk -- Improves UTXO spend policies - including HTLC support for Atomic Swaps -- More efficient contract renewals - reducing lock up requirements for hosts and renters -- Improved transfer speeds - enables hot storage - -### Phase 1 - Allow Height -- **Activation Height:** `513400` (March 10th, 2025) -- **New Features:** V2 transactions, contracts, and RHP4 -- **V1 Support:** Both V1 and V2 will be supported during this phase -- **Purpose:** This period gives time for integrators to transition from V1 to V2 -- **Requirements:** Users will need to update to support the hardfork before this block height - -### Phase 2 - Require Height -- **Activation Height:** `526000` (June 6th, 2025) -- **New Features:** The consensus database can be trimmed to only store the Merkle proofs -- **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted -- **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height \ No newline at end of file diff --git a/.changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md b/.changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md deleted file mode 100644 index 1792e57..0000000 --- a/.changeset/fix_rhp4_server_not_returning_errnotenoughfunds_when_account_has_insufficient_balance.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Fix rhp4 server not returning ErrNotEnoughFunds when account has insufficient balance diff --git a/CHANGELOG.md b/CHANGELOG.md index 603c4c2..479eaba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,34 @@ +## 0.9.0 (2024-12-19) + +### Breaking Changes + +#### Finalize V2 Hardfork Dates + +The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has been untouched since Sia's mainnet launch back in 2014, and improve accessibility of the storage network. To ensure a smooth transition from V1, it will be executed in two phases. Additional documentation on upgrading will be released in the near future. + +#### V2 Highlights +- Drastically reduces blockchain size on disk +- Improves UTXO spend policies - including HTLC support for Atomic Swaps +- More efficient contract renewals - reducing lock up requirements for hosts and renters +- Improved transfer speeds - enables hot storage + +#### Phase 1 - Allow Height +- **Activation Height:** `513400` (March 10th, 2025) +- **New Features:** V2 transactions, contracts, and RHP4 +- **V1 Support:** Both V1 and V2 will be supported during this phase +- **Purpose:** This period gives time for integrators to transition from V1 to V2 +- **Requirements:** Users will need to update to support the hardfork before this block height + +#### Phase 2 - Require Height +- **Activation Height:** `526000` (June 6th, 2025) +- **New Features:** The consensus database can be trimmed to only store the Merkle proofs +- **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted +- **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height + +### Fixes + +- Fix rhp4 server not returning ErrNotEnoughFunds when account has insufficient balance + ## 0.8.0 (2024-12-13) ### Breaking Changes diff --git a/go.mod b/go.mod index 0fae397..6a6cf6e 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/coreutils // v0.8.0 +module go.sia.tech/coreutils // v0.9.0 go 1.23.1 From 09d33c20cf2daf21bf5aab7e201158fd57ceb823 Mon Sep 17 00:00:00 2001 From: mike76-dev Date: Fri, 10 Jan 2025 19:10:37 +0100 Subject: [PATCH 37/46] Release inputs if RPCFormContract, RPCRenewContract, or RPCRefreshContract fail --- rhp/v4/rpc.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/rhp/v4/rpc.go b/rhp/v4/rpc.go index 1600453..5782545 100644 --- a/rhp/v4/rpc.go +++ b/rhp/v4/rpc.go @@ -568,11 +568,13 @@ func RPCFormContract(ctx context.Context, t TransportClient, tp TxPool, signer F RenterParents: formationSet, } if err := rhp4.WriteRequest(s, rhp4.RPCFormContractID, &req); err != nil { + signer.ReleaseInputs([]types.V2Transaction{formationTxn}) return RPCFormContractResult{}, fmt.Errorf("failed to write request: %w", err) } var hostInputsResp rhp4.RPCFormContractResponse if err := rhp4.ReadResponse(s, &hostInputsResp); err != nil { + signer.ReleaseInputs([]types.V2Transaction{formationTxn}) return RPCFormContractResult{}, fmt.Errorf("failed to read host inputs response: %w", err) } @@ -584,6 +586,7 @@ func RPCFormContract(ctx context.Context, t TransportClient, tp TxPool, signer F } if n := hostInputSum.Cmp(fc.TotalCollateral); n < 0 { + signer.ReleaseInputs([]types.V2Transaction{formationTxn}) return RPCFormContractResult{}, fmt.Errorf("expected host to fund at least %v, got %v", fc.TotalCollateral, hostInputSum) } else if n > 0 { // add change output @@ -606,12 +609,14 @@ func RPCFormContract(ctx context.Context, t TransportClient, tp TxPool, signer F } // send the renter signatures if err := rhp4.WriteResponse(s, &renterPolicyResp); err != nil { + signer.ReleaseInputs([]types.V2Transaction{formationTxn}) return RPCFormContractResult{}, fmt.Errorf("failed to write signature response: %w", err) } // read the finalized transaction set var hostTransactionSetResp rhp4.RPCFormContractThirdResponse if err := rhp4.ReadResponse(s, &hostTransactionSetResp); err != nil { + // at this point the formation txn must already have been broadcast, so no need to release inputs return RPCFormContractResult{}, fmt.Errorf("failed to read final response: %w", err) } @@ -681,6 +686,7 @@ func RPCRenewContract(ctx context.Context, t TransportClient, tp TxPool, signer req.Basis, req.RenterParents, err = tp.V2TransactionSet(basis, renewalTxn) if err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRenewContractResult{}, fmt.Errorf("failed to get transaction set: %w", err) } for _, si := range renewalTxn.SiacoinInputs { @@ -695,11 +701,13 @@ func RPCRenewContract(ctx context.Context, t TransportClient, tp TxPool, signer defer s.Close() if err := rhp4.WriteRequest(s, rhp4.RPCRenewContractID, &req); err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRenewContractResult{}, fmt.Errorf("failed to write request: %w", err) } var hostInputsResp rhp4.RPCRenewContractResponse if err := rhp4.ReadResponse(s, &hostInputsResp); err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRenewContractResult{}, fmt.Errorf("failed to read host inputs response: %w", err) } @@ -712,6 +720,7 @@ func RPCRenewContract(ctx context.Context, t TransportClient, tp TxPool, signer // verify the host added enough inputs if n := hostInputSum.Cmp(hostCost); n < 0 { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRenewContractResult{}, fmt.Errorf("expected host to fund %v, got %v", hostCost, hostInputSum) } else if n > 0 { // add change output @@ -739,12 +748,14 @@ func RPCRenewContract(ctx context.Context, t TransportClient, tp TxPool, signer renterPolicyResp.RenterSatisfiedPolicies = append(renterPolicyResp.RenterSatisfiedPolicies, si.SatisfiedPolicy) } if err := rhp4.WriteResponse(s, &renterPolicyResp); err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRenewContractResult{}, fmt.Errorf("failed to write signature response: %w", err) } // read the finalized transaction set var hostTransactionSetResp rhp4.RPCRenewContractThirdResponse if err := rhp4.ReadResponse(s, &hostTransactionSetResp); err != nil { + // at this point the renewal txn must already have been broadcast, so no need to release inputs return RPCRenewContractResult{}, fmt.Errorf("failed to read final response: %w", err) } @@ -812,6 +823,7 @@ func RPCRefreshContract(ctx context.Context, t TransportClient, tp TxPool, signe req.Basis, req.RenterParents, err = tp.V2TransactionSet(basis, renewalTxn) if err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRefreshContractResult{}, fmt.Errorf("failed to get transaction set: %w", err) } for _, si := range renewalTxn.SiacoinInputs { @@ -826,11 +838,13 @@ func RPCRefreshContract(ctx context.Context, t TransportClient, tp TxPool, signe defer s.Close() if err := rhp4.WriteRequest(s, rhp4.RPCRefreshContractID, &req); err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRefreshContractResult{}, fmt.Errorf("failed to write request: %w", err) } var hostInputsResp rhp4.RPCRefreshContractResponse if err := rhp4.ReadResponse(s, &hostInputsResp); err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRefreshContractResult{}, fmt.Errorf("failed to read host inputs response: %w", err) } @@ -843,6 +857,7 @@ func RPCRefreshContract(ctx context.Context, t TransportClient, tp TxPool, signe // verify the host added enough inputs if n := hostInputSum.Cmp(hostCost); n < 0 { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRefreshContractResult{}, fmt.Errorf("expected host to fund %v, got %v", hostCost, hostInputSum) } else if n > 0 { // add change output @@ -870,12 +885,14 @@ func RPCRefreshContract(ctx context.Context, t TransportClient, tp TxPool, signe renterPolicyResp.RenterSatisfiedPolicies = append(renterPolicyResp.RenterSatisfiedPolicies, si.SatisfiedPolicy) } if err := rhp4.WriteResponse(s, &renterPolicyResp); err != nil { + signer.ReleaseInputs([]types.V2Transaction{renewalTxn}) return RPCRefreshContractResult{}, fmt.Errorf("failed to write signature response: %w", err) } // read the finalized transaction set var hostTransactionSetResp rhp4.RPCRefreshContractThirdResponse if err := rhp4.ReadResponse(s, &hostTransactionSetResp); err != nil { + // at this point the renewal txn must already have been broadcast, so no need to release inputs return RPCRefreshContractResult{}, fmt.Errorf("failed to read final response: %w", err) } From 54a0611351ed918740885e6f24fc9161ed19b4be Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 10 Jan 2025 11:01:45 -0800 Subject: [PATCH 38/46] chore: document change --- ...d_utxos_if_contract_formation_renewal_or_refresh_fails.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md diff --git a/.changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md b/.changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md new file mode 100644 index 0000000..0f198e3 --- /dev/null +++ b/.changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Release locked UTXOs if contract formation, renewal, or refresh fails From b6af4d23330c554b3ec9ba7f9aa172261920c17e Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 10 Jan 2025 11:13:24 -0800 Subject: [PATCH 39/46] rhp4: release host utxos when formation, refresh, or renew fails --- ...ract_formation_renewal_or_refresh_fails.md | 5 +++ rhp/v4/server.go | 38 ++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 .changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md diff --git a/.changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md b/.changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md new file mode 100644 index 0000000..9560b6a --- /dev/null +++ b/.changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Release locked host UTXOs if contract formation, renewal, or refresh fails diff --git a/rhp/v4/server.go b/rhp/v4/server.go index a64b51d..dcfd9e3 100644 --- a/rhp/v4/server.go +++ b/rhp/v4/server.go @@ -566,6 +566,7 @@ func (s *Server) handleRPCFormContract(stream net.Conn) error { }) } + var broadcast bool // fund the host collateral basis, toSign, err := s.wallet.FundV2Transaction(&formationTxn, hostCost, true) if errors.Is(err, wallet.ErrNotEnoughFunds) { @@ -573,6 +574,13 @@ func (s *Server) handleRPCFormContract(stream net.Conn) error { } else if err != nil { return fmt.Errorf("failed to fund transaction: %w", err) } + defer func() { + if broadcast { + return + } + // release the inputs if the transaction is not going to be broadcast + s.wallet.ReleaseInputs(nil, []types.V2Transaction{formationTxn}) + }() // sign the transaction inputs s.wallet.SignV2Inputs(&formationTxn, toSign) // send the host inputs to the renter @@ -633,7 +641,6 @@ func (s *Server) handleRPCFormContract(stream net.Conn) error { } else if _, err = s.chain.AddV2PoolTransactions(basis, formationSet); err != nil { return errorBadRequest("failed to broadcast formation transaction: %v", err) } - s.syncer.BroadcastV2TransactionSet(basis, formationSet) // add the contract to the contractor err = s.contractor.AddV2Contract(TransactionSet{ @@ -643,6 +650,9 @@ func (s *Server) handleRPCFormContract(stream net.Conn) error { if err != nil { return fmt.Errorf("failed to add contract: %w", err) } + // broadcast the finalized contract formation set + broadcast = true // set broadcast so the UTXOs will not be released if the renter happens to disconnect before receiving the last response + s.syncer.BroadcastV2TransactionSet(basis, formationSet) // send the finalized transaction set to the renter return rhp4.WriteResponse(stream, &rhp4.RPCFormContractThirdResponse{ @@ -713,12 +723,20 @@ func (s *Server) handleRPCRefreshContract(stream net.Conn) error { return fmt.Errorf("failed to get contract element: %w", err) } + var broadcast bool basis, toSign, err := s.wallet.FundV2Transaction(&renewalTxn, hostCost, true) if errors.Is(err, wallet.ErrNotEnoughFunds) { return rhp4.ErrHostFundError } else if err != nil { return fmt.Errorf("failed to fund transaction: %w", err) } + defer func() { + if broadcast { + return + } + // release the locked UTXOs if the transaction is not going to be broadcast + s.wallet.ReleaseInputs(nil, []types.V2Transaction{renewalTxn}) + }() // update renter inputs to reflect our chain state if basis != req.Basis { @@ -799,8 +817,6 @@ func (s *Server) handleRPCRefreshContract(stream net.Conn) error { } else if _, err = s.chain.AddV2PoolTransactions(basis, renewalSet); err != nil { return errorBadRequest("failed to broadcast renewal set: %v", err) } - // broadcast the transaction set - s.syncer.BroadcastV2TransactionSet(basis, renewalSet) // add the contract to the contractor err = s.contractor.RenewV2Contract(TransactionSet{ @@ -811,6 +827,9 @@ func (s *Server) handleRPCRefreshContract(stream net.Conn) error { return fmt.Errorf("failed to add contract: %w", err) } + broadcast = true // set broadcast so the UTXOs will not be released if the renter happens to disconnect before receiving the last response + s.syncer.BroadcastV2TransactionSet(basis, renewalSet) + // send the finalized transaction set to the renter return rhp4.WriteResponse(stream, &rhp4.RPCRefreshContractThirdResponse{ Basis: basis, @@ -882,12 +901,20 @@ func (s *Server) handleRPCRenewContract(stream net.Conn) error { return fmt.Errorf("failed to get contract element: %w", err) } + var broadcast bool basis, toSign, err := s.wallet.FundV2Transaction(&renewalTxn, hostCost, true) if errors.Is(err, wallet.ErrNotEnoughFunds) { return rhp4.ErrHostFundError } else if err != nil { return fmt.Errorf("failed to fund transaction: %w", err) } + defer func() { + if broadcast { + return + } + // release the locked UTXOs if the transaction is not going to be broadcast + s.wallet.ReleaseInputs(nil, []types.V2Transaction{renewalTxn}) + }() // update renter inputs to reflect our chain state if basis != req.Basis { @@ -968,8 +995,6 @@ func (s *Server) handleRPCRenewContract(stream net.Conn) error { } else if _, err = s.chain.AddV2PoolTransactions(basis, renewalSet); err != nil { return errorBadRequest("failed to broadcast renewal set: %v", err) } - // broadcast the transaction set - s.syncer.BroadcastV2TransactionSet(basis, renewalSet) // add the contract to the contractor err = s.contractor.RenewV2Contract(TransactionSet{ @@ -980,6 +1005,9 @@ func (s *Server) handleRPCRenewContract(stream net.Conn) error { return fmt.Errorf("failed to add contract: %w", err) } + broadcast = true // set broadcast so the UTXOs will not be released if the renter happens to disconnect before receiving the last response + s.syncer.BroadcastV2TransactionSet(basis, renewalSet) + // send the finalized transaction set to the renter return rhp4.WriteResponse(stream, &rhp4.RPCRenewContractThirdResponse{ Basis: basis, From e3d0db163fc0af6bd7952150bd12e215fef5b416 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 12 Jan 2025 05:31:32 +0000 Subject: [PATCH 40/46] chore: prepare release 0.9.1 --- ...utxos_if_contract_formation_renewal_or_refresh_fails.md | 5 ----- ...utxos_if_contract_formation_renewal_or_refresh_fails.md | 5 ----- CHANGELOG.md | 7 +++++++ go.mod | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 .changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md delete mode 100644 .changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md diff --git a/.changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md b/.changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md deleted file mode 100644 index 9560b6a..0000000 --- a/.changeset/release_locked_host_utxos_if_contract_formation_renewal_or_refresh_fails.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Release locked host UTXOs if contract formation, renewal, or refresh fails diff --git a/.changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md b/.changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md deleted file mode 100644 index 0f198e3..0000000 --- a/.changeset/release_locked_utxos_if_contract_formation_renewal_or_refresh_fails.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Release locked UTXOs if contract formation, renewal, or refresh fails diff --git a/CHANGELOG.md b/CHANGELOG.md index 479eaba..99b6c9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 0.9.1 (2025-01-12) + +### Fixes + +- Release locked host UTXOs if contract formation, renewal, or refresh fails +- Release locked UTXOs if contract formation, renewal, or refresh fails + ## 0.9.0 (2024-12-19) ### Breaking Changes diff --git a/go.mod b/go.mod index 6a6cf6e..145f55b 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/coreutils // v0.9.0 +module go.sia.tech/coreutils // v0.9.1 go 1.23.1 From 287001db9bbd3417c1777f69f2321c4eba31ab31 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 14 Jan 2025 16:18:20 +0100 Subject: [PATCH 41/46] fix locking --- wallet/update.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/wallet/update.go b/wallet/update.go index eb474b7..d996cf6 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -290,9 +290,6 @@ func appliedEvents(cau chain.ApplyUpdate, walletAddress types.Address) (events [ // applyChainUpdate atomically applies a chain update func (sw *SingleAddressWallet) applyChainUpdate(tx UpdateTx, address types.Address, cau chain.ApplyUpdate) error { - sw.mu.Lock() - defer sw.mu.Unlock() - // update current state elements if err := tx.UpdateWalletSiacoinElementProofs(cau); err != nil { return fmt.Errorf("failed to update state elements: %w", err) @@ -319,15 +316,14 @@ func (sw *SingleAddressWallet) applyChainUpdate(tx UpdateTx, address types.Addre if err := tx.WalletApplyIndex(cau.State.Index, createdUTXOs, spentUTXOs, appliedEvents(cau, address), cau.Block.Timestamp); err != nil { return fmt.Errorf("failed to apply index: %w", err) } + sw.mu.Lock() sw.tip = cau.State.Index + sw.mu.Unlock() return nil } // revertChainUpdate atomically reverts a chain update from a wallet func (sw *SingleAddressWallet) revertChainUpdate(tx UpdateTx, revertedIndex types.ChainIndex, address types.Address, cru chain.RevertUpdate) error { - sw.mu.Lock() - defer sw.mu.Unlock() - var removedUTXOs, unspentUTXOs []types.SiacoinElement cru.ForEachSiacoinElement(func(se types.SiacoinElement, created, spent bool) { switch { @@ -355,7 +351,9 @@ func (sw *SingleAddressWallet) revertChainUpdate(tx UpdateTx, revertedIndex type if err := tx.UpdateWalletSiacoinElementProofs(cru); err != nil { return fmt.Errorf("failed to update state elements: %w", err) } + sw.mu.Lock() sw.tip = revertedIndex + sw.mu.Unlock() return nil } From 100e169835718dab7cd0363d723a6920e829672a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 14 Jan 2025 16:46:08 +0100 Subject: [PATCH 42/46] move store access out of selection --- wallet/wallet.go | 52 +++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 7f24352..1f5f645 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -218,16 +218,11 @@ func (sw *SingleAddressWallet) SpendableOutputs() ([]types.SiacoinElement, error return unspent, nil } -func (sw *SingleAddressWallet) selectUTXOs(amount types.Currency, inputs int, useUnconfirmed bool) ([]types.SiacoinElement, types.Currency, error) { +func (sw *SingleAddressWallet) selectUTXOs(amount types.Currency, inputs int, useUnconfirmed bool, elements []types.SiacoinElement) ([]types.SiacoinElement, types.Currency, error) { if amount.IsZero() { return nil, types.ZeroCurrency, nil } - elements, err := sw.store.UnspentSiacoinElements() - if err != nil { - return nil, types.ZeroCurrency, err - } - tpoolSpent := make(map[types.SiacoinOutputID]bool) tpoolUtxos := make(map[types.SiacoinOutputID]types.SiacoinElement) for _, txn := range sw.cm.PoolTransactions() { @@ -353,10 +348,15 @@ func (sw *SingleAddressWallet) FundTransaction(txn *types.Transaction, amount ty return nil, nil } + elements, err := sw.store.UnspentSiacoinElements() + if err != nil { + return nil, err + } + sw.mu.Lock() defer sw.mu.Unlock() - selected, inputSum, err := sw.selectUTXOs(amount, len(txn.SiacoinInputs), useUnconfirmed) + selected, inputSum, err := sw.selectUTXOs(amount, len(txn.SiacoinInputs), useUnconfirmed, elements) if err != nil { return nil, err } @@ -413,14 +413,20 @@ func (sw *SingleAddressWallet) SignTransaction(txn *types.Transaction, toSign [] // // The returned index should be used as the basis for AddV2PoolTransactions. func (sw *SingleAddressWallet) FundV2Transaction(txn *types.V2Transaction, amount types.Currency, useUnconfirmed bool) (types.ChainIndex, []int, error) { - sw.mu.Lock() - defer sw.mu.Unlock() - if amount.IsZero() { return sw.tip, nil, nil } - selected, inputSum, err := sw.selectUTXOs(amount, len(txn.SiacoinInputs), useUnconfirmed) + // fetch outputs from the store + elements, err := sw.store.UnspentSiacoinElements() + if err != nil { + return types.ChainIndex{}, nil, err + } + + sw.mu.Lock() + defer sw.mu.Unlock() + + selected, inputSum, err := sw.selectUTXOs(amount, len(txn.SiacoinInputs), useUnconfirmed, elements) if err != nil { return types.ChainIndex{}, nil, err } @@ -581,13 +587,7 @@ func (sw *SingleAddressWallet) UnconfirmedEvents() (annotated []Event, err error return annotated, nil } -func (sw *SingleAddressWallet) selectRedistributeUTXOs(bh uint64, outputs int, amount types.Currency) ([]types.SiacoinElement, int, error) { - // fetch outputs from the store - elements, err := sw.store.UnspentSiacoinElements() - if err != nil { - return nil, 0, err - } - +func (sw *SingleAddressWallet) selectRedistributeUTXOs(bh uint64, outputs int, amount types.Currency, elements []types.SiacoinElement) ([]types.SiacoinElement, int, error) { // fetch outputs currently in the pool inPool := make(map[types.SiacoinOutputID]bool) for _, txn := range sw.cm.PoolTransactions() { @@ -631,10 +631,16 @@ func (sw *SingleAddressWallet) selectRedistributeUTXOs(bh uint64, outputs int, a // outputs. It also returns a list of output IDs that need to be signed. func (sw *SingleAddressWallet) Redistribute(outputs int, amount, feePerByte types.Currency) (txns []types.Transaction, toSign []types.Hash256, err error) { state := sw.cm.TipState() + + elements, err := sw.store.UnspentSiacoinElements() + if err != nil { + return nil, nil, err + } + sw.mu.Lock() defer sw.mu.Unlock() - utxos, outputs, err := sw.selectRedistributeUTXOs(state.Index.Height, outputs, amount) + utxos, outputs, err := sw.selectRedistributeUTXOs(state.Index.Height, outputs, amount, elements) if err != nil { return nil, nil, err } @@ -724,10 +730,16 @@ func (sw *SingleAddressWallet) Redistribute(outputs int, amount, feePerByte type // outputs. It also returns a list of output IDs that need to be signed. func (sw *SingleAddressWallet) RedistributeV2(outputs int, amount, feePerByte types.Currency) (txns []types.V2Transaction, toSign [][]int, err error) { state := sw.cm.TipState() + + elements, err := sw.store.UnspentSiacoinElements() + if err != nil { + return nil, nil, err + } + sw.mu.Lock() defer sw.mu.Unlock() - utxos, outputs, err := sw.selectRedistributeUTXOs(state.Index.Height, outputs, amount) + utxos, outputs, err := sw.selectRedistributeUTXOs(state.Index.Height, outputs, amount, elements) if err != nil { return nil, nil, err } From fa9d759f6fb5a94c96a48e40677a2d97b3dcff36 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 15 Jan 2025 11:58:09 +0100 Subject: [PATCH 43/46] add changeset entry --- ...y_avoiding_acquiring_the_mutex_before_a_db_transaction.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md diff --git a/.changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md b/.changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md new file mode 100644 index 0000000..5433b5d --- /dev/null +++ b/.changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Improve locking in SingleAddressWallet by avoiding acquiring the mutex before a db transaction From 08c281eb3b2e92378b494a9aad363624707c5b62 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 15 Jan 2025 12:01:14 -0800 Subject: [PATCH 44/46] wallet: increase default reservation duration --- ...ult_utxo_reservation_in_singleaddresswallet_to_3_hours.md | 5 +++++ wallet/wallet.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md diff --git a/.changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md b/.changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md new file mode 100644 index 0000000..899c6da --- /dev/null +++ b/.changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Increased default UTXO reservation in SingleAddressWallet to 3 hours diff --git a/wallet/wallet.go b/wallet/wallet.go index 1f5f645..6cffa8a 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -925,7 +925,7 @@ func NewSingleAddressWallet(priv types.PrivateKey, cm ChainManager, store Single DefragThreshold: 30, MaxInputsForDefrag: 30, MaxDefragUTXOs: 10, - ReservationDuration: 15 * time.Minute, + ReservationDuration: 3 * time.Hour, Log: zap.NewNop(), } From f806681b085c5be2933131f5561f6eb06e8f7dce Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 15 Jan 2025 14:13:16 -0800 Subject: [PATCH 45/46] update mainnet activation heights --- .changeset/increased_v2_allow_height_to_52400.md | 7 +++++++ chain/network.go | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 .changeset/increased_v2_allow_height_to_52400.md diff --git a/.changeset/increased_v2_allow_height_to_52400.md b/.changeset/increased_v2_allow_height_to_52400.md new file mode 100644 index 0000000..cc96d47 --- /dev/null +++ b/.changeset/increased_v2_allow_height_to_52400.md @@ -0,0 +1,7 @@ +--- +default: major +--- + +# Increased V2 Allow Height to 526,000 + +Delays activation of the v2 hardfork to June 6th, 2025 in response to concerns about the scope of updates necessary for partners to support v2 \ No newline at end of file diff --git a/chain/network.go b/chain/network.go index 68d60df..6fbb63b 100644 --- a/chain/network.go +++ b/chain/network.go @@ -47,8 +47,8 @@ func Mainnet() (*consensus.Network, types.Block) { n.HardforkFoundation.PrimaryAddress = parseAddr("053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807") n.HardforkFoundation.FailsafeAddress = parseAddr("27c22a6c6e6645802a3b8fa0e5374657438ef12716d2205d3e866272de1b644dbabd53d6d560") - n.HardforkV2.AllowHeight = 513400 // March 10th, 2025 @ 6:00pm UTC - n.HardforkV2.RequireHeight = 526000 // June 6th, 2025 @ 6:00am UTC + n.HardforkV2.AllowHeight = 526000 // June 6th, 2025 @ 6:00am UTC + n.HardforkV2.RequireHeight = 530000 // July 4th, 2025 @ 2:00am UTC b := types.Block{ Timestamp: n.HardforkOak.GenesisTimestamp, @@ -142,7 +142,7 @@ func TestnetZen() (*consensus.Network, types.Block) { n.HardforkFoundation.FailsafeAddress = types.VoidAddress n.HardforkV2.AllowHeight = 112000 // March 1, 2025 @ 7:00:00 UTC - n.HardforkV2.RequireHeight = 116380 // 1 month later + n.HardforkV2.RequireHeight = 114000 // ~ 2 weeks later b := types.Block{ Timestamp: n.HardforkOak.GenesisTimestamp, From 07e6439f6ab109e8bb4d2ea294e0edabc50a3154 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 23:05:34 +0000 Subject: [PATCH 46/46] chore: prepare release 0.10.0 --- ...g_acquiring_the_mutex_before_a_db_transaction.md | 5 ----- ...reservation_in_singleaddresswallet_to_3_hours.md | 5 ----- .changeset/increased_v2_allow_height_to_52400.md | 7 ------- CHANGELOG.md | 13 +++++++++++++ go.mod | 2 +- 5 files changed, 14 insertions(+), 18 deletions(-) delete mode 100644 .changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md delete mode 100644 .changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md delete mode 100644 .changeset/increased_v2_allow_height_to_52400.md diff --git a/.changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md b/.changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md deleted file mode 100644 index 5433b5d..0000000 --- a/.changeset/improve_locking_in_singleaddresswallet_by_avoiding_acquiring_the_mutex_before_a_db_transaction.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Improve locking in SingleAddressWallet by avoiding acquiring the mutex before a db transaction diff --git a/.changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md b/.changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md deleted file mode 100644 index 899c6da..0000000 --- a/.changeset/increase_default_utxo_reservation_in_singleaddresswallet_to_3_hours.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Increased default UTXO reservation in SingleAddressWallet to 3 hours diff --git a/.changeset/increased_v2_allow_height_to_52400.md b/.changeset/increased_v2_allow_height_to_52400.md deleted file mode 100644 index cc96d47..0000000 --- a/.changeset/increased_v2_allow_height_to_52400.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: major ---- - -# Increased V2 Allow Height to 526,000 - -Delays activation of the v2 hardfork to June 6th, 2025 in response to concerns about the scope of updates necessary for partners to support v2 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 99b6c9e..ee746fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## 0.10.0 (2025-01-15) + +### Breaking Changes + +#### Increased V2 Allow Height to 526,000 + +Delays activation of the v2 hardfork to June 6th, 2025 in response to concerns about the scope of updates necessary for partners to support v2 + +### Fixes + +- Improve locking in SingleAddressWallet by avoiding acquiring the mutex before a db transaction +- Increased default UTXO reservation in SingleAddressWallet to 3 hours + ## 0.9.1 (2025-01-12) ### Fixes diff --git a/go.mod b/go.mod index 145f55b..fe4e8d4 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/coreutils // v0.9.1 +module go.sia.tech/coreutils // v0.10.0 go 1.23.1