From d24ee4c32ba36d80bc1e526abcfbbb9c9d93dbfc Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 11 Sep 2023 16:20:26 -0400
Subject: [PATCH 01/40] initial commit

---
 database/pebble/batch.go      | 111 +++++++++++
 database/pebble/batch_test.go |  47 +++++
 database/pebble/db.go         | 366 ++++++++++++++++++++++++++++++++++
 database/pebble/db_test.go    | 112 +++++++++++
 database/pebble/iterator.go   | 159 +++++++++++++++
 go.mod                        |  11 +-
 go.sum                        |  25 ++-
 network/p2p/client.go         |   2 +
 network/p2p/gossip/gossip.go  |   1 +
 network/p2p/router.go         |  16 +-
 network/p2p/router_test.go    |  17 +-
 11 files changed, 841 insertions(+), 26 deletions(-)
 create mode 100644 database/pebble/batch.go
 create mode 100644 database/pebble/batch_test.go
 create mode 100644 database/pebble/db.go
 create mode 100644 database/pebble/db_test.go
 create mode 100644 database/pebble/iterator.go

diff --git a/database/pebble/batch.go b/database/pebble/batch.go
new file mode 100644
index 000000000000..3b08df08aec8
--- /dev/null
+++ b/database/pebble/batch.go
@@ -0,0 +1,111 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package pebble
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/cockroachdb/pebble"
+
+	"github.com/ava-labs/avalanchego/database"
+)
+
+var _ database.Batch = (*batch)(nil)
+
+// Not safe for concurrent use.
+type batch struct {
+	batch *pebble.Batch
+	db    *Database
+	size  int
+
+	// True iff [batch] has been written to the database
+	// since the last time [Reset] was called.
+	written atomic.Bool
+}
+
+func (db *Database) NewBatch() database.Batch {
+	return &batch{
+		db:    db,
+		batch: db.pebbleDB.NewBatch(),
+	}
+}
+
+func (b *batch) Put(key, value []byte) error {
+	b.size += len(key) + len(value) + pebbleByteOverHead
+	return b.batch.Set(key, value, pebble.Sync)
+}
+
+func (b *batch) Delete(key []byte) error {
+	b.size += len(key) + pebbleByteOverHead
+	return b.batch.Delete(key, pebble.Sync)
+}
+
+func (b *batch) Size() int {
+	return b.size
+}
+
+// Assumes [b.db.lock] is not held.
+func (b *batch) Write() error {
+	b.db.lock.RLock()
+	defer b.db.lock.RUnlock()
+
+	// Committing to a closed database makes pebble panic
+	// so make sure [b.db] isn't closed.
+	if b.db.closed {
+		return database.ErrClosed
+	}
+
+	if b.written.CompareAndSwap(false, true) {
+		// This batch has not been written to the database yet.
+		return updateError(b.batch.Commit(pebble.Sync))
+	}
+
+	// pebble doesn't support writing a batch twice so we have to clone
+	// [b] and commit the clone.
+	clone := &batch{
+		db:    b.db,
+		batch: b.db.pebbleDB.NewBatch(),
+	}
+
+	// Copy the batch.
+	if err := clone.batch.Apply(b.batch, nil); err != nil {
+		return err
+	}
+
+	// Commit the new batch.
+	return updateError(clone.batch.Commit(pebble.Sync))
+}
+
+func (b *batch) Reset() {
+	b.batch.Reset()
+	b.written.Store(false)
+	b.size = 0
+}
+
+func (b *batch) Replay(w database.KeyValueWriterDeleter) error {
+	reader := b.batch.Reader()
+	for {
+		kind, k, v, ok := reader.Next()
+		if !ok {
+			return nil
+		}
+		switch kind {
+		case pebble.InternalKeyKindSet:
+			if err := w.Put(k, v); err != nil {
+				return err
+			}
+		case pebble.InternalKeyKindDelete:
+			if err := w.Delete(k); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("%w: %v", ErrInvalidOperation, kind)
+		}
+	}
+}
+
+func (b *batch) Inner() database.Batch {
+	return b
+}
diff --git a/database/pebble/batch_test.go b/database/pebble/batch_test.go
new file mode 100644
index 000000000000..4ec7b8e96e58
--- /dev/null
+++ b/database/pebble/batch_test.go
@@ -0,0 +1,47 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package pebble
+
+import (
+	"os"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ava-labs/avalanchego/utils/logging"
+)
+
+// Note: TestInterface tests other batch functionality.
+func TestBatch(t *testing.T) {
+	require := require.New(t)
+	dirName := os.TempDir()
+	defer os.Remove(dirName)
+
+	db, err := New(dirName, DefaultConfig, logging.NoLog{}, "", prometheus.NewRegistry())
+	require.NoError(err)
+
+	batchIntf := db.NewBatch()
+	batch, ok := batchIntf.(*batch)
+	require.True(ok)
+
+	require.False(batch.written.Load())
+
+	key1, value1 := []byte("key1"), []byte("value1")
+	require.NoError(batch.Put(key1, value1))
+	require.Equal(len(key1)+len(value1)+pebbleByteOverHead, batch.Size())
+
+	require.NoError(batch.Write())
+
+	require.True(batch.written.Load())
+
+	got, err := db.Get(key1)
+	require.NoError(err)
+	require.Equal(value1, got)
+
+	batch.Reset()
+	require.False(batch.written.Load())
+	require.Zero(batch.Size())
+}
diff --git a/database/pebble/db.go b/database/pebble/db.go
new file mode 100644
index 000000000000..c76b30436859
--- /dev/null
+++ b/database/pebble/db.go
@@ -0,0 +1,366 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package pebble
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"sync"
+
+	"github.com/cockroachdb/pebble"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"golang.org/x/exp/slices"
+
+	"go.uber.org/zap"
+
+	"github.com/ava-labs/avalanchego/database"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/ava-labs/avalanchego/utils/set"
+	"github.com/ava-labs/avalanchego/utils/units"
+)
+
+// pebbleByteOverHead is the number of bytes of constant overhead that
+// should be added to a batch size per operation.
+const pebbleByteOverHead = 8
+
+var (
+	_ database.Database = (*Database)(nil)
+
+	ErrInvalidOperation = errors.New("invalid operation")
+
+	defaultCacheSize = uint64(512 * units.MiB)
+	DefaultConfig    = Config{
+		CacheSize:                   int(defaultCacheSize),
+		BytesPerSync:                512 * units.KiB,
+		WALBytesPerSync:             0, // Default to no background syncing.
+		MemTableStopWritesThreshold: 8,
+		MemTableSize:                defaultCacheSize / 4,
+		MaxOpenFiles:                4096,
+	}
+
+	comparer = pebble.DefaultComparer
+)
+
+type Database struct {
+	lock          sync.RWMutex
+	pebbleDB      *pebble.DB
+	closed        bool
+	openIterators set.Set[*iter]
+}
+
+type Config struct {
+	CacheSize                   int
+	BytesPerSync                int
+	WALBytesPerSync             int // 0 means no background syncing
+	MemTableStopWritesThreshold int
+	MemTableSize                uint64
+	MaxOpenFiles                int
+	MaxConcurrentCompactions    int
+}
+
+// TODO: Add support for adding a custom logger
+// TODO: Add metrics
+func New(file string, cfg Config, log logging.Logger, _ string, _ prometheus.Registerer) (*Database, error) {
+	opts := &pebble.Options{
+		Cache:                       pebble.NewCache(int64(cfg.CacheSize)),
+		BytesPerSync:                cfg.BytesPerSync,
+		Comparer:                    comparer,
+		WALBytesPerSync:             cfg.WALBytesPerSync,
+		MemTableStopWritesThreshold: cfg.MemTableStopWritesThreshold,
+		MemTableSize:                cfg.MemTableSize,
+		MaxOpenFiles:                cfg.MaxOpenFiles,
+		MaxConcurrentCompactions:    func() int { return cfg.MaxConcurrentCompactions },
+	}
+	opts.Experimental.ReadSamplingMultiplier = -1 // explicitly disable seek compaction
+
+	log.Info("opening pebble")
+	log.Debug("config", zap.Any("config", opts))
+
+	db, err := pebble.Open(file, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Database{
+		pebbleDB:      db,
+		openIterators: set.Set[*iter]{},
+	}, nil
+}
+
+func (db *Database) Close() error {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	if db.closed {
+		return database.ErrClosed
+	}
+
+	db.closed = true
+
+	for iter := range db.openIterators {
+		iter.lock.Lock()
+		iter.release()
+		iter.lock.Unlock()
+	}
+	db.openIterators.Clear()
+
+	return updateError(db.pebbleDB.Close())
+}
+
+func (db *Database) HealthCheck(_ context.Context) (interface{}, error) {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	if db.closed {
+		return nil, database.ErrClosed
+	}
+	return nil, nil
+}
+
+func (db *Database) Has(key []byte) (bool, error) {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	if db.closed {
+		return false, database.ErrClosed
+	}
+
+	_, closer, err := db.pebbleDB.Get(key)
+	if err == pebble.ErrNotFound {
+		return false, nil
+	}
+	if err != nil {
+		return false, updateError(err)
+	}
+	return true, closer.Close()
+}
+
+func (db *Database) Get(key []byte) ([]byte, error) {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	if db.closed {
+		return nil, database.ErrClosed
+	}
+
+	data, closer, err := db.pebbleDB.Get(key)
+	if err != nil {
+		return nil, updateError(err)
+	}
+	return slices.Clone(data), closer.Close()
+}
+
+func (db *Database) Put(key []byte, value []byte) error {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	if db.closed {
+		return database.ErrClosed
+	}
+
+	return updateError(db.pebbleDB.Set(key, value, pebble.Sync))
+}
+
+func (db *Database) Delete(key []byte) error {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	if db.closed {
+		return database.ErrClosed
+	}
+
+	return updateError(db.pebbleDB.Delete(key, pebble.Sync))
+}
+
+func (db *Database) Compact(start []byte, limit []byte) error {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	switch {
+	case db.closed:
+		return database.ErrClosed
+	case comparer.Compare(start, limit) >= 0:
+		// pebble's Compact will no-op & error if start >= limit
+		// according to pebble's comparer.
+		return nil
+	case limit != nil:
+		return updateError(db.pebbleDB.Compact(start, limit, true /* parallelize */))
+	}
+
+	// The database.Database spec treats a nil [limit] as a key after all keys
+	// but pebble treats a nil [limit] as a key before all keys.
+	// Use the greatest key in the database as the [limit] to get the desired behavior.
+	it, err := db.pebbleDB.NewIter(&pebble.IterOptions{})
+	if err != nil {
+		return updateError(err)
+	}
+	if it.Last() {
+		if lastkey := it.Key(); lastkey != nil {
+			return updateError(db.pebbleDB.Compact(start, lastkey, true /* parallelize */))
+		}
+	}
+
+	// Either this database is empty or the only key in it is nil.
+	return nil
+}
+
+func (db *Database) NewIterator() database.Iterator {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	if db.closed {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    database.ErrClosed,
+		}
+	}
+
+	innerIter, err := db.pebbleDB.NewIter(&pebble.IterOptions{})
+	if err != nil {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    updateError(err),
+		}
+	}
+
+	iter := &iter{
+		db:   db,
+		iter: innerIter,
+	}
+	db.openIterators.Add(iter)
+	return iter
+}
+
+func (db *Database) NewIteratorWithStart(start []byte) database.Iterator {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	if db.closed {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    database.ErrClosed,
+		}
+	}
+
+	innerIter, err := db.pebbleDB.NewIter(&pebble.IterOptions{
+		LowerBound: start,
+	})
+	if err != nil {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    updateError(err),
+		}
+	}
+
+	iter := &iter{
+		db:   db,
+		iter: innerIter,
+	}
+	db.openIterators.Add(iter)
+	return iter
+}
+
+func (db *Database) NewIteratorWithPrefix(prefix []byte) database.Iterator {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	if db.closed {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    database.ErrClosed,
+		}
+	}
+
+	innerIter, err := db.pebbleDB.NewIter(prefixBounds(prefix))
+	if err != nil {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    updateError(err),
+		}
+	}
+
+	iter := &iter{
+		db:   db,
+		iter: innerIter,
+	}
+	db.openIterators.Add(iter)
+	return iter
+}
+
+func (db *Database) NewIteratorWithStartAndPrefix(start, prefix []byte) database.Iterator {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	if db.closed {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    database.ErrClosed,
+		}
+	}
+
+	iterRange := prefixBounds(prefix)
+	if bytes.Compare(start, prefix) == 1 {
+		iterRange.LowerBound = start
+	}
+
+	innerIter, err := db.pebbleDB.NewIter(iterRange)
+	if err != nil {
+		return &iter{
+			db:     db,
+			closed: true,
+			err:    updateError(err),
+		}
+	}
+
+	iter := &iter{
+		db:   db,
+		iter: innerIter,
+	}
+	db.openIterators.Add(iter)
+	return iter
+}
+
+// Converts a pebble-specific error to to its
+// Avalanche equivalent, if applicable.
+func updateError(err error) error {
+	switch err {
+	case pebble.ErrClosed:
+		return database.ErrClosed
+	case pebble.ErrNotFound:
+		return database.ErrNotFound
+	default:
+		return err
+	}
+}
+
+// Returns a key range that covers all keys with the
+// given [prefix].
+// Assumes the Database uses bytes.Compare for key
+// comparison and not a custom comparer.
+func prefixBounds(prefix []byte) *pebble.IterOptions {
+	var upperBound []byte
+	for i := len(prefix) - 1; i >= 0; i-- {
+		c := prefix[i]
+		if c < 0xFF {
+			upperBound = make([]byte, i+1)
+			copy(upperBound, prefix)
+			upperBound[i] = c + 1
+			break
+		}
+	}
+	return &pebble.IterOptions{
+		LowerBound: prefix,
+		UpperBound: upperBound,
+	}
+}
diff --git a/database/pebble/db_test.go b/database/pebble/db_test.go
new file mode 100644
index 000000000000..2660da5e8d61
--- /dev/null
+++ b/database/pebble/db_test.go
@@ -0,0 +1,112 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package pebble
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ava-labs/avalanchego/database"
+	"github.com/ava-labs/avalanchego/utils/logging"
+)
+
+func TestInterface(t *testing.T) {
+	for _, test := range database.Tests {
+		folder := t.TempDir()
+		cfg := DefaultConfig
+		db, err := New(folder, cfg, logging.NoLog{}, "pebble", prometheus.NewRegistry())
+		require.NoError(t, err)
+
+		test(t, db)
+
+		_ = db.Close()
+	}
+}
+
+func FuzzInterface(f *testing.F) {
+	for _, test := range database.FuzzTests {
+		tmpDir := f.TempDir()
+		db, err := New(tmpDir, DefaultConfig, logging.NoLog{}, "", prometheus.NewRegistry())
+		require.NoError(f, err)
+
+		test(f, db)
+
+		_ = db.Close()
+	}
+}
+
+func BenchmarkInterface(b *testing.B) {
+	for _, size := range database.BenchmarkSizes {
+		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])
+		for _, bench := range database.Benchmarks {
+			folder := b.TempDir()
+			cfg := DefaultConfig
+
+			db, err := New(folder, cfg, logging.NoLog{}, "", prometheus.NewRegistry())
+			require.NoError(b, err)
+			bench(b, db, "pebble", keys, values)
+
+			_ = db.Close()
+		}
+	}
+}
+
+func TestPrefixBounds(t *testing.T) {
+	require := require.New(t)
+
+	type test struct {
+		prefix        []byte
+		expectedLower []byte
+		expectedUpper []byte
+	}
+
+	tests := []test{
+		{
+			prefix:        nil,
+			expectedLower: nil,
+			expectedUpper: nil,
+		},
+		{
+			prefix:        []byte{},
+			expectedLower: []byte{},
+			expectedUpper: nil,
+		},
+		{
+			prefix:        []byte{0x00},
+			expectedLower: []byte{0x00},
+			expectedUpper: []byte{0x01},
+		},
+		{
+			prefix:        []byte{0x01},
+			expectedLower: []byte{0x01},
+			expectedUpper: []byte{0x02},
+		},
+		{
+			prefix:        []byte{0xFF},
+			expectedLower: []byte{0xFF},
+			expectedUpper: nil,
+		},
+		{
+			prefix:        []byte{0x01, 0x02},
+			expectedLower: []byte{0x01, 0x02},
+			expectedUpper: []byte{0x01, 0x03},
+		},
+		{
+			prefix:        []byte{0x01, 0x02, 0xFF},
+			expectedLower: []byte{0x01, 0x02, 0xFF},
+			expectedUpper: []byte{0x01, 0x03},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(string(tt.prefix), func(t *testing.T) {
+			itopts := prefixBounds(tt.prefix)
+			require.Equal(tt.expectedLower, itopts.LowerBound)
+			require.Equal(tt.expectedUpper, itopts.UpperBound)
+		})
+	}
+}
diff --git a/database/pebble/iterator.go b/database/pebble/iterator.go
new file mode 100644
index 000000000000..136d4f97b81b
--- /dev/null
+++ b/database/pebble/iterator.go
@@ -0,0 +1,159 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package pebble
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/cockroachdb/pebble"
+
+	"golang.org/x/exp/slices"
+
+	"github.com/ava-labs/avalanchego/database"
+)
+
+var (
+	_ database.Iterator = (*iter)(nil)
+
+	errCouldntGetValue = errors.New("couldnt get iterator value")
+	errNoPointKey      = errors.New("iterator has no point key")
+)
+
+type iter struct {
+	// [lock] ensures that only one goroutine can access [iter] at a time.
+	// Note that [Database.Close] calls [iter.Release] so we need [lock] to ensure
+	// that the user and [Database.Close] don't execute [iter.Release] concurrently.
+	// Invariant: [Database.lock] is never grabbed while holding [lock].
+	lock sync.Mutex
+
+	db   *Database
+	iter *pebble.Iterator
+
+	initialized bool
+	closed      bool
+	err         error
+
+	hasNext bool
+	nextKey []byte
+	nextVal []byte
+}
+
+// Must not be called with [db.lock] held.
+func (it *iter) Next() bool {
+	it.db.lock.RLock()
+	dbClosed := it.db.closed
+	it.db.lock.RUnlock()
+
+	it.lock.Lock()
+	defer it.lock.Unlock()
+
+	switch {
+	case it.closed || dbClosed:
+		it.hasNext = false
+		it.err = database.ErrClosed
+		return false
+	case it.err != nil:
+		it.hasNext = false
+		return false
+	case !it.initialized:
+		it.hasNext = it.iter.First()
+		it.initialized = true
+	default:
+		it.hasNext = it.iter.Next()
+	}
+
+	if !it.hasNext {
+		return false
+	}
+
+	// Set the next key.
+	it.nextKey = it.iter.Key()
+
+	// Set the next value.
+	// TODO is the below necessary?
+	// Need to make sure the following invariant for [i.iter.ValueAndErr] holds:
+	// "REQUIRES: iter.Error() == nil and HasPointAndRange() returns true for hasPoint."
+	hasPoint, _ := it.iter.HasPointAndRange()
+	if !hasPoint {
+		it.hasNext = false
+		it.err = fmt.Errorf("%w: %w", errCouldntGetValue, errNoPointKey)
+		return false
+	}
+
+	if err := it.iter.Error(); err != nil {
+		it.hasNext = false
+		it.err = fmt.Errorf("%w: %w", errCouldntGetValue, err)
+		return false
+	}
+
+	var err error
+	it.nextVal, err = it.iter.ValueAndErr()
+	if err != nil {
+		it.hasNext = false
+		it.err = fmt.Errorf("%w: %w", errCouldntGetValue, err)
+		return false
+	}
+
+	return true
+}
+
+func (it *iter) Error() error {
+	it.lock.Lock()
+	defer it.lock.Unlock()
+
+	if it.err != nil {
+		return it.err
+	}
+	if it.closed {
+		return nil
+	}
+	return updateError(it.iter.Error())
+}
+
+func (it *iter) Key() []byte {
+	it.lock.Lock()
+	defer it.lock.Unlock()
+
+	if !it.hasNext {
+		return nil
+	}
+	return slices.Clone(it.nextKey)
+}
+
+func (it *iter) Value() []byte {
+	it.lock.Lock()
+	defer it.lock.Unlock()
+
+	if !it.hasNext {
+		return nil
+	}
+	return slices.Clone(it.nextVal)
+}
+
+func (it *iter) Release() {
+	it.db.lock.Lock()
+	defer it.db.lock.Unlock()
+
+	it.lock.Lock()
+	defer it.lock.Unlock()
+
+	it.release()
+}
+
+// Assumes [it.lock] and [it.db.lock] are held.
+func (it *iter) release() {
+	if it.closed {
+		return
+	}
+
+	// Remove the iterator from the list of open iterators.
+	it.db.openIterators.Remove(it)
+
+	it.closed = true
+	if err := it.iter.Close(); err != nil {
+		it.err = updateError(err)
+	}
+}
diff --git a/go.mod b/go.mod
index 01ce36526314..a8bf0af4f18b 100644
--- a/go.mod
+++ b/go.mod
@@ -15,6 +15,7 @@ require (
 	github.com/ava-labs/coreth v0.12.5-rc.3
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20230105152938-00a24d05a8c7
 	github.com/btcsuite/btcd/btcutil v1.1.3
+	github.com/cockroachdb/pebble v0.0.0-20230905224231-5283f24b2a69
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0
 	github.com/ethereum/go-ethereum v1.12.0
 	github.com/golang-jwt/jwt/v4 v4.3.0
@@ -56,7 +57,7 @@ require (
 	go.uber.org/mock v0.2.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.1.0
-	golang.org/x/exp v0.0.0-20230206171751-46f607a40771
+	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
 	golang.org/x/sync v0.1.0
 	golang.org/x/term v0.7.0
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af
@@ -68,6 +69,7 @@ require (
 )
 
 require (
+	github.com/BurntSushi/toml v1.2.1 // indirect
 	github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -76,14 +78,15 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
-	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 // indirect
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/frankban/quicktest v1.14.4 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
 	github.com/getsentry/sentry-go v0.18.0 // indirect
@@ -122,7 +125,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sanity-io/litter v1.5.1 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
@@ -142,7 +145,7 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
diff --git a/go.sum b/go.sum
index 96d9a5198ae9..5faf5d93a8a6 100644
--- a/go.sum
+++ b/go.sum
@@ -38,7 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
@@ -122,17 +123,19 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
 github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
 github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8=
 github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk=
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk=
-github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM=
+github.com/cockroachdb/pebble v0.0.0-20230905224231-5283f24b2a69 h1:H21xjLBzMsXkWC5DYV6H3pqrTRjP+njUn4ZdqGuf0Zc=
+github.com/cockroachdb/pebble v0.0.0-20230905224231-5283f24b2a69/go.mod h1:nindLFinxeDPjP4qI9LtVHAwDef57/0s5KMfWgdktQc=
 github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -188,7 +191,8 @@ github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
-github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
@@ -522,8 +526,9 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -697,8 +702,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -866,8 +871,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
diff --git a/network/p2p/client.go b/network/p2p/client.go
index 796aa5683cf7..0e48e210e19d 100644
--- a/network/p2p/client.go
+++ b/network/p2p/client.go
@@ -23,6 +23,7 @@ var (
 // issued by Client.
 // Callers should check [err] to see whether the AppRequest failed or not.
 type AppResponseCallback func(
+	ctx context.Context,
 	nodeID ids.NodeID,
 	responseBytes []byte,
 	err error,
@@ -32,6 +33,7 @@ type AppResponseCallback func(
 // CrossChainAppResponse for a CrossChainAppRequest issued by Client.
 // Callers should check [err] to see whether the AppRequest failed or not.
 type CrossChainAppResponseCallback func(
+	ctx context.Context,
 	chainID ids.ID,
 	responseBytes []byte,
 	err error,
diff --git a/network/p2p/gossip/gossip.go b/network/p2p/gossip/gossip.go
index 3bf08904fa5b..f48391782fa6 100644
--- a/network/p2p/gossip/gossip.go
+++ b/network/p2p/gossip/gossip.go
@@ -92,6 +92,7 @@ func (g *Gossiper[_, _]) gossip(ctx context.Context) error {
 }
 
 func (g *Gossiper[T, U]) handleResponse(
+	_ context.Context,
 	nodeID ids.NodeID,
 	responseBytes []byte,
 	err error,
diff --git a/network/p2p/router.go b/network/p2p/router.go
index ffd77cda01ea..5723cef1ba9d 100644
--- a/network/p2p/router.go
+++ b/network/p2p/router.go
@@ -95,23 +95,23 @@ func (r *Router) AppRequest(ctx context.Context, nodeID ids.NodeID, requestID ui
 	return handler.AppRequest(ctx, nodeID, requestID, deadline, parsedMsg)
 }
 
-func (r *Router) AppRequestFailed(_ context.Context, nodeID ids.NodeID, requestID uint32) error {
+func (r *Router) AppRequestFailed(ctx context.Context, nodeID ids.NodeID, requestID uint32) error {
 	callback, ok := r.clearAppRequest(requestID)
 	if !ok {
 		return ErrUnrequestedResponse
 	}
 
-	callback(nodeID, nil, ErrAppRequestFailed)
+	callback(ctx, nodeID, nil, ErrAppRequestFailed)
 	return nil
 }
 
-func (r *Router) AppResponse(_ context.Context, nodeID ids.NodeID, requestID uint32, response []byte) error {
+func (r *Router) AppResponse(ctx context.Context, nodeID ids.NodeID, requestID uint32, response []byte) error {
 	callback, ok := r.clearAppRequest(requestID)
 	if !ok {
 		return ErrUnrequestedResponse
 	}
 
-	callback(nodeID, response, nil)
+	callback(ctx, nodeID, response, nil)
 	return nil
 }
 
@@ -151,23 +151,23 @@ func (r *Router) CrossChainAppRequest(
 	return handler.CrossChainAppRequest(ctx, chainID, requestID, deadline, parsedMsg)
 }
 
-func (r *Router) CrossChainAppRequestFailed(_ context.Context, chainID ids.ID, requestID uint32) error {
+func (r *Router) CrossChainAppRequestFailed(ctx context.Context, chainID ids.ID, requestID uint32) error {
 	callback, ok := r.clearCrossChainAppRequest(requestID)
 	if !ok {
 		return ErrUnrequestedResponse
 	}
 
-	callback(chainID, nil, ErrAppRequestFailed)
+	callback(ctx, chainID, nil, ErrAppRequestFailed)
 	return nil
 }
 
-func (r *Router) CrossChainAppResponse(_ context.Context, chainID ids.ID, requestID uint32, response []byte) error {
+func (r *Router) CrossChainAppResponse(ctx context.Context, chainID ids.ID, requestID uint32, response []byte) error {
 	callback, ok := r.clearCrossChainAppRequest(requestID)
 	if !ok {
 		return ErrUnrequestedResponse
 	}
 
-	callback(chainID, response, nil)
+	callback(ctx, chainID, response, nil)
 	return nil
 }
 
diff --git a/network/p2p/router_test.go b/network/p2p/router_test.go
index 90cedd2976b2..c35fe9fee989 100644
--- a/network/p2p/router_test.go
+++ b/network/p2p/router_test.go
@@ -27,6 +27,11 @@ func TestAppRequestResponse(t *testing.T) {
 	nodeID := ids.GenerateTestNodeID()
 	chainID := ids.GenerateTestID()
 
+	ctxKey := new(string)
+	ctxVal := new(string)
+	*ctxKey = "foo"
+	*ctxVal = "bar"
+
 	tests := []struct {
 		name        string
 		requestFunc func(t *testing.T, router *Router, client *Client, sender *common.MockSender, handler *mocks.MockHandler, wg *sync.WaitGroup)
@@ -45,6 +50,7 @@ func TestAppRequestResponse(t *testing.T) {
 				sender.EXPECT().SendAppResponse(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 					Do(func(ctx context.Context, _ ids.NodeID, requestID uint32, response []byte) {
 						go func() {
+							ctx = context.WithValue(ctx, ctxKey, ctxVal)
 							require.NoError(t, router.AppResponse(ctx, nodeID, requestID, response))
 						}()
 					}).AnyTimes()
@@ -54,10 +60,11 @@ func TestAppRequestResponse(t *testing.T) {
 						return response, nil
 					})
 
-				callback := func(actualNodeID ids.NodeID, actualResponse []byte, err error) {
+				callback := func(ctx context.Context, actualNodeID ids.NodeID, actualResponse []byte, err error) {
 					defer wg.Done()
 
 					require.NoError(t, err)
+					require.Equal(t, ctxVal, ctx.Value(ctxKey))
 					require.Equal(t, nodeID, actualNodeID)
 					require.Equal(t, response, actualResponse)
 				}
@@ -77,7 +84,7 @@ func TestAppRequestResponse(t *testing.T) {
 						}
 					})
 
-				callback := func(actualNodeID ids.NodeID, actualResponse []byte, err error) {
+				callback := func(_ context.Context, actualNodeID ids.NodeID, actualResponse []byte, err error) {
 					defer wg.Done()
 
 					require.ErrorIs(t, err, ErrAppRequestFailed)
@@ -101,6 +108,7 @@ func TestAppRequestResponse(t *testing.T) {
 				sender.EXPECT().SendCrossChainAppResponse(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 					Do(func(ctx context.Context, chainID ids.ID, requestID uint32, response []byte) {
 						go func() {
+							ctx = context.WithValue(ctx, ctxKey, ctxVal)
 							require.NoError(t, router.CrossChainAppResponse(ctx, chainID, requestID, response))
 						}()
 					}).AnyTimes()
@@ -110,9 +118,10 @@ func TestAppRequestResponse(t *testing.T) {
 						return response, nil
 					})
 
-				callback := func(actualChainID ids.ID, actualResponse []byte, err error) {
+				callback := func(ctx context.Context, actualChainID ids.ID, actualResponse []byte, err error) {
 					defer wg.Done()
 					require.NoError(t, err)
+					require.Equal(t, ctxVal, ctx.Value(ctxKey))
 					require.Equal(t, chainID, actualChainID)
 					require.Equal(t, response, actualResponse)
 				}
@@ -130,7 +139,7 @@ func TestAppRequestResponse(t *testing.T) {
 						}()
 					})
 
-				callback := func(actualChainID ids.ID, actualResponse []byte, err error) {
+				callback := func(_ context.Context, actualChainID ids.ID, actualResponse []byte, err error) {
 					defer wg.Done()
 
 					require.ErrorIs(t, err, ErrAppRequestFailed)

From db2eeeb75ec77358c2658af2b6b4bd9713dfc54c Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 11 Sep 2023 16:50:25 -0400
Subject: [PATCH 02/40] update pebble dep

---
 database/pebble/db.go | 59 ++++++++-----------------------------------
 go.mod                |  3 +--
 go.sum                |  8 +++---
 3 files changed, 14 insertions(+), 56 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index c76b30436859..72fa12e9ccc3 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -32,7 +32,7 @@ var (
 
 	ErrInvalidOperation = errors.New("invalid operation")
 
-	defaultCacheSize = uint64(512 * units.MiB)
+	defaultCacheSize = 512 * units.MiB
 	DefaultConfig    = Config{
 		CacheSize:                   int(defaultCacheSize),
 		BytesPerSync:                512 * units.KiB,
@@ -57,7 +57,7 @@ type Config struct {
 	BytesPerSync                int
 	WALBytesPerSync             int // 0 means no background syncing
 	MemTableStopWritesThreshold int
-	MemTableSize                uint64
+	MemTableSize                int
 	MaxOpenFiles                int
 	MaxConcurrentCompactions    int
 }
@@ -194,10 +194,7 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 	// The database.Database spec treats a nil [limit] as a key after all keys
 	// but pebble treats a nil [limit] as a key before all keys.
 	// Use the greatest key in the database as the [limit] to get the desired behavior.
-	it, err := db.pebbleDB.NewIter(&pebble.IterOptions{})
-	if err != nil {
-		return updateError(err)
-	}
+	it := db.pebbleDB.NewIter(&pebble.IterOptions{})
 	if it.Last() {
 		if lastkey := it.Key(); lastkey != nil {
 			return updateError(db.pebbleDB.Compact(start, lastkey, true /* parallelize */))
@@ -220,18 +217,9 @@ func (db *Database) NewIterator() database.Iterator {
 		}
 	}
 
-	innerIter, err := db.pebbleDB.NewIter(&pebble.IterOptions{})
-	if err != nil {
-		return &iter{
-			db:     db,
-			closed: true,
-			err:    updateError(err),
-		}
-	}
-
 	iter := &iter{
 		db:   db,
-		iter: innerIter,
+		iter: db.pebbleDB.NewIter(&pebble.IterOptions{}),
 	}
 	db.openIterators.Add(iter)
 	return iter
@@ -249,20 +237,11 @@ func (db *Database) NewIteratorWithStart(start []byte) database.Iterator {
 		}
 	}
 
-	innerIter, err := db.pebbleDB.NewIter(&pebble.IterOptions{
-		LowerBound: start,
-	})
-	if err != nil {
-		return &iter{
-			db:     db,
-			closed: true,
-			err:    updateError(err),
-		}
-	}
-
 	iter := &iter{
-		db:   db,
-		iter: innerIter,
+		db: db,
+		iter: db.pebbleDB.NewIter(&pebble.IterOptions{
+			LowerBound: start,
+		}),
 	}
 	db.openIterators.Add(iter)
 	return iter
@@ -280,18 +259,9 @@ func (db *Database) NewIteratorWithPrefix(prefix []byte) database.Iterator {
 		}
 	}
 
-	innerIter, err := db.pebbleDB.NewIter(prefixBounds(prefix))
-	if err != nil {
-		return &iter{
-			db:     db,
-			closed: true,
-			err:    updateError(err),
-		}
-	}
-
 	iter := &iter{
 		db:   db,
-		iter: innerIter,
+		iter: db.pebbleDB.NewIter(prefixBounds(prefix)),
 	}
 	db.openIterators.Add(iter)
 	return iter
@@ -314,18 +284,9 @@ func (db *Database) NewIteratorWithStartAndPrefix(start, prefix []byte) database
 		iterRange.LowerBound = start
 	}
 
-	innerIter, err := db.pebbleDB.NewIter(iterRange)
-	if err != nil {
-		return &iter{
-			db:     db,
-			closed: true,
-			err:    updateError(err),
-		}
-	}
-
 	iter := &iter{
 		db:   db,
-		iter: innerIter,
+		iter: db.pebbleDB.NewIter(iterRange),
 	}
 	db.openIterators.Add(iter)
 	return iter
diff --git a/go.mod b/go.mod
index a8bf0af4f18b..5699692d1963 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/ava-labs/coreth v0.12.5-rc.3
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20230105152938-00a24d05a8c7
 	github.com/btcsuite/btcd/btcutil v1.1.3
-	github.com/cockroachdb/pebble v0.0.0-20230905224231-5283f24b2a69
+	github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0
 	github.com/ethereum/go-ethereum v1.12.0
 	github.com/golang-jwt/jwt/v4 v4.3.0
@@ -79,7 +79,6 @@ require (
 	github.com/cockroachdb/errors v1.9.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
-	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
diff --git a/go.sum b/go.sum
index 5faf5d93a8a6..e2622f7916b9 100644
--- a/go.sum
+++ b/go.sum
@@ -124,18 +124,16 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
-github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
+github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o=
 github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8=
 github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk=
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230905224231-5283f24b2a69 h1:H21xjLBzMsXkWC5DYV6H3pqrTRjP+njUn4ZdqGuf0Zc=
-github.com/cockroachdb/pebble v0.0.0-20230905224231-5283f24b2a69/go.mod h1:nindLFinxeDPjP4qI9LtVHAwDef57/0s5KMfWgdktQc=
+github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w=
+github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s=
 github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
-github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
-github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

From 528cc2d0cf29820f1ad6685b4696ded3be53ee0b Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Tue, 12 Sep 2023 10:58:00 -0400
Subject: [PATCH 03/40] appease linter

---
 database/pebble/db.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 72fa12e9ccc3..13bcd3acb2f1 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -34,7 +34,7 @@ var (
 
 	defaultCacheSize = 512 * units.MiB
 	DefaultConfig    = Config{
-		CacheSize:                   int(defaultCacheSize),
+		CacheSize:                   defaultCacheSize,
 		BytesPerSync:                512 * units.KiB,
 		WALBytesPerSync:             0, // Default to no background syncing.
 		MemTableStopWritesThreshold: 8,

From 1bcfa656fc44a92829818d3a914a4460377af7bd Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Tue, 12 Sep 2023 11:35:57 -0400
Subject: [PATCH 04/40] nit

---
 database/pebble/db.go | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 13bcd3acb2f1..46dd887b8a91 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -305,20 +305,19 @@ func updateError(err error) error {
 	}
 }
 
-// Returns a key range that covers all keys with the
-// given [prefix].
-// Assumes the Database uses bytes.Compare for key
-// comparison and not a custom comparer.
+// Returns a key range that covers all keys with the given [prefix].
+// Assumes the Database uses bytes.Compare for key comparison and
+// not a custom comparer.
 func prefixBounds(prefix []byte) *pebble.IterOptions {
 	var upperBound []byte
 	for i := len(prefix) - 1; i >= 0; i-- {
-		c := prefix[i]
-		if c < 0xFF {
-			upperBound = make([]byte, i+1)
-			copy(upperBound, prefix)
-			upperBound[i] = c + 1
-			break
+		if prefix[i] == 0xFF {
+			continue
 		}
+		upperBound = make([]byte, i+1)
+		copy(upperBound, prefix)
+		upperBound[i] = prefix[i] + 1
+		break
 	}
 	return &pebble.IterOptions{
 		LowerBound: prefix,

From 73a934f55c7b64a5ae97647d34bdd4407315ae89 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Tue, 12 Sep 2023 11:42:22 -0400
Subject: [PATCH 05/40] nit

---
 database/pebble/db.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 46dd887b8a91..0544c0255e53 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -320,7 +320,7 @@ func prefixBounds(prefix []byte) *pebble.IterOptions {
 		break
 	}
 	return &pebble.IterOptions{
-		LowerBound: prefix,
-		UpperBound: upperBound,
+		LowerBound: prefix,     // Note this bound is inclusive.
+		UpperBound: upperBound, // Note this bound is exclusive.
 	}
 }

From 1e094b46ed9f7bc5102cfbf19b2a3af213e597d7 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Tue, 12 Sep 2023 11:44:34 -0400
Subject: [PATCH 06/40] nit

---
 database/pebble/db_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/database/pebble/db_test.go b/database/pebble/db_test.go
index 2660da5e8d61..13e5fb13b95a 100644
--- a/database/pebble/db_test.go
+++ b/database/pebble/db_test.go
@@ -104,9 +104,9 @@ func TestPrefixBounds(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(string(tt.prefix), func(t *testing.T) {
-			itopts := prefixBounds(tt.prefix)
-			require.Equal(tt.expectedLower, itopts.LowerBound)
-			require.Equal(tt.expectedUpper, itopts.UpperBound)
+			bounds := prefixBounds(tt.prefix)
+			require.Equal(tt.expectedLower, bounds.LowerBound)
+			require.Equal(tt.expectedUpper, bounds.UpperBound)
 		})
 	}
 }

From 721275e28a9cab07dc2ea3bba38e0ed8d39f91d0 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Fri, 15 Sep 2023 16:10:08 -0400
Subject: [PATCH 07/40] nit

---
 database/pebble/db.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 0544c0255e53..34498adeb4b4 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -192,7 +192,7 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 	}
 
 	// The database.Database spec treats a nil [limit] as a key after all keys
-	// but pebble treats a nil [limit] as a key before all keys.
+	// but pebble treats a nil [limit] as a key before all keys in Compact.
 	// Use the greatest key in the database as the [limit] to get the desired behavior.
 	it := db.pebbleDB.NewIter(&pebble.IterOptions{})
 	if it.Last() {

From 9f7665b0a75d95e1de191085c0f3923d6ad89ba5 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 18 Sep 2023 09:20:26 -0400
Subject: [PATCH 08/40] fix tests

---
 database/pebble/db_test.go | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/database/pebble/db_test.go b/database/pebble/db_test.go
index 13e5fb13b95a..53c063948e7b 100644
--- a/database/pebble/db_test.go
+++ b/database/pebble/db_test.go
@@ -14,6 +14,14 @@ import (
 	"github.com/ava-labs/avalanchego/utils/logging"
 )
 
+func newDB(t testing.TB) *Database {
+	folder := t.TempDir()
+	cfg := DefaultConfig
+	db, err := New(folder, cfg, logging.NoLog{}, "pebble", prometheus.NewRegistry())
+	require.NoError(t, err)
+	return db
+}
+
 func TestInterface(t *testing.T) {
 	for _, test := range database.Tests {
 		folder := t.TempDir()
@@ -27,16 +35,12 @@ func TestInterface(t *testing.T) {
 	}
 }
 
-func FuzzInterface(f *testing.F) {
-	for _, test := range database.FuzzTests {
-		tmpDir := f.TempDir()
-		db, err := New(tmpDir, DefaultConfig, logging.NoLog{}, "", prometheus.NewRegistry())
-		require.NoError(f, err)
-
-		test(f, db)
+func FuzzKeyValue(f *testing.F) {
+	database.FuzzKeyValue(f, newDB(f))
+}
 
-		_ = db.Close()
-	}
+func FuzzNewIteratorWithPrefix(f *testing.F) {
+	database.FuzzNewIteratorWithPrefix(f, newDB(f))
 }
 
 func BenchmarkInterface(b *testing.B) {

From 1db44c4df75a506309e3b7d11717e3b18f778974 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 18 Sep 2023 09:21:52 -0400
Subject: [PATCH 09/40] nit

---
 database/pebble/db_test.go | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/database/pebble/db_test.go b/database/pebble/db_test.go
index 53c063948e7b..81bd0e18d48e 100644
--- a/database/pebble/db_test.go
+++ b/database/pebble/db_test.go
@@ -24,36 +24,30 @@ func newDB(t testing.TB) *Database {
 
 func TestInterface(t *testing.T) {
 	for _, test := range database.Tests {
-		folder := t.TempDir()
-		cfg := DefaultConfig
-		db, err := New(folder, cfg, logging.NoLog{}, "pebble", prometheus.NewRegistry())
-		require.NoError(t, err)
-
+		db := newDB(t)
 		test(t, db)
-
 		_ = db.Close()
 	}
 }
 
 func FuzzKeyValue(f *testing.F) {
-	database.FuzzKeyValue(f, newDB(f))
+	db := newDB(f)
+	database.FuzzKeyValue(f, db)
+	_ = db.Close()
 }
 
 func FuzzNewIteratorWithPrefix(f *testing.F) {
-	database.FuzzNewIteratorWithPrefix(f, newDB(f))
+	db := newDB(f)
+	database.FuzzNewIteratorWithPrefix(f, db)
+	_ = db.Close()
 }
 
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])
 		for _, bench := range database.Benchmarks {
-			folder := b.TempDir()
-			cfg := DefaultConfig
-
-			db, err := New(folder, cfg, logging.NoLog{}, "", prometheus.NewRegistry())
-			require.NoError(b, err)
+			db := newDB(b)
 			bench(b, db, "pebble", keys, values)
-
 			_ = db.Close()
 		}
 	}

From 4051e88e801c6761e51a56b8565322f74e176815 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Tue, 19 Sep 2023 13:54:34 -0400
Subject: [PATCH 10/40] nit

---
 database/pebble/batch.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/database/pebble/batch.go b/database/pebble/batch.go
index 3b08df08aec8..42aefe403422 100644
--- a/database/pebble/batch.go
+++ b/database/pebble/batch.go
@@ -64,18 +64,15 @@ func (b *batch) Write() error {
 
 	// pebble doesn't support writing a batch twice so we have to clone
 	// [b] and commit the clone.
-	clone := &batch{
-		db:    b.db,
-		batch: b.db.pebbleDB.NewBatch(),
-	}
+	batchClone := b.db.pebbleDB.NewBatch()
 
 	// Copy the batch.
-	if err := clone.batch.Apply(b.batch, nil); err != nil {
+	if err := batchClone.Apply(b.batch, nil); err != nil {
 		return err
 	}
 
 	// Commit the new batch.
-	return updateError(clone.batch.Commit(pebble.Sync))
+	return updateError(batchClone.Commit(pebble.Sync))
 }
 
 func (b *batch) Reset() {

From cd0a8e218f62f4c6bba82c7644f8c2e02a804143 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 25 Sep 2023 14:52:52 -0400
Subject: [PATCH 11/40] appease linter

---
 database/pebble/db.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 34498adeb4b4..15ec7f02f3d1 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -292,7 +292,7 @@ func (db *Database) NewIteratorWithStartAndPrefix(start, prefix []byte) database
 	return iter
 }
 
-// Converts a pebble-specific error to to its
+// Converts a pebble-specific error to its
 // Avalanche equivalent, if applicable.
 func updateError(err error) error {
 	switch err {

From a710f5483f7b913fb8cfb983cdbf5f24ed00601f Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 2 Oct 2023 10:34:53 -0400
Subject: [PATCH 12/40] remove TODO

---
 database/pebble/db.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 15ec7f02f3d1..8390f34066b4 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -62,7 +62,6 @@ type Config struct {
 	MaxConcurrentCompactions    int
 }
 
-// TODO: Add support for adding a custom logger
 // TODO: Add metrics
 func New(file string, cfg Config, log logging.Logger, _ string, _ prometheus.Registerer) (*Database, error) {
 	opts := &pebble.Options{

From f21c81dd07af91bd240be5965194aa60da7ef818 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 2 Oct 2023 10:35:34 -0400
Subject: [PATCH 13/40] remove unneeded var

---
 database/pebble/db.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 8390f34066b4..4b90b118a0b8 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -41,8 +41,6 @@ var (
 		MemTableSize:                defaultCacheSize / 4,
 		MaxOpenFiles:                4096,
 	}
-
-	comparer = pebble.DefaultComparer
 )
 
 type Database struct {
@@ -67,7 +65,7 @@ func New(file string, cfg Config, log logging.Logger, _ string, _ prometheus.Reg
 	opts := &pebble.Options{
 		Cache:                       pebble.NewCache(int64(cfg.CacheSize)),
 		BytesPerSync:                cfg.BytesPerSync,
-		Comparer:                    comparer,
+		Comparer:                    pebble.DefaultComparer,
 		WALBytesPerSync:             cfg.WALBytesPerSync,
 		MemTableStopWritesThreshold: cfg.MemTableStopWritesThreshold,
 		MemTableSize:                cfg.MemTableSize,
@@ -182,7 +180,7 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 	switch {
 	case db.closed:
 		return database.ErrClosed
-	case comparer.Compare(start, limit) >= 0:
+	case pebble.DefaultComparer.Compare(start, limit) >= 0:
 		// pebble's Compact will no-op & error if start >= limit
 		// according to pebble's comparer.
 		return nil

From cc7a15e13f0fb8fc40b9c59f3c4b45b68809f3f7 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 2 Oct 2023 10:36:09 -0400
Subject: [PATCH 14/40] unexport error

---
 database/pebble/batch.go | 2 +-
 database/pebble/db.go    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/database/pebble/batch.go b/database/pebble/batch.go
index 42aefe403422..e2026d6d7ec5 100644
--- a/database/pebble/batch.go
+++ b/database/pebble/batch.go
@@ -98,7 +98,7 @@ func (b *batch) Replay(w database.KeyValueWriterDeleter) error {
 				return err
 			}
 		default:
-			return fmt.Errorf("%w: %v", ErrInvalidOperation, kind)
+			return fmt.Errorf("%w: %v", errInvalidOperation, kind)
 		}
 	}
 }
diff --git a/database/pebble/db.go b/database/pebble/db.go
index 4b90b118a0b8..631ccc2db8b4 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -30,7 +30,7 @@ const pebbleByteOverHead = 8
 var (
 	_ database.Database = (*Database)(nil)
 
-	ErrInvalidOperation = errors.New("invalid operation")
+	errInvalidOperation = errors.New("invalid operation")
 
 	defaultCacheSize = 512 * units.MiB
 	DefaultConfig    = Config{

From f9192ed14d1c649e85e7bacbf9405f1da2abb7a5 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 2 Oct 2023 11:03:02 -0400
Subject: [PATCH 15/40] atomic bool --> bool

---
 database/pebble/batch.go      | 13 ++++++++-----
 database/pebble/batch_test.go |  6 +++---
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/database/pebble/batch.go b/database/pebble/batch.go
index e2026d6d7ec5..b6c9d283b64d 100644
--- a/database/pebble/batch.go
+++ b/database/pebble/batch.go
@@ -5,7 +5,6 @@ package pebble
 
 import (
 	"fmt"
-	"sync/atomic"
 
 	"github.com/cockroachdb/pebble"
 
@@ -22,7 +21,7 @@ type batch struct {
 
 	// True iff [batch] has been written to the database
 	// since the last time [Reset] was called.
-	written atomic.Bool
+	written bool
 }
 
 func (db *Database) NewBatch() database.Batch {
@@ -57,9 +56,13 @@ func (b *batch) Write() error {
 		return database.ErrClosed
 	}
 
-	if b.written.CompareAndSwap(false, true) {
+	if !b.written {
 		// This batch has not been written to the database yet.
-		return updateError(b.batch.Commit(pebble.Sync))
+		if err := updateError(b.batch.Commit(pebble.Sync)); err != nil {
+			return err
+		}
+		b.written = true
+		return nil
 	}
 
 	// pebble doesn't support writing a batch twice so we have to clone
@@ -77,7 +80,7 @@ func (b *batch) Write() error {
 
 func (b *batch) Reset() {
 	b.batch.Reset()
-	b.written.Store(false)
+	b.written = false
 	b.size = 0
 }
 
diff --git a/database/pebble/batch_test.go b/database/pebble/batch_test.go
index 4ec7b8e96e58..6632bf4907ee 100644
--- a/database/pebble/batch_test.go
+++ b/database/pebble/batch_test.go
@@ -27,7 +27,7 @@ func TestBatch(t *testing.T) {
 	batch, ok := batchIntf.(*batch)
 	require.True(ok)
 
-	require.False(batch.written.Load())
+	require.False(batch.written)
 
 	key1, value1 := []byte("key1"), []byte("value1")
 	require.NoError(batch.Put(key1, value1))
@@ -35,13 +35,13 @@ func TestBatch(t *testing.T) {
 
 	require.NoError(batch.Write())
 
-	require.True(batch.written.Load())
+	require.True(batch.written)
 
 	got, err := db.Get(key1)
 	require.NoError(err)
 	require.Equal(value1, got)
 
 	batch.Reset()
-	require.False(batch.written.Load())
+	require.False(batch.written)
 	require.Zero(batch.Size())
 }

From 72a249d71a9d5f379bc8f0dba6102072a8f8c989 Mon Sep 17 00:00:00 2001
From: Stephen Buttolph <stephen@avalabs.org>
Date: Tue, 24 Oct 2023 14:55:00 -0400
Subject: [PATCH 16/40] match pebble version

---
 go.mod |  4 ++--
 go.sum | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/go.mod b/go.mod
index b799d1a9579d..34c4b0cb7475 100644
--- a/go.mod
+++ b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/ava-labs/coreth v0.12.6-rc.2
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20230105152938-00a24d05a8c7
 	github.com/btcsuite/btcd/btcutil v1.1.3
-	github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06
+	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0
 	github.com/ethereum/go-ethereum v1.12.0
 	github.com/golang-jwt/jwt/v4 v4.3.0
@@ -58,7 +58,7 @@ require (
 	go.uber.org/mock v0.2.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.14.0
-	golang.org/x/exp v0.0.0-20230206171751-46f607a40771
+	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
 	golang.org/x/net v0.17.0
 	golang.org/x/sync v0.3.0
 	golang.org/x/term v0.13.0
diff --git a/go.sum b/go.sum
index f9fa8ab84ed1..f4d30e37b89f 100644
--- a/go.sum
+++ b/go.sum
@@ -121,15 +121,15 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
 github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
-github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o=
 github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8=
 github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk=
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w=
-github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s=
+github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk=
+github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM=
 github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
@@ -700,8 +700,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

From 4b08b25d57b1e747dbfa40b2ba2d3372c2e8efc5 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 09:35:33 -0400
Subject: [PATCH 17/40] fix test:

---
 database/pebble/batch_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/database/pebble/batch_test.go b/database/pebble/batch_test.go
index 6632bf4907ee..472f3de376cc 100644
--- a/database/pebble/batch_test.go
+++ b/database/pebble/batch_test.go
@@ -17,7 +17,8 @@ import (
 // Note: TestInterface tests other batch functionality.
 func TestBatch(t *testing.T) {
 	require := require.New(t)
-	dirName := os.TempDir()
+	dirName, err := os.MkdirTemp("", "pebbleTestBatch*")
+	require.NoError(err)
 	defer os.Remove(dirName)
 
 	db, err := New(dirName, DefaultConfig, logging.NoLog{}, "", prometheus.NewRegistry())

From b8fa698b1a59f16d5bd58bc8b4d82d3ed9fcd339 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 09:37:12 -0400
Subject: [PATCH 18/40] change pebble opening log; edit comment

---
 database/pebble/db.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 631ccc2db8b4..98b860926996 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -72,10 +72,12 @@ func New(file string, cfg Config, log logging.Logger, _ string, _ prometheus.Reg
 		MaxOpenFiles:                cfg.MaxOpenFiles,
 		MaxConcurrentCompactions:    func() int { return cfg.MaxConcurrentCompactions },
 	}
-	opts.Experimental.ReadSamplingMultiplier = -1 // explicitly disable seek compaction
+	opts.Experimental.ReadSamplingMultiplier = -1 // Disable seek compaction
 
-	log.Info("opening pebble")
-	log.Debug("config", zap.Any("config", opts))
+	log.Info(
+		"opening pebble",
+		zap.Reflect("config", cfg),
+	)
 
 	db, err := pebble.Open(file, opts)
 	if err != nil {

From a1b7fba0b09a8bc49ff8aafeb3f71eae84835cc0 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 09:40:48 -0400
Subject: [PATCH 19/40] pebbledb.New nit

---
 database/pebble/db.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 98b860926996..1f230c072d07 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -80,14 +80,10 @@ func New(file string, cfg Config, log logging.Logger, _ string, _ prometheus.Reg
 	)
 
 	db, err := pebble.Open(file, opts)
-	if err != nil {
-		return nil, err
-	}
-
 	return &Database{
 		pebbleDB:      db,
 		openIterators: set.Set[*iter]{},
-	}, nil
+	}, err
 }
 
 func (db *Database) Close() error {

From 6b93c90f2af7da189a446866171d11d180c8f2f9 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 09:46:20 -0400
Subject: [PATCH 20/40] Lock --> RLock in Put, Delete

---
 database/pebble/db.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 1f230c072d07..d77ccef806e3 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -150,8 +150,8 @@ func (db *Database) Get(key []byte) ([]byte, error) {
 }
 
 func (db *Database) Put(key []byte, value []byte) error {
-	db.lock.Lock()
-	defer db.lock.Unlock()
+	db.lock.RLock()
+	defer db.lock.RUnlock()
 
 	if db.closed {
 		return database.ErrClosed
@@ -161,8 +161,8 @@ func (db *Database) Put(key []byte, value []byte) error {
 }
 
 func (db *Database) Delete(key []byte) error {
-	db.lock.Lock()
-	defer db.lock.Unlock()
+	db.lock.RLock()
+	defer db.lock.RUnlock()
 
 	if db.closed {
 		return database.ErrClosed
@@ -263,8 +263,8 @@ func (db *Database) NewIteratorWithPrefix(prefix []byte) database.Iterator {
 }
 
 func (db *Database) NewIteratorWithStartAndPrefix(start, prefix []byte) database.Iterator {
-	db.lock.RLock()
-	defer db.lock.RUnlock()
+	db.lock.Lock()
+	defer db.lock.Unlock()
 
 	if db.closed {
 		return &iter{

From 610e7f8de11d3e097e0440567d30bfdb6dfdac24 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 09:51:35 -0400
Subject: [PATCH 21/40] iterator nits

---
 database/pebble/iterator.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/database/pebble/iterator.go b/database/pebble/iterator.go
index 136d4f97b81b..6dda946e81e7 100644
--- a/database/pebble/iterator.go
+++ b/database/pebble/iterator.go
@@ -69,7 +69,6 @@ func (it *iter) Next() bool {
 		return false
 	}
 
-	// Set the next key.
 	it.nextKey = it.iter.Key()
 
 	// Set the next value.
@@ -104,12 +103,9 @@ func (it *iter) Error() error {
 	it.lock.Lock()
 	defer it.lock.Unlock()
 
-	if it.err != nil {
+	if it.err != nil || it.closed {
 		return it.err
 	}
-	if it.closed {
-		return nil
-	}
 	return updateError(it.iter.Error())
 }
 

From fc91cd1f9895bcf3b6ad8e2f61a22d9d702082e7 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 09:52:45 -0400
Subject: [PATCH 22/40] Compact Lock --> RLock

---
 database/pebble/db.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index d77ccef806e3..208efde9ce3c 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -172,8 +172,8 @@ func (db *Database) Delete(key []byte) error {
 }
 
 func (db *Database) Compact(start []byte, limit []byte) error {
-	db.lock.Lock()
-	defer db.lock.Unlock()
+	db.lock.RLock()
+	defer db.lock.RUnlock()
 
 	switch {
 	case db.closed:

From 5aff591f7135735d76bed905369db2501aef2fcf Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 10:17:20 -0400
Subject: [PATCH 23/40] prefixBounds --> keyRange and prefixToUpperBound

---
 database/pebble/db.go      | 48 +++++++++++++++++++-------------------
 database/pebble/db_test.go | 44 +++++++++++++++++++++++++++++++---
 2 files changed, 65 insertions(+), 27 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 208efde9ce3c..2ec75b7ff9e9 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -256,7 +256,7 @@ func (db *Database) NewIteratorWithPrefix(prefix []byte) database.Iterator {
 
 	iter := &iter{
 		db:   db,
-		iter: db.pebbleDB.NewIter(prefixBounds(prefix)),
+		iter: db.pebbleDB.NewIter(keyRange(nil, prefix)),
 	}
 	db.openIterators.Add(iter)
 	return iter
@@ -274,21 +274,15 @@ func (db *Database) NewIteratorWithStartAndPrefix(start, prefix []byte) database
 		}
 	}
 
-	iterRange := prefixBounds(prefix)
-	if bytes.Compare(start, prefix) == 1 {
-		iterRange.LowerBound = start
-	}
-
 	iter := &iter{
 		db:   db,
-		iter: db.pebbleDB.NewIter(iterRange),
+		iter: db.pebbleDB.NewIter(keyRange(start, prefix)),
 	}
 	db.openIterators.Add(iter)
 	return iter
 }
 
-// Converts a pebble-specific error to its
-// Avalanche equivalent, if applicable.
+// Converts a pebble-specific error to its Avalanche equivalent, if applicable.
 func updateError(err error) error {
 	switch err {
 	case pebble.ErrClosed:
@@ -300,22 +294,28 @@ func updateError(err error) error {
 	}
 }
 
-// Returns a key range that covers all keys with the given [prefix].
-// Assumes the Database uses bytes.Compare for key comparison and
-// not a custom comparer.
-func prefixBounds(prefix []byte) *pebble.IterOptions {
-	var upperBound []byte
+func keyRange(start, prefix []byte) *pebble.IterOptions {
+	opt := &pebble.IterOptions{
+		LowerBound: prefix,
+		UpperBound: prefixToUpperBound(prefix),
+	}
+	if bytes.Compare(start, prefix) == 1 {
+		opt.LowerBound = start
+	}
+	return opt
+}
+
+// Returns an upper bound that stops after all keys with the given [prefix].
+// Assumes the Database uses bytes.Compare for key comparison and not a custom
+// comparer.
+func prefixToUpperBound(prefix []byte) []byte {
 	for i := len(prefix) - 1; i >= 0; i-- {
-		if prefix[i] == 0xFF {
-			continue
+		if prefix[i] != 0xFF {
+			upperBound := make([]byte, i+1)
+			copy(upperBound, prefix)
+			upperBound[i]++
+			return upperBound
 		}
-		upperBound = make([]byte, i+1)
-		copy(upperBound, prefix)
-		upperBound[i] = prefix[i] + 1
-		break
-	}
-	return &pebble.IterOptions{
-		LowerBound: prefix,     // Note this bound is inclusive.
-		UpperBound: upperBound, // Note this bound is exclusive.
 	}
+	return nil
 }
diff --git a/database/pebble/db_test.go b/database/pebble/db_test.go
index 81bd0e18d48e..7c74a0707178 100644
--- a/database/pebble/db_test.go
+++ b/database/pebble/db_test.go
@@ -53,10 +53,11 @@ func BenchmarkInterface(b *testing.B) {
 	}
 }
 
-func TestPrefixBounds(t *testing.T) {
+func TestKeyRange(t *testing.T) {
 	require := require.New(t)
 
 	type test struct {
+		start         []byte
 		prefix        []byte
 		expectedLower []byte
 		expectedUpper []byte
@@ -64,36 +65,73 @@ func TestPrefixBounds(t *testing.T) {
 
 	tests := []test{
 		{
+			start:         nil,
 			prefix:        nil,
 			expectedLower: nil,
 			expectedUpper: nil,
 		},
 		{
+			start:         nil,
 			prefix:        []byte{},
 			expectedLower: []byte{},
 			expectedUpper: nil,
 		},
 		{
+			start:         nil,
 			prefix:        []byte{0x00},
 			expectedLower: []byte{0x00},
 			expectedUpper: []byte{0x01},
 		},
 		{
+			start:         []byte{0x00, 0x02},
+			prefix:        []byte{0x00},
+			expectedLower: []byte{0x00, 0x02},
+			expectedUpper: []byte{0x01},
+		},
+		{
+			start:         []byte{0x01},
+			prefix:        []byte{0x00},
+			expectedLower: []byte{0x01},
+			expectedUpper: []byte{0x01},
+		},
+		{
+			start:         nil,
 			prefix:        []byte{0x01},
 			expectedLower: []byte{0x01},
 			expectedUpper: []byte{0x02},
 		},
 		{
+			start:         nil,
 			prefix:        []byte{0xFF},
 			expectedLower: []byte{0xFF},
 			expectedUpper: nil,
 		},
 		{
+			start:         []byte{0x00},
+			prefix:        []byte{0xFF},
+			expectedLower: []byte{0xFF},
+			expectedUpper: nil,
+		},
+		{
+			start:         nil,
 			prefix:        []byte{0x01, 0x02},
 			expectedLower: []byte{0x01, 0x02},
 			expectedUpper: []byte{0x01, 0x03},
 		},
 		{
+			start:         []byte{0x01, 0x02},
+			prefix:        []byte{0x01, 0x02},
+			expectedLower: []byte{0x01, 0x02},
+			expectedUpper: []byte{0x01, 0x03},
+		},
+		{
+			start:         []byte{0x01, 0x02, 0x05},
+			prefix:        []byte{0x01, 0x02},
+			expectedLower: []byte{0x01, 0x02, 0x05},
+			expectedUpper: []byte{0x01, 0x03},
+		},
+		{
+			start:         nil,
 			prefix:        []byte{0x01, 0x02, 0xFF},
 			expectedLower: []byte{0x01, 0x02, 0xFF},
 			expectedUpper: []byte{0x01, 0x03},
@@ -101,8 +139,8 @@ func TestPrefixBounds(t *testing.T) {
 	}
 
 	for _, tt := range tests {
-		t.Run(string(tt.prefix), func(t *testing.T) {
-			bounds := prefixBounds(tt.prefix)
+		t.Run(string(tt.start)+" "+string(tt.prefix), func(t *testing.T) {
+			bounds := keyRange(tt.start, tt.prefix)
 			require.Equal(tt.expectedLower, bounds.LowerBound)
 			require.Equal(tt.expectedUpper, bounds.UpperBound)
 		})

From bba30485bcae44f2b56fa9724f1d9890bd0b6d0f Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 10:21:56 -0400
Subject: [PATCH 24/40] remove unneeded iterator creator code

---
 database/pebble/db.go | 58 +++----------------------------------------
 1 file changed, 3 insertions(+), 55 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 2ec75b7ff9e9..d89bf8251a51 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -201,65 +201,13 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 }
 
 func (db *Database) NewIterator() database.Iterator {
-	db.lock.RLock()
-	defer db.lock.RUnlock()
-
-	if db.closed {
-		return &iter{
-			db:     db,
-			closed: true,
-			err:    database.ErrClosed,
-		}
-	}
-
-	iter := &iter{
-		db:   db,
-		iter: db.pebbleDB.NewIter(&pebble.IterOptions{}),
-	}
-	db.openIterators.Add(iter)
-	return iter
+	return db.NewIteratorWithStartAndPrefix(nil, nil)
 }
-
 func (db *Database) NewIteratorWithStart(start []byte) database.Iterator {
-	db.lock.RLock()
-	defer db.lock.RUnlock()
-
-	if db.closed {
-		return &iter{
-			db:     db,
-			closed: true,
-			err:    database.ErrClosed,
-		}
-	}
-
-	iter := &iter{
-		db: db,
-		iter: db.pebbleDB.NewIter(&pebble.IterOptions{
-			LowerBound: start,
-		}),
-	}
-	db.openIterators.Add(iter)
-	return iter
+	return db.NewIteratorWithStartAndPrefix(start, nil)
 }
-
 func (db *Database) NewIteratorWithPrefix(prefix []byte) database.Iterator {
-	db.lock.RLock()
-	defer db.lock.RUnlock()
-
-	if db.closed {
-		return &iter{
-			db:     db,
-			closed: true,
-			err:    database.ErrClosed,
-		}
-	}
-
-	iter := &iter{
-		db:   db,
-		iter: db.pebbleDB.NewIter(keyRange(nil, prefix)),
-	}
-	db.openIterators.Add(iter)
-	return iter
+	return db.NewIteratorWithStartAndPrefix(nil, prefix)
 }
 
 func (db *Database) NewIteratorWithStartAndPrefix(start, prefix []byte) database.Iterator {

From 3ffc6a88875a23bea46c4dcbd6eb69c6d325a95a Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 10:24:55 -0400
Subject: [PATCH 25/40] remove unneeded check in iter.Next

---
 database/pebble/iterator.go | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/database/pebble/iterator.go b/database/pebble/iterator.go
index 6dda946e81e7..999642225961 100644
--- a/database/pebble/iterator.go
+++ b/database/pebble/iterator.go
@@ -71,23 +71,6 @@ func (it *iter) Next() bool {
 
 	it.nextKey = it.iter.Key()
 
-	// Set the next value.
-	// TODO is the below necessary?
-	// Need to make sure the following invariant for [i.iter.ValueAndErr] holds:
-	// "REQUIRES: iter.Error() == nil and HasPointAndRange() returns true for hasPoint."
-	hasPoint, _ := it.iter.HasPointAndRange()
-	if !hasPoint {
-		it.hasNext = false
-		it.err = fmt.Errorf("%w: %w", errCouldntGetValue, errNoPointKey)
-		return false
-	}
-
-	if err := it.iter.Error(); err != nil {
-		it.hasNext = false
-		it.err = fmt.Errorf("%w: %w", errCouldntGetValue, err)
-		return false
-	}
-
 	var err error
 	it.nextVal, err = it.iter.ValueAndErr()
 	if err != nil {

From a8ff7b86bec4a8fc28f67078eacc01ba870a42e7 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 10:25:11 -0400
Subject: [PATCH 26/40] remove unused var

---
 database/pebble/iterator.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/database/pebble/iterator.go b/database/pebble/iterator.go
index 999642225961..b1c5215b1c46 100644
--- a/database/pebble/iterator.go
+++ b/database/pebble/iterator.go
@@ -19,7 +19,6 @@ var (
 	_ database.Iterator = (*iter)(nil)
 
 	errCouldntGetValue = errors.New("couldnt get iterator value")
-	errNoPointKey      = errors.New("iterator has no point key")
 )
 
 type iter struct {

From c6bacc7f72b19595d33d87274a9836ce2fb1057c Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 10:25:57 -0400
Subject: [PATCH 27/40] change switch order in iter.Next to not overwrite iter
 error

---
 database/pebble/iterator.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/database/pebble/iterator.go b/database/pebble/iterator.go
index b1c5215b1c46..0aa402570ddf 100644
--- a/database/pebble/iterator.go
+++ b/database/pebble/iterator.go
@@ -50,12 +50,12 @@ func (it *iter) Next() bool {
 	defer it.lock.Unlock()
 
 	switch {
-	case it.closed || dbClosed:
+	case it.err != nil:
 		it.hasNext = false
-		it.err = database.ErrClosed
 		return false
-	case it.err != nil:
+	case it.closed || dbClosed:
 		it.hasNext = false
+		it.err = database.ErrClosed
 		return false
 	case !it.initialized:
 		it.hasNext = it.iter.First()

From b88679090c21a27dc8293fe5a47719923c0777db Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 10:30:08 -0400
Subject: [PATCH 28/40] remove unneeded lock grab and db closed check in
 iter.Next

---
 database/pebble/iterator.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/database/pebble/iterator.go b/database/pebble/iterator.go
index 0aa402570ddf..115c122e30f4 100644
--- a/database/pebble/iterator.go
+++ b/database/pebble/iterator.go
@@ -42,10 +42,6 @@ type iter struct {
 
 // Must not be called with [db.lock] held.
 func (it *iter) Next() bool {
-	it.db.lock.RLock()
-	dbClosed := it.db.closed
-	it.db.lock.RUnlock()
-
 	it.lock.Lock()
 	defer it.lock.Unlock()
 
@@ -53,7 +49,7 @@ func (it *iter) Next() bool {
 	case it.err != nil:
 		it.hasNext = false
 		return false
-	case it.closed || dbClosed:
+	case it.closed:
 		it.hasNext = false
 		it.err = database.ErrClosed
 		return false

From f7716659d1d5232582e02e3e6c31e644fff5b345 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 10:40:08 -0400
Subject: [PATCH 29/40] prevent Compact panic

---
 database/pebble/db.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index d89bf8251a51..d10ff1ae8993 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -192,6 +192,10 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 	it := db.pebbleDB.NewIter(&pebble.IterOptions{})
 	if it.Last() {
 		if lastkey := it.Key(); lastkey != nil {
+			if bytes.Compare(start, lastkey) >= 0 {
+				// pebbleDB.Compact requires start < limit or it panics.
+				return nil
+			}
 			return updateError(db.pebbleDB.Compact(start, lastkey, true /* parallelize */))
 		}
 	}

From 34122164bdaa2fa11dc264182aadac44ee2f6213 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 11:26:15 -0400
Subject: [PATCH 30/40] fix default MaxConcurrentCompactions; fix compaction
 iterator leak; appease linter; add to tests

---
 database/pebble/db.go     | 22 ++++++++++++++--------
 database/test_database.go |  5 +++++
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index d10ff1ae8993..3bc3255b7484 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -40,6 +40,7 @@ var (
 		MemTableStopWritesThreshold: 8,
 		MemTableSize:                defaultCacheSize / 4,
 		MaxOpenFiles:                4096,
+		MaxConcurrentCompactions:    1,
 	}
 )
 
@@ -175,14 +176,15 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
 
-	switch {
-	case db.closed:
+	if db.closed {
 		return database.ErrClosed
-	case pebble.DefaultComparer.Compare(start, limit) >= 0:
-		// pebble's Compact will no-op & error if start >= limit
-		// according to pebble's comparer.
-		return nil
-	case limit != nil:
+	}
+
+	if limit != nil {
+		if pebble.DefaultComparer.Compare(start, limit) >= 0 {
+			// pebbleDB.Compact requires start < limit or it panics.
+			return nil
+		}
 		return updateError(db.pebbleDB.Compact(start, limit, true /* parallelize */))
 	}
 
@@ -190,9 +192,11 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 	// but pebble treats a nil [limit] as a key before all keys in Compact.
 	// Use the greatest key in the database as the [limit] to get the desired behavior.
 	it := db.pebbleDB.NewIter(&pebble.IterOptions{})
+	defer it.Close()
+
 	if it.Last() {
 		if lastkey := it.Key(); lastkey != nil {
-			if bytes.Compare(start, lastkey) >= 0 {
+			if pebble.DefaultComparer.Compare(start, lastkey) >= 0 {
 				// pebbleDB.Compact requires start < limit or it panics.
 				return nil
 			}
@@ -207,9 +211,11 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 func (db *Database) NewIterator() database.Iterator {
 	return db.NewIteratorWithStartAndPrefix(nil, nil)
 }
+
 func (db *Database) NewIteratorWithStart(start []byte) database.Iterator {
 	return db.NewIteratorWithStartAndPrefix(start, nil)
 }
+
 func (db *Database) NewIteratorWithPrefix(prefix []byte) database.Iterator {
 	return db.NewIteratorWithStartAndPrefix(nil, prefix)
 }
diff --git a/database/test_database.go b/database/test_database.go
index 69fb1d2b7948..9419d1457087 100644
--- a/database/test_database.go
+++ b/database/test_database.go
@@ -934,6 +934,11 @@ func TestCompactNoPanic(t *testing.T, db Database) {
 	require.NoError(db.Put(key3, value3))
 
 	require.NoError(db.Compact(nil, nil))
+
+	require.NoError(db.Compact([]byte{2}, []byte{1}))
+
+	require.NoError(db.Compact([]byte{255}, nil))
+
 	require.NoError(db.Close())
 	err := db.Compact(nil, nil)
 	require.ErrorIs(err, ErrClosed)

From f0e3f2e3628647fe8136e7c37659a77c4dc19fdd Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 11:35:29 -0400
Subject: [PATCH 31/40] cleanup and comments

---
 database/pebble/db.go     | 41 ++++++++++++++++++---------------------
 database/test_database.go |  3 +++
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/database/pebble/db.go b/database/pebble/db.go
index 3bc3255b7484..b297ddf8743d 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -172,7 +172,7 @@ func (db *Database) Delete(key []byte) error {
 	return updateError(db.pebbleDB.Delete(key, pebble.Sync))
 }
 
-func (db *Database) Compact(start []byte, limit []byte) error {
+func (db *Database) Compact(start []byte, end []byte) error {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
 
@@ -180,32 +180,29 @@ func (db *Database) Compact(start []byte, limit []byte) error {
 		return database.ErrClosed
 	}
 
-	if limit != nil {
-		if pebble.DefaultComparer.Compare(start, limit) >= 0 {
-			// pebbleDB.Compact requires start < limit or it panics.
-			return nil
+	if end == nil {
+		// The database.Database spec treats a nil [limit] as a key after all keys
+		// but pebble treats a nil [limit] as a key before all keys in Compact.
+		// Use the greatest key in the database as the [limit] to get the desired behavior.
+		it := db.pebbleDB.NewIter(&pebble.IterOptions{})
+
+		if !it.Last() {
+			// The database is empty.
+			return it.Close()
 		}
-		return updateError(db.pebbleDB.Compact(start, limit, true /* parallelize */))
-	}
 
-	// The database.Database spec treats a nil [limit] as a key after all keys
-	// but pebble treats a nil [limit] as a key before all keys in Compact.
-	// Use the greatest key in the database as the [limit] to get the desired behavior.
-	it := db.pebbleDB.NewIter(&pebble.IterOptions{})
-	defer it.Close()
-
-	if it.Last() {
-		if lastkey := it.Key(); lastkey != nil {
-			if pebble.DefaultComparer.Compare(start, lastkey) >= 0 {
-				// pebbleDB.Compact requires start < limit or it panics.
-				return nil
-			}
-			return updateError(db.pebbleDB.Compact(start, lastkey, true /* parallelize */))
+		end = it.Key()
+		if err := it.Close(); err != nil {
+			return err
 		}
 	}
 
-	// Either this database is empty or the only key in it is nil.
-	return nil
+	if pebble.DefaultComparer.Compare(start, end) >= 1 {
+		// pebble requires [start] < [end]
+		return nil
+	}
+
+	return updateError(db.pebbleDB.Compact(start, end, true /* parallelize */))
 }
 
 func (db *Database) NewIterator() database.Iterator {
diff --git a/database/test_database.go b/database/test_database.go
index 9419d1457087..2e68f53341b8 100644
--- a/database/test_database.go
+++ b/database/test_database.go
@@ -933,10 +933,13 @@ func TestCompactNoPanic(t *testing.T, db Database) {
 	require.NoError(db.Put(key2, value2))
 	require.NoError(db.Put(key3, value3))
 
+	// Test compacting with nil bounds
 	require.NoError(db.Compact(nil, nil))
 
+	// Test compacting when start > end
 	require.NoError(db.Compact([]byte{2}, []byte{1}))
 
+	// Test compacting when start > largest key
 	require.NoError(db.Compact([]byte{255}, nil))
 
 	require.NoError(db.Close())

From c4d1a871c0489b4cf4fe756fe70f2535b794527b Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 11:45:22 -0400
Subject: [PATCH 32/40] add Key and Value invariant on database.Iterator

---
 database/iterator.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/database/iterator.go b/database/iterator.go
index 3cfd075cc9d3..20db39d0cf9b 100644
--- a/database/iterator.go
+++ b/database/iterator.go
@@ -34,10 +34,12 @@ type Iterator interface {
 
 	// Key returns the key of the current key/value pair, or nil if done.
 	// If the database is closed, must still report the current contents.
+	// Behavior is underfined after Release is called.
 	Key() []byte
 
 	// Value returns the value of the current key/value pair, or nil if done.
 	// If the database is closed, must still report the current contents.
+	// Behavior is underfined after Release is called.
 	Value() []byte
 
 	// Release releases associated resources. Release should always succeed and

From d379e775e439173ab4a053aa59f1d61583e70a8b Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 11:46:04 -0400
Subject: [PATCH 33/40] spelling is hard

---
 database/iterator.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/database/iterator.go b/database/iterator.go
index 20db39d0cf9b..c83ceac49639 100644
--- a/database/iterator.go
+++ b/database/iterator.go
@@ -34,12 +34,12 @@ type Iterator interface {
 
 	// Key returns the key of the current key/value pair, or nil if done.
 	// If the database is closed, must still report the current contents.
-	// Behavior is underfined after Release is called.
+	// Behavior is undefined after Release is called.
 	Key() []byte
 
 	// Value returns the value of the current key/value pair, or nil if done.
 	// If the database is closed, must still report the current contents.
-	// Behavior is underfined after Release is called.
+	// Behavior is undefined after Release is called.
 	Value() []byte
 
 	// Release releases associated resources. Release should always succeed and

From 8eea9ebd60a8b02e65cbaa411d96ca9b657e7cc3 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 12:14:41 -0400
Subject: [PATCH 34/40] make pebble.New take in config bytes instead of config

---
 database/pebble/batch_test.go |  2 +-
 database/pebble/db.go         | 32 ++++++++++++++++++++++++--------
 database/pebble/db_test.go    |  3 +--
 3 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/database/pebble/batch_test.go b/database/pebble/batch_test.go
index 472f3de376cc..eeabd5788e8c 100644
--- a/database/pebble/batch_test.go
+++ b/database/pebble/batch_test.go
@@ -21,7 +21,7 @@ func TestBatch(t *testing.T) {
 	require.NoError(err)
 	defer os.Remove(dirName)
 
-	db, err := New(dirName, DefaultConfig, logging.NoLog{}, "", prometheus.NewRegistry())
+	db, err := New(dirName, DefaultConfigBytes, logging.NoLog{}, "", prometheus.NewRegistry())
 	require.NoError(err)
 
 	batchIntf := db.NewBatch()
diff --git a/database/pebble/db.go b/database/pebble/db.go
index b297ddf8743d..13ab1db04977 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -6,6 +6,7 @@ package pebble
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"sync"
 
@@ -42,8 +43,18 @@ var (
 		MaxOpenFiles:                4096,
 		MaxConcurrentCompactions:    1,
 	}
+
+	DefaultConfigBytes []byte
 )
 
+func init() {
+	var err error
+	DefaultConfigBytes, err = json.Marshal(DefaultConfig)
+	if err != nil {
+		panic(err)
+	}
+}
+
 type Database struct {
 	lock          sync.RWMutex
 	pebbleDB      *pebble.DB
@@ -52,17 +63,22 @@ type Database struct {
 }
 
 type Config struct {
-	CacheSize                   int
-	BytesPerSync                int
-	WALBytesPerSync             int // 0 means no background syncing
-	MemTableStopWritesThreshold int
-	MemTableSize                int
-	MaxOpenFiles                int
-	MaxConcurrentCompactions    int
+	CacheSize                   int `json:"cacheSize"`
+	BytesPerSync                int `json:"bytesPerSync"`
+	WALBytesPerSync             int `json:"walBytesPerSync"` // 0 means no background syncing
+	MemTableStopWritesThreshold int `json:"memTableStopWritesThreshold"`
+	MemTableSize                int `json:"memTableSize"`
+	MaxOpenFiles                int `json:"maxOpenFiles"`
+	MaxConcurrentCompactions    int `json:"maxConcurrentCompactions"`
 }
 
 // TODO: Add metrics
-func New(file string, cfg Config, log logging.Logger, _ string, _ prometheus.Registerer) (*Database, error) {
+func New(file string, configBytes []byte, log logging.Logger, _ string, _ prometheus.Registerer) (*Database, error) {
+	var cfg Config
+	if err := json.Unmarshal(configBytes, &cfg); err != nil {
+		return nil, err
+	}
+
 	opts := &pebble.Options{
 		Cache:                       pebble.NewCache(int64(cfg.CacheSize)),
 		BytesPerSync:                cfg.BytesPerSync,
diff --git a/database/pebble/db_test.go b/database/pebble/db_test.go
index 7c74a0707178..c72a9d687c88 100644
--- a/database/pebble/db_test.go
+++ b/database/pebble/db_test.go
@@ -16,8 +16,7 @@ import (
 
 func newDB(t testing.TB) *Database {
 	folder := t.TempDir()
-	cfg := DefaultConfig
-	db, err := New(folder, cfg, logging.NoLog{}, "pebble", prometheus.NewRegistry())
+	db, err := New(folder, DefaultConfigBytes, logging.NoLog{}, "pebble", prometheus.NewRegistry())
 	require.NoError(t, err)
 	return db
 }

From b64de08cb7568102c77237d24057c1539829f0f9 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 15:16:35 -0400
Subject: [PATCH 35/40] os.MkDirTemp --> t.TempDir

---
 database/pebble/batch_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/database/pebble/batch_test.go b/database/pebble/batch_test.go
index eeabd5788e8c..77d8fedccf42 100644
--- a/database/pebble/batch_test.go
+++ b/database/pebble/batch_test.go
@@ -17,8 +17,7 @@ import (
 // Note: TestInterface tests other batch functionality.
 func TestBatch(t *testing.T) {
 	require := require.New(t)
-	dirName, err := os.MkdirTemp("", "pebbleTestBatch*")
-	require.NoError(err)
+	dirName := t.TempDir()
 	defer os.Remove(dirName)
 
 	db, err := New(dirName, DefaultConfigBytes, logging.NoLog{}, "", prometheus.NewRegistry())

From b1768c15e9fd74813d105914b24896783626c589 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 15:18:06 -0400
Subject: [PATCH 36/40] remove unneeded file removal

---
 database/pebble/batch_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/database/pebble/batch_test.go b/database/pebble/batch_test.go
index 77d8fedccf42..af9e68b26f33 100644
--- a/database/pebble/batch_test.go
+++ b/database/pebble/batch_test.go
@@ -4,7 +4,6 @@
 package pebble
 
 import (
-	"os"
 	"testing"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -18,7 +17,6 @@ import (
 func TestBatch(t *testing.T) {
 	require := require.New(t)
 	dirName := t.TempDir()
-	defer os.Remove(dirName)
 
 	db, err := New(dirName, DefaultConfigBytes, logging.NoLog{}, "", prometheus.NewRegistry())
 	require.NoError(err)

From afd88e365e1a8f3e47eaf498cf492c11c295163c Mon Sep 17 00:00:00 2001
From: Stephen Buttolph <stephen@avalabs.org>
Date: Wed, 25 Oct 2023 16:16:17 -0400
Subject: [PATCH 37/40] Close db

---
 database/pebble/batch_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/database/pebble/batch_test.go b/database/pebble/batch_test.go
index af9e68b26f33..a84134708956 100644
--- a/database/pebble/batch_test.go
+++ b/database/pebble/batch_test.go
@@ -42,4 +42,6 @@ func TestBatch(t *testing.T) {
 	batch.Reset()
 	require.False(batch.written)
 	require.Zero(batch.Size())
+
+	require.NoError(db.Close())
 }

From b360819fef009b44190d9f787adb7ca0a07c60c6 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Wed, 25 Oct 2023 17:07:33 -0400
Subject: [PATCH 38/40] add flags for pebble; only unmarshal config bytes if
 given

---
 config/flags.go             |  3 ++-
 database/manager/manager.go | 24 ++++++++++++++++++++++++
 database/pebble/batch.go    |  2 ++
 database/pebble/db.go       | 10 ++++++----
 database/pebble/db_test.go  |  2 +-
 node/node.go                | 22 ++++++++++++++++++++--
 6 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/config/flags.go b/config/flags.go
index 254d70fea2dd..7f049ff3747d 100644
--- a/config/flags.go
+++ b/config/flags.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/ava-labs/avalanchego/database/leveldb"
 	"github.com/ava-labs/avalanchego/database/memdb"
+	"github.com/ava-labs/avalanchego/database/pebble"
 	"github.com/ava-labs/avalanchego/genesis"
 	"github.com/ava-labs/avalanchego/snow/consensus/snowball"
 	"github.com/ava-labs/avalanchego/trace"
@@ -103,7 +104,7 @@ func addNodeFlags(fs *pflag.FlagSet) {
 	fs.Uint64(AddSubnetDelegatorFeeKey, genesis.LocalParams.AddSubnetDelegatorFee, "Transaction fee, in nAVAX, for transactions that add new subnet delegators")
 
 	// Database
-	fs.String(DBTypeKey, leveldb.Name, fmt.Sprintf("Database type to use. Should be one of {%s, %s}", leveldb.Name, memdb.Name))
+	fs.String(DBTypeKey, leveldb.Name, fmt.Sprintf("Database type to use. Must be one of {%s, %s, %s}", leveldb.Name, memdb.Name, pebble.Name))
 	fs.String(DBPathKey, defaultDBDir, "Path to database directory")
 	fs.String(DBConfigFileKey, "", fmt.Sprintf("Path to database config file. Ignored if %s is specified", DBConfigContentKey))
 	fs.String(DBConfigContentKey, "", "Specifies base64 encoded database config content")
diff --git a/database/manager/manager.go b/database/manager/manager.go
index fd9c36969b79..676f1d741735 100644
--- a/database/manager/manager.go
+++ b/database/manager/manager.go
@@ -17,6 +17,7 @@ import (
 	"github.com/ava-labs/avalanchego/database/leveldb"
 	"github.com/ava-labs/avalanchego/database/memdb"
 	"github.com/ava-labs/avalanchego/database/meterdb"
+	"github.com/ava-labs/avalanchego/database/pebble"
 	"github.com/ava-labs/avalanchego/database/prefixdb"
 	"github.com/ava-labs/avalanchego/utils"
 	"github.com/ava-labs/avalanchego/utils/logging"
@@ -96,6 +97,29 @@ func NewLevelDB(
 	)
 }
 
+// NewLevelDB creates a database manager of levelDBs at [filePath] by creating a
+// database instance from each directory with a version <= [currentVersion]. If
+// [includePreviousVersions], opens previous database versions and includes them
+// in the returned Manager.
+func NewPebbleDB(
+	dbDirPath string,
+	dbConfig []byte,
+	log logging.Logger,
+	currentVersion *version.Semantic,
+	namespace string,
+	reg prometheus.Registerer,
+) (Manager, error) {
+	return new(
+		pebble.New,
+		dbDirPath,
+		dbConfig,
+		log,
+		currentVersion,
+		namespace,
+		reg,
+	)
+}
+
 // new creates a database manager at [filePath] by creating a database instance
 // from each directory with a version <= [currentVersion]. If
 // [includePreviousVersions], opens previous database versions and includes them
diff --git a/database/pebble/batch.go b/database/pebble/batch.go
index b6c9d283b64d..75048f5b4f2c 100644
--- a/database/pebble/batch.go
+++ b/database/pebble/batch.go
@@ -11,6 +11,8 @@ import (
 	"github.com/ava-labs/avalanchego/database"
 )
 
+const Name = "pebble"
+
 var _ database.Batch = (*batch)(nil)
 
 // Not safe for concurrent use.
diff --git a/database/pebble/db.go b/database/pebble/db.go
index 13ab1db04977..e3fe776770e6 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -73,10 +73,12 @@ type Config struct {
 }
 
 // TODO: Add metrics
-func New(file string, configBytes []byte, log logging.Logger, _ string, _ prometheus.Registerer) (*Database, error) {
-	var cfg Config
-	if err := json.Unmarshal(configBytes, &cfg); err != nil {
-		return nil, err
+func New(file string, configBytes []byte, log logging.Logger, _ string, _ prometheus.Registerer) (database.Database, error) {
+	cfg := DefaultConfig
+	if len(configBytes) > 0 {
+		if err := json.Unmarshal(configBytes, &cfg); err != nil {
+			return nil, err
+		}
 	}
 
 	opts := &pebble.Options{
diff --git a/database/pebble/db_test.go b/database/pebble/db_test.go
index c72a9d687c88..cba67a79a88f 100644
--- a/database/pebble/db_test.go
+++ b/database/pebble/db_test.go
@@ -18,7 +18,7 @@ func newDB(t testing.TB) *Database {
 	folder := t.TempDir()
 	db, err := New(folder, DefaultConfigBytes, logging.NoLog{}, "pebble", prometheus.NewRegistry())
 	require.NoError(t, err)
-	return db
+	return db.(*Database)
 }
 
 func TestInterface(t *testing.T) {
diff --git a/node/node.go b/node/node.go
index a2c41f83c623..4e0f63a507c9 100644
--- a/node/node.go
+++ b/node/node.go
@@ -39,6 +39,7 @@ import (
 	"github.com/ava-labs/avalanchego/database/leveldb"
 	"github.com/ava-labs/avalanchego/database/manager"
 	"github.com/ava-labs/avalanchego/database/memdb"
+	"github.com/ava-labs/avalanchego/database/pebble"
 	"github.com/ava-labs/avalanchego/database/prefixdb"
 	"github.com/ava-labs/avalanchego/genesis"
 	"github.com/ava-labs/avalanchego/ids"
@@ -505,15 +506,32 @@ func (n *Node) initDatabase() error {
 	)
 	switch n.Config.DatabaseConfig.Name {
 	case leveldb.Name:
-		dbManager, err = manager.NewLevelDB(n.Config.DatabaseConfig.Path, n.Config.DatabaseConfig.Config, n.Log, version.CurrentDatabase, "db_internal", n.MetricsRegisterer)
+		dbManager, err = manager.NewLevelDB(
+			n.Config.DatabaseConfig.Path,
+			n.Config.DatabaseConfig.Config,
+			n.Log,
+			version.CurrentDatabase,
+			"db_internal",
+			n.MetricsRegisterer,
+		)
+	case pebble.Name:
+		dbManager, err = manager.NewPebbleDB(
+			n.Config.DatabaseConfig.Path,
+			n.Config.DatabaseConfig.Config,
+			n.Log,
+			version.CurrentDatabase,
+			"db_internal",
+			n.MetricsRegisterer,
+		)
 	case memdb.Name:
 		dbManager = manager.NewMemDB(version.CurrentDatabase)
 	default:
 		err = fmt.Errorf(
-			"db-type was %q but should have been one of {%s, %s}",
+			"db-type was %q but should have been one of {%s, %s, %s}",
 			n.Config.DatabaseConfig.Name,
 			leveldb.Name,
 			memdb.Name,
+			pebble.Name,
 		)
 	}
 	if err != nil {

From 417785f88ddec767b16a02b13fb56ba0c661df91 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Thu, 26 Oct 2023 10:18:42 -0400
Subject: [PATCH 39/40] comments

---
 database/manager/manager.go | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/database/manager/manager.go b/database/manager/manager.go
index 676f1d741735..652385c42ce8 100644
--- a/database/manager/manager.go
+++ b/database/manager/manager.go
@@ -74,10 +74,8 @@ type manager struct {
 	databases []*VersionedDatabase
 }
 
-// NewLevelDB creates a database manager of levelDBs at [filePath] by creating a
-// database instance from each directory with a version <= [currentVersion]. If
-// [includePreviousVersions], opens previous database versions and includes them
-// in the returned Manager.
+// NewLevelDB creates a database manager of levelDBs at [dbDirPath] by creating a
+// database instance from each directory with a version <= [currentVersion].
 func NewLevelDB(
 	dbDirPath string,
 	dbConfig []byte,
@@ -97,10 +95,8 @@ func NewLevelDB(
 	)
 }
 
-// NewLevelDB creates a database manager of levelDBs at [filePath] by creating a
-// database instance from each directory with a version <= [currentVersion]. If
-// [includePreviousVersions], opens previous database versions and includes them
-// in the returned Manager.
+// NewPebbleDB creates a database manager of pebble instances at [dbDirPath] by creating a
+// database instance from each directory with a version <= [currentVersion].
 func NewPebbleDB(
 	dbDirPath string,
 	dbConfig []byte,

From b49340bcf7667a69bfc38aafdc6d0b37bc38cf31 Mon Sep 17 00:00:00 2001
From: Dan Laine <daniel.laine@avalabs.org>
Date: Mon, 30 Oct 2023 10:42:48 -0400
Subject: [PATCH 40/40] nit move name declaration

---
 database/pebble/batch.go |  2 --
 database/pebble/db.go    | 10 +++++++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/database/pebble/batch.go b/database/pebble/batch.go
index 75048f5b4f2c..b6c9d283b64d 100644
--- a/database/pebble/batch.go
+++ b/database/pebble/batch.go
@@ -11,8 +11,6 @@ import (
 	"github.com/ava-labs/avalanchego/database"
 )
 
-const Name = "pebble"
-
 var _ database.Batch = (*batch)(nil)
 
 // Not safe for concurrent use.
diff --git a/database/pebble/db.go b/database/pebble/db.go
index e3fe776770e6..7aa718082a35 100644
--- a/database/pebble/db.go
+++ b/database/pebble/db.go
@@ -24,9 +24,13 @@ import (
 	"github.com/ava-labs/avalanchego/utils/units"
 )
 
-// pebbleByteOverHead is the number of bytes of constant overhead that
-// should be added to a batch size per operation.
-const pebbleByteOverHead = 8
+const (
+	Name = "pebble"
+
+	// pebbleByteOverHead is the number of bytes of constant overhead that
+	// should be added to a batch size per operation.
+	pebbleByteOverHead = 8
+)
 
 var (
 	_ database.Database = (*Database)(nil)