Make rocksdb configurable (backport #1658) (#1698)

* Make rocksdb configurable (#1658)

* Make rocksdb configurable

* Make sure rocksdb tests are running in CI

* Updating ci-rocksdb-build workflow

* Remove test.sh

* Update tm-db dependency

(cherry picked from commit 90fbe1a)

# Conflicts:
#	.github/workflows/ci-rocksdb-build.yml
#	Dockerfile-rocksdb
#	cmd/kava/cmd/root.go
#	go.mod
#	go.sum

* Resolve conflicts after mergify

* Rocksdb Metrics (#1692)

* Rocksdb Metrics

* Add rocksdb namespace for options

* Adding help to the metrics

* CR's fixes

* CR's fixes

* CR's fixes

* Increase number of options to configure rocksdb (#1696)

* Change ethermint dependency to release tag

---------

Co-authored-by: Evgeniy Scherbina <[email protected]>

.github/workflows/ci-rocksdb-build.yml

@@ -0,0 +1,47 @@
+name: Continuous Integration (Rocksdb Build)
+
+env:
+  ROCKSDB_VERSION: v8.1.1
+
+on:
+  workflow_call:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repo from current commit
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.20"
+          check-latest: true
+          cache: true
+      - name: build rocksdb dependency
+        run: bash ${GITHUB_WORKSPACE}/.github/scripts/install-rocksdb.sh
+      - name: build application
+        run: make build COSMOS_BUILD_OPTIONS=rocksdb
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: install RocksDB dependencies
+        run: sudo apt-get update
+          && sudo apt-get install -y git make gcc libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+      - name: install RocksDB as shared library
+        run: git clone https://github.com/facebook/rocksdb.git
+          && cd rocksdb
+          && git checkout $ROCKSDB_VERSION
+          && sudo make -j$(nproc) install-shared
+          && sudo ldconfig
+      - name: checkout repo from current commit
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.20"
+          check-latest: true
+          cache: true
+      - name: run unit tests
+        run: make test-rocksdb

Dockerfile-rocksdb

@@ -0,0 +1,47 @@
+FROM golang:1.20-bullseye AS kava-builder
+
+# Set up dependencies
+RUN apt-get update \
+    && apt-get install -y git make gcc libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory for the build
+WORKDIR /root
+# default home directory is /root
+
+# install rocksdb
+ARG rocksdb_version=v8.1.1
+ENV ROCKSDB_VERSION=$rocksdb_version
+
+RUN git clone https://github.com/facebook/rocksdb.git \
+    && cd rocksdb \
+    && git checkout $ROCKSDB_VERSION \
+    && make -j$(nproc) install-shared \
+    && ldconfig
+
+# Add source files for kava
+COPY . kava
+
+ARG kava_database_backend=rocksdb
+ENV KAVA_DATABASE_BACKEND=$kava_database_backend
+
+# Mount go build and mod caches as container caches, persisted between builder invocations
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    cd kava && make install COSMOS_BUILD_OPTIONS=$KAVA_DATABASE_BACKEND
+
+
+FROM ubuntu:22.04
+
+RUN apt-get update \
+    && apt-get install -y libsnappy1v5 libgflags2.2 zlib1g libbz2-1.0 curl jq \
+    && rm -rf /var/lib/apt/lists/*
+
+# copy rocksdb shared objects
+COPY --from=kava-builder /usr/local/lib/ /usr/local/lib/
+RUN ldconfig
+
+# copy kava binary
+COPY --from=kava-builder /go/bin/kava /bin/kava
+
+CMD ["kava"]

Makefile

@@ -278,6 +278,9 @@ test-basic: test
 test:
 	@go test $$(go list ./... | grep -v 'contrib')
 
+test-rocksdb:
+	@go test -tags=rocksdb ./cmd/kava/opendb
+
 # Run cli integration tests
 # `-p 4` to use 4 cores, `-tags cli_test` to tell go not to ignore the cli package
 # These tests use the `kvd` or `kvcli` binaries in the build dir, or in `$BUILDDIR` if that env var is set.

cmd/kava/cmd/root.go

@@ -22,6 +22,7 @@ import (
 	"github.com/kava-labs/kava/app"
 	"github.com/kava-labs/kava/app/params"
 	kavaclient "github.com/kava-labs/kava/client"
+	"github.com/kava-labs/kava/cmd/kava/opendb"
 	"github.com/kava-labs/kava/migrate"
 )
 
@@ -100,8 +101,18 @@ func addSubCmds(rootCmd *cobra.Command, encodingConfig params.EncodingConfig, de
 		encodingConfig: encodingConfig,
 	}
 
+	opts := ethermintserver.StartOptions{
+		AppCreator:      ac.newApp,
+		DefaultNodeHome: app.DefaultNodeHome,
+		DBOpener:        opendb.OpenDB,
+	}
 	// ethermintserver adds additional flags to start the JSON-RPC server for evm support
-	ethermintserver.AddCommands(rootCmd, defaultNodeHome, ac.newApp, ac.appExport, ac.addStartCmdFlags)
+	ethermintserver.AddCommands(
+		rootCmd,
+		opts,
+		ac.appExport,
+		ac.addStartCmdFlags,
+	)
 
 	// add keybase, auxiliary RPC, query, and tx child commands
 	rootCmd.AddCommand(

cmd/kava/opendb/metrics.go

@@ -0,0 +1,166 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+// rocksdbMetrics will be initialized in registerMetrics() if enableRocksdbMetrics flag set to true
+var rocksdbMetrics *Metrics
+
+// Metrics contains all rocksdb metrics which will be reported to prometheus
+type Metrics struct {
+	// Keys
+	NumberKeysWritten metrics.Gauge
+	NumberKeysRead    metrics.Gauge
+	NumberKeysUpdated metrics.Gauge
+	EstimateNumKeys   metrics.Gauge
+
+	// Files
+	NumberFileOpens  metrics.Gauge
+	NumberFileErrors metrics.Gauge
+
+	// Memory
+	BlockCacheUsage         metrics.Gauge
+	EstimateTableReadersMem metrics.Gauge
+	CurSizeAllMemTables     metrics.Gauge
+	BlockCachePinnedUsage   metrics.Gauge
+
+	// Cache
+	BlockCacheMiss        metrics.Gauge
+	BlockCacheHit         metrics.Gauge
+	BlockCacheAdd         metrics.Gauge
+	BlockCacheAddFailures metrics.Gauge
+}
+
+// registerMetrics registers metrics in prometheus and initializes rocksdbMetrics variable
+func registerMetrics() {
+	if rocksdbMetrics != nil {
+		// metrics already registered
+		return
+	}
+
+	labels := make([]string, 0)
+	rocksdbMetrics = &Metrics{
+		// Keys
+		NumberKeysWritten: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_written",
+			Help:      "",
+		}, labels),
+		NumberKeysRead: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_read",
+			Help:      "",
+		}, labels),
+		NumberKeysUpdated: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_updated",
+			Help:      "",
+		}, labels),
+		EstimateNumKeys: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "estimate_num_keys",
+			Help:      "estimated number of total keys in the active and unflushed immutable memtables and storage",
+		}, labels),
+
+		// Files
+		NumberFileOpens: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "file",
+			Name:      "number_file_opens",
+			Help:      "",
+		}, labels),
+		NumberFileErrors: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "file",
+			Name:      "number_file_errors",
+			Help:      "",
+		}, labels),
+
+		// Memory
+		BlockCacheUsage: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "block_cache_usage",
+			Help:      "memory size for the entries residing in block cache",
+		}, labels),
+		EstimateTableReadersMem: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "estimate_table_readers_mem",
+			Help:      "estimated memory used for reading SST tables, excluding memory used in block cache (e.g., filter and index blocks)",
+		}, labels),
+		CurSizeAllMemTables: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "cur_size_all_mem_tables",
+			Help:      "approximate size of active and unflushed immutable memtables (bytes)",
+		}, labels),
+		BlockCachePinnedUsage: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "block_cache_pinned_usage",
+			Help:      "returns the memory size for the entries being pinned",
+		}, labels),
+
+		// Cache
+		BlockCacheMiss: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_miss",
+			Help:      "block_cache_miss == block_cache_index_miss + block_cache_filter_miss + block_cache_data_miss",
+		}, labels),
+		BlockCacheHit: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_hit",
+			Help:      "block_cache_hit == block_cache_index_hit + block_cache_filter_hit + block_cache_data_hit",
+		}, labels),
+		BlockCacheAdd: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_add",
+			Help:      "number of blocks added to block cache",
+		}, labels),
+		BlockCacheAddFailures: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_add_failures",
+			Help:      "number of failures when adding blocks to block cache",
+		}, labels),
+	}
+}
+
+// report reports metrics to prometheus based on rocksdb props and stats
+func (m *Metrics) report(props *properties, stats *stats) {
+	// Keys
+	m.NumberKeysWritten.Set(float64(stats.NumberKeysWritten))
+	m.NumberKeysRead.Set(float64(stats.NumberKeysRead))
+	m.NumberKeysUpdated.Set(float64(stats.NumberKeysUpdated))
+	m.EstimateNumKeys.Set(float64(props.EstimateNumKeys))
+
+	// Files
+	m.NumberFileOpens.Set(float64(stats.NumberFileOpens))
+	m.NumberFileErrors.Set(float64(stats.NumberFileErrors))
+
+	// Memory
+	m.BlockCacheUsage.Set(float64(props.BlockCacheUsage))
+	m.EstimateTableReadersMem.Set(float64(props.EstimateTableReadersMem))
+	m.CurSizeAllMemTables.Set(float64(props.CurSizeAllMemTables))
+	m.BlockCachePinnedUsage.Set(float64(props.BlockCachePinnedUsage))
+
+	// Cache
+	m.BlockCacheMiss.Set(float64(stats.BlockCacheMiss))
+	m.BlockCacheHit.Set(float64(stats.BlockCacheHit))
+	m.BlockCacheAdd.Set(float64(stats.BlockCacheAdd))
+	m.BlockCacheAddFailures.Set(float64(stats.BlockCacheAddFailures))
+}

cmd/kava/opendb/opendb.go

@@ -0,0 +1,18 @@
+//go:build !rocksdb
+// +build !rocksdb
+
+package opendb
+
+import (
+	"path/filepath"
+
+	"github.com/cosmos/cosmos-sdk/server/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+// OpenDB is a copy of default DBOpener function used by ethermint, see for details:
+// https://github.com/evmos/ethermint/blob/07cf2bd2b1ce9bdb2e44ec42a39e7239292a14af/server/start.go#L647
+func OpenDB(_ types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(home, "data")
+	return dbm.NewDB("application", backendType, dataDir)
+}