Make rocksdb configurable (#1658)

* Make rocksdb configurable

* Make sure rocksdb tests are running in CI

* Updating ci-rocksdb-build workflow

* Remove test.sh

* Update tm-db dependency

(cherry picked from commit 90fbe1a)

# Conflicts:
#	.github/workflows/ci-rocksdb-build.yml
#	Dockerfile-rocksdb
#	cmd/kava/cmd/root.go
#	go.mod
#	go.sum

.github/workflows/ci-rocksdb-build.yml

@@ -0,0 +1,47 @@
+name: Continuous Integration (Rocksdb Build)
+
+env:
+  ROCKSDB_VERSION: v8.1.1
+
+on:
+  workflow_call:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repo from current commit
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.20"
+          check-latest: true
+          cache: true
+      - name: build rocksdb dependency
+        run: bash ${GITHUB_WORKSPACE}/.github/scripts/install-rocksdb.sh
+      - name: build application
+        run: make build COSMOS_BUILD_OPTIONS=rocksdb
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: install RocksDB dependencies
+        run: sudo apt-get update
+          && sudo apt-get install -y git make gcc libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+      - name: install RocksDB as shared library
+        run: git clone https://github.com/facebook/rocksdb.git
+          && cd rocksdb
+          && git checkout $ROCKSDB_VERSION
+          && sudo make -j$(nproc) install-shared
+          && sudo ldconfig
+      - name: checkout repo from current commit
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.20"
+          check-latest: true
+          cache: true
+      - name: run unit tests
+        run: make test-rocksdb

Dockerfile-rocksdb

@@ -0,0 +1,47 @@
+FROM golang:1.20-bullseye AS kava-builder
+
+# Set up dependencies
+RUN apt-get update \
+    && apt-get install -y git make gcc libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory for the build
+WORKDIR /root
+# default home directory is /root
+
+# install rocksdb
+ARG rocksdb_version=v8.1.1
+ENV ROCKSDB_VERSION=$rocksdb_version
+
+RUN git clone https://github.com/facebook/rocksdb.git \
+    && cd rocksdb \
+    && git checkout $ROCKSDB_VERSION \
+    && make -j$(nproc) install-shared \
+    && ldconfig
+
+# Add source files for kava
+COPY . kava
+
+ARG kava_database_backend=rocksdb
+ENV KAVA_DATABASE_BACKEND=$kava_database_backend
+
+# Mount go build and mod caches as container caches, persisted between builder invocations
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    cd kava && make install COSMOS_BUILD_OPTIONS=$KAVA_DATABASE_BACKEND
+
+
+FROM ubuntu:22.04
+
+RUN apt-get update \
+    && apt-get install -y libsnappy1v5 libgflags2.2 zlib1g libbz2-1.0 curl jq \
+    && rm -rf /var/lib/apt/lists/*
+
+# copy rocksdb shared objects
+COPY --from=kava-builder /usr/local/lib/ /usr/local/lib/
+RUN ldconfig
+
+# copy kava binary
+COPY --from=kava-builder /go/bin/kava /bin/kava
+
+CMD ["kava"]

Makefile

@@ -278,6 +278,9 @@ test-basic: test
 test:
 	@go test $$(go list ./... | grep -v 'contrib')
 
+test-rocksdb:
+	@go test -tags=rocksdb ./cmd/kava/opendb
+
 # Run cli integration tests
 # `-p 4` to use 4 cores, `-tags cli_test` to tell go not to ignore the cli package
 # These tests use the `kvd` or `kvcli` binaries in the build dir, or in `$BUILDDIR` if that env var is set.

cmd/kava/cmd/root.go

@@ -22,7 +22,11 @@ import (
 	"github.com/kava-labs/kava/app"
 	"github.com/kava-labs/kava/app/params"
 	kavaclient "github.com/kava-labs/kava/client"
+<<<<<<< HEAD
 	"github.com/kava-labs/kava/migrate"
+=======
+	"github.com/kava-labs/kava/cmd/kava/opendb"
+>>>>>>> 90fbe1aa (Make rocksdb configurable (#1658))
 )
 
 // EnvPrefix is the prefix environment variables must have to configure the app.
@@ -100,8 +104,22 @@ func addSubCmds(rootCmd *cobra.Command, encodingConfig params.EncodingConfig, de
 		encodingConfig: encodingConfig,
 	}
 
+	opts := ethermintserver.StartOptions{
+		AppCreator:      ac.newApp,
+		DefaultNodeHome: app.DefaultNodeHome,
+		DBOpener:        opendb.OpenDB,
+	}
 	// ethermintserver adds additional flags to start the JSON-RPC server for evm support
+<<<<<<< HEAD
 	ethermintserver.AddCommands(rootCmd, defaultNodeHome, ac.newApp, ac.appExport, ac.addStartCmdFlags)
+=======
+	ethermintserver.AddCommands(
+		rootCmd,
+		opts,
+		ac.appExport,
+		ac.addStartCmdFlags,
+	)
+>>>>>>> 90fbe1aa (Make rocksdb configurable (#1658))
 
 	// add keybase, auxiliary RPC, query, and tx child commands
 	rootCmd.AddCommand(

cmd/kava/opendb/opendb.go

@@ -0,0 +1,18 @@
+//go:build !rocksdb
+// +build !rocksdb
+
+package opendb
+
+import (
+	"path/filepath"
+
+	"github.com/cosmos/cosmos-sdk/server/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+// OpenDB is a copy of default DBOpener function used by ethermint, see for details:
+// https://github.com/evmos/ethermint/blob/07cf2bd2b1ce9bdb2e44ec42a39e7239292a14af/server/start.go#L647
+func OpenDB(_ types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(home, "data")
+	return dbm.NewDB("application", backendType, dataDir)
+}

cmd/kava/opendb/opendb_rocksdb.go

@@ -0,0 +1,170 @@
+//go:build rocksdb
+// +build rocksdb
+
+// Copyright 2023 Kava Labs, Inc.
+// Copyright 2023 Cronos Labs, Inc.
+//
+// Derived from https://github.com/crypto-org-chain/cronos@496ce7e
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package opendb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/cosmos/cosmos-sdk/server/types"
+	"github.com/linxGnu/grocksdb"
+	"github.com/spf13/cast"
+	dbm "github.com/tendermint/tm-db"
+)
+
+var ErrUnexpectedConfiguration = errors.New("unexpected rocksdb configuration, rocksdb should have only one column family named default")
+
+const (
+	// default tm-db block cache size for RocksDB
+	blockCacheSize = 1 << 30
+
+	defaultColumnFamilyName = "default"
+
+	maxOpenFilesDBOptName          = "max_open_files"
+	maxFileOpeningThreadsDBOptName = "max_file_opening_threads"
+
+	writeBufferSizeCFOptName = "write_buffer_size"
+	numLevelsCFOptName       = "num_levels"
+)
+
+func OpenDB(appOpts types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(home, "data")
+	if backendType == dbm.RocksDBBackend {
+		return openRocksdb(filepath.Join(dataDir, "application.db"), appOpts)
+	}
+
+	return dbm.NewDB("application", backendType, dataDir)
+}
+
+// openRocksdb loads existing options, overrides some of them with appOpts and opens database
+// option will be overridden only in case if it explicitly specified in appOpts
+func openRocksdb(dir string, appOpts types.AppOptions) (dbm.DB, error) {
+	dbOpts, cfOpts, err := loadLatestOptions(dir)
+	if err != nil {
+		return nil, err
+	}
+	// customize rocksdb options
+	dbOpts = overrideDBOpts(dbOpts, appOpts)
+	cfOpts = overrideCFOpts(cfOpts, appOpts)
+
+	return newRocksDBWithOptions("application", dir, dbOpts, cfOpts)
+}
+
+// loadLatestOptions loads and returns database and column family options
+// if options file not found, it means database isn't created yet, in such case default tm-db options will be returned
+// if database exists it should have only one column family named default
+func loadLatestOptions(dir string) (*grocksdb.Options, *grocksdb.Options, error) {
+	latestOpts, err := grocksdb.LoadLatestOptions(dir, grocksdb.NewDefaultEnv(), true, grocksdb.NewLRUCache(blockCacheSize))
+	if err != nil && strings.HasPrefix(err.Error(), "NotFound: ") {
+		return newDefaultOptions(), newDefaultOptions(), nil
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfNames := latestOpts.ColumnFamilyNames()
+	cfOpts := latestOpts.ColumnFamilyOpts()
+	// db should have only one column family named default
+	ok := len(cfNames) == 1 && cfNames[0] == defaultColumnFamilyName
+	if !ok {
+		return nil, nil, ErrUnexpectedConfiguration
+	}
+
+	// return db and cf opts
+	return latestOpts.Options(), &cfOpts[0], nil
+}
+
+// overrideDBOpts merges dbOpts and appOpts, appOpts takes precedence
+func overrideDBOpts(dbOpts *grocksdb.Options, appOpts types.AppOptions) *grocksdb.Options {
+	maxOpenFiles := appOpts.Get(maxOpenFilesDBOptName)
+	if maxOpenFiles != nil {
+		dbOpts.SetMaxOpenFiles(cast.ToInt(maxOpenFiles))
+	}
+
+	maxFileOpeningThreads := appOpts.Get(maxFileOpeningThreadsDBOptName)
+	if maxFileOpeningThreads != nil {
+		dbOpts.SetMaxFileOpeningThreads(cast.ToInt(maxFileOpeningThreads))
+	}
+
+	return dbOpts
+}
+
+// overrideCFOpts merges cfOpts and appOpts, appOpts takes precedence
+func overrideCFOpts(cfOpts *grocksdb.Options, appOpts types.AppOptions) *grocksdb.Options {
+	writeBufferSize := appOpts.Get(writeBufferSizeCFOptName)
+	if writeBufferSize != nil {
+		cfOpts.SetWriteBufferSize(cast.ToUint64(writeBufferSize))
+	}
+
+	numLevels := appOpts.Get(numLevelsCFOptName)
+	if numLevels != nil {
+		cfOpts.SetNumLevels(cast.ToInt(numLevels))
+	}
+
+	return cfOpts
+}
+
+// newRocksDBWithOptions opens rocksdb with provided database and column family options
+// newRocksDBWithOptions expects that db has only one column family named default
+func newRocksDBWithOptions(name string, dir string, dbOpts, cfOpts *grocksdb.Options) (*dbm.RocksDB, error) {
+	dbPath := filepath.Join(dir, name+".db")
+
+	// Ensure path exists
+	if err := os.MkdirAll(dbPath, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create db path: %w", err)
+	}
+
+	db, _, err := grocksdb.OpenDbColumnFamilies(dbOpts, dbPath, []string{defaultColumnFamilyName}, []*grocksdb.Options{cfOpts})
+	if err != nil {
+		return nil, err
+	}
+	ro := grocksdb.NewDefaultReadOptions()
+	wo := grocksdb.NewDefaultWriteOptions()
+	woSync := grocksdb.NewDefaultWriteOptions()
+	woSync.SetSync(true)
+	return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+}
+
+// newDefaultOptions returns default tm-db options for RocksDB, see for details:
+// https://github.com/Kava-Labs/tm-db/blob/94ff76d31724965f8883cddebabe91e0d01bc03f/rocksdb.go#L30
+func newDefaultOptions() *grocksdb.Options {
+	// default rocksdb option, good enough for most cases, including heavy workloads.
+	// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads).
+	// compression: snappy as default, need to -lsnappy to enable.
+	bbto := grocksdb.NewDefaultBlockBasedTableOptions()
+	bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30))
+	bbto.SetFilterPolicy(grocksdb.NewBloomFilter(10))
+
+	opts := grocksdb.NewDefaultOptions()
+	opts.SetBlockBasedTableFactory(bbto)
+	// SetMaxOpenFiles to 4096 seems to provide a reliable performance boost
+	opts.SetMaxOpenFiles(4096)
+	opts.SetCreateIfMissing(true)
+	opts.IncreaseParallelism(runtime.NumCPU())
+	// 1.5GB maximum memory use for writebuffer.
+	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
+
+	return opts
+}