release/v0.1.0 (#34)

* deps: use forked wasmvm, update Dockerfile * deps: use forked tm-db * fix: run indexer before batch flush * fix: use viper-provided app.toml for wasmconfig * feat: support FlagSkipGenesisInvariants * docs: explain shared/static * docs: specify notable differences from core * docs: mention healthcheck * fix: wasmvm artefact url * chore: flatten directory structure * deps: use terra-money/wasmvm for go.mod * deps: bump terra-money/core * chore: go.sum * chore: envvar name, dockerfile cleanup
terra-money · Mar 11, 2022 · 3a78214 · 3a78214
1 parent f493ffd
commit 3a78214
Show file tree

Hide file tree

Showing 58 changed files with 161 additions and 105 deletions.
diff --git a/bin/v0.34.x/Dockerfile → Dockerfile b/bin/v0.34.x/Dockerfile → Dockerfile
@@ -15,12 +15,12 @@ WORKDIR /code
 COPY . /code/
 
 # See https://github.com/CosmWasm/wasmvm/releases
-ADD https://github.com/CosmWasm/wasmvm/releases/download/v0.16.3/libwasmvm_muslc.a /lib/libwasmvm_muslc.a
+ADD https://github.com/terra-money/wasmvm/releases/download/v0.16.4/libwasmvm_muslc.a /lib/libwasmvm_muslc.a
 
 # use mimalloc for musl
 RUN git clone --depth 1 https://github.com/microsoft/mimalloc; cd mimalloc; mkdir build; cd build; cmake ..; make -j$(nproc); make install
 
-ENV MIMALLOC_LARGE_OS_PAGES=1
+ENV MIMALLOC_RESERVE_HUGE_OS_PAGES=4
 
 # force it to use static lib (from above) not standard libgo_cosmwasm.so file
 RUN LEDGER_ENABLED=false go build -work -tags muslc,linux -mod=readonly -ldflags="-extldflags '-L/code/mimalloc/build -lmimalloc'" -o build/mantlemint ./sync.go

diff --git a/bin/v0.34.x/Makefile → Makefile b/bin/v0.34.x/Makefile → Makefile
@@ -6,16 +6,16 @@ build: go.sum
 ifeq ($(OS),Windows_NT)
 	exit 1
 else
-	go build -mod=readonly $(BUILD_FLAGS) -o build/mantlemint ./sync
+	go build -mod=readonly $(BUILD_FLAGS) -o build/mantlemint ./sync.go
 endif
 
 
-build-linux:
+build-static:
 	mkdir -p $(BUILDDIR)
-	docker buildx build --platform=linux/amd64 --tag terramoney/mantlemint ./
-	docker create --platform=linux/amd64 --name temp terramoney/mantlemint:latest
+	docker buildx build --tag terramoney/mantlemint ./
+	docker create --name temp terramoney/mantlemint:latest
 	docker cp temp:/usr/local/bin/mantlemint $(BUILDDIR)/
 	docker rm temp
 
 install: go.sum
-	go install -mod=readonly $(BUILD_FLAGS) ./
+	go install -mod=readonly $(BUILD_FLAGS) ./
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 
 Mantlemint is a fast core optimized for serving massive user queries.
 
-Native query performance on RPC is very slow and is not suitable for massive query handling, due to the inefficiencies introduced by IAVL tree. Mantlemint is running on `fauxMerkleTree` mode, basically removing the IAVL inefficiencies while using the same core to compute the same module outputs. 
+Native query performance on RPC is slow and is not suitable for massive query handling, due to the inefficiencies introduced by IAVL tree. Mantlemint is running on `fauxMerkleTree` mode, basically removing the IAVL inefficiencies while using the same core to compute the same module outputs. 
 
 If you are looking to serve any kind of public node accepting varying degrees of end-user queries, it is recommended that you run a mantlemint instance alongside of your RPC. While mantlemint is indeed faster at resolving queries, due to the absence of IAVL tree and native tendermint, it cannot join p2p network by itself. Rather, you would have to relay finalized blocks to mantlemint, using RPC's websocket.
 
@@ -51,6 +51,22 @@ Mantlemint internally runs the same Terra Core, therefore you need to provide th
 
 It is __required__ to run mantlemint in a separate `$HOME` directory than RPC; while mantlemint maintains its own database, some of the data may be overwritten by either mantlemint or RPC and may cause trouble.
 
+
+### Building
+
+#### 1. As a statically-linked application
+```sh
+$ make build-static # results in build/mantlemint
+```
+
+
+#### 2. As a dynamically-linked application
+```sh
+$ make build # results in build/mantlemint
+$ make install # results in $GOPATH/bin/mantlemint
+```
+
+
 ### Running
 
 Mantlemint depends on 2 configs:
@@ -67,7 +83,7 @@ GENESIS_PATH=config/genesis.json \
 # - create and maintain $HOME/mantlemint.db directory
 # - create and maintain $HOME/data/* for wasm blobs; (unsafe to share with RPC!)
 # - create and maintain $HOME/$(INDEXER_DB).db for mantle indexers
-HOME=mantlemint \
+MANTLEMINT_HOME=mantlemint \
 
 # Chain ID 
 CHAIN_ID=columbus-5 \
@@ -82,18 +98,49 @@ WS_ENDPOINTS=ws://rpc1:26657/websocket,ws://rpc2:26657/websocket \
 INDEXER_DB=indexer \
 
 # Flag to enable/disable mantlemint sync, mainly for debugging
-DISABLE_SYNC=true \
+DISABLE_SYNC=false \
 
 # Run sync binary
 sync
 ```
 
+## Health check
+
+`mantlemint` implements a separate `/health` endpoint. It is particularly useful if you want to suppress traffics being routed to `mantlemint` nodes still syncing or unavailable due to whatever reason.
+
+The endpoint will response:
+- `200 OK` if mantlemint sync status is up-to date (i.e. syncing using websocket from RPC)
+- `400 NOK` if mantlemint is still syncing past blocks, and is not ready to serve the latest state yet.
+
+Please note that mantlemint still is able to serve queries while `/health` returns `NOK`.
 
 ## Default Indexes
 
 - `/index/tx/by_height/{height}`: List all transactions and their responses in a block. Equivalent to `tendermint/block?height=xxx`, with tx responses base64-decoded for better usability.
 - `/index/tx/by_hash/{txHash}`: Get transaction and its response by hash. Equivalent to `lcd/txs/{hash}`, but without hitting RPC.
 
-# LICENSE
+## Notable Differences from [core](https://github.com/terra-money/core)
+
+- Uses a forked [tendermint/tm-db](https://github.com/terra-money/tm-db/commit/c71e8b6e9f20d7f5be32527db4a92ae19ac0d2b2): Disables unncessary mutexes in `prefixdb` methods
+- Replaces ABCIClient with [NewConcurrentQueryClient](https://github.com/terra-money/mantlemint/blob/main/bin/v0.34.x/mantlemint/client.go#L110): Removal of mutexes allow better concurrency, even during block injection
+- Uses single batch-protected db: All state changes are flushed at once, making it safe to read from db during block injection
+- Automatic failover: In case of block injection failure, mantlemint reverts back to the previous known state and retry
+
+
+## Community
+
+- [Offical Website](https://terra.money)
+- [Discord](https://discord.gg/e29HWwC2Mz)
+- [Telegram](https://t.me/terra_announcements)
+- [Twitter](https://twitter.com/terra_money)
+- [YouTube](https://goo.gl/3G4T1z)
+
+## Contributing
+
+If you are interested in contributing to Terra Core source, please review our [code of conduct](./CODE_OF_CONDUCT.md).
+
+# License
+
+This software is licensed under the Apache 2.0 license. Read more about it here.
 
-Apache 2.0
+© 2021 Terraform Labs, PTE LTD
diff --git a/bin/v0.34.x/block_feed/aggregate.go → block_feed/aggregate.go b/bin/v0.34.x/block_feed/aggregate.go → block_feed/aggregate.go
diff --git a/bin/v0.34.x/block_feed/helpers.go → block_feed/helpers.go b/bin/v0.34.x/block_feed/helpers.go → block_feed/helpers.go
diff --git a/bin/v0.34.x/block_feed/rpc.go → block_feed/rpc.go b/bin/v0.34.x/block_feed/rpc.go → block_feed/rpc.go
diff --git a/bin/v0.34.x/block_feed/types.go → block_feed/types.go b/bin/v0.34.x/block_feed/types.go → block_feed/types.go
diff --git a/bin/v0.34.x/block_feed/websocket.go → block_feed/websocket.go b/bin/v0.34.x/block_feed/websocket.go → block_feed/websocket.go
diff --git a/bin/v0.34.x/config/config.go → config/config.go b/bin/v0.34.x/config/config.go → config/config.go
@@ -2,30 +2,34 @@ package config
 
 import (
 	"fmt"
+	"github.com/cosmos/cosmos-sdk/x/crisis"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 	terra "github.com/terra-money/core/app"
 	"os"
+	"path/filepath"
 	"strings"
 )
 
 type Config struct {
-	GenesisPath         string
-	Home                string
-	ChainID             string
-	RPCEndpoints        []string
-	WSEndpoints         []string
-	MantlemintDB        string
-	IndexerDB           string
-	DisableSync         bool
+	GenesisPath  string
+	Home         string
+	ChainID      string
+	RPCEndpoints []string
+	WSEndpoints  []string
+	MantlemintDB string
+	IndexerDB    string
+	DisableSync  bool
 }
 
 // NewConfig converts envvars into consumable config chunks
 func NewConfig() Config {
-	return Config{
+	cfg := Config{
 		// GenesisPath sets the location of genesis
 		GenesisPath: getValidEnv("GENESIS_PATH"),
 
 		// Home sets where the default terra home is.
-		Home: getValidEnv("HOME"),
+		Home: getValidEnv("MANTLEMINT_HOME"),
 
 		// ChainID sets expected chain id for this mantlemint instance
 		ChainID: getValidEnv("CHAIN_ID"),
@@ -61,6 +65,23 @@ func NewConfig() Config {
 			return disableSync == "true"
 		}(),
 	}
+
+	viper.SetConfigType("toml")
+	viper.SetConfigName("app")
+	viper.AutomaticEnv()
+	viper.AddConfigPath(filepath.Join(cfg.Home, "config"))
+
+	pflag.Bool(crisis.FlagSkipGenesisInvariants, false, "Skip x/crisis invariants check on startup")
+	pflag.Parse()
+	if bindErr := viper.BindPFlags(pflag.CommandLine); bindErr != nil {
+		panic(bindErr)
+	}
+
+	if err := viper.MergeInConfig(); err != nil {
+		panic(fmt.Errorf("failed to merge configuration: %w", err))
+	}
+
+	return cfg
 }
 
 func (cfg Config) Print() {

diff --git a/bin/v0.34.x/db/heleveldb/config.go → db/heleveldb/config.go b/bin/v0.34.x/db/heleveldb/config.go → db/heleveldb/config.go
diff --git a/bin/v0.34.x/db/heleveldb/leveldb_batch.go → db/heleveldb/leveldb_batch.go b/bin/v0.34.x/db/heleveldb/leveldb_batch.go → db/heleveldb/leveldb_batch.go
@@ -4,8 +4,8 @@ import (
 	"fmt"
 
 	tmdb "github.com/tendermint/tm-db"
-	"github.com/terra-money/mantlemint-provider-v0.34.x/db/hld"
-	"github.com/terra-money/mantlemint-provider-v0.34.x/db/rollbackable"
+	"github.com/terra-money/mantlemint/db/hld"
+	"github.com/terra-money/mantlemint/db/rollbackable"
 )
 
 var _ hld.HeightLimitEnabledBatch = (*LevelBatch)(nil)

diff --git a/bin/v0.34.x/db/heleveldb/leveldb_driver.go → db/heleveldb/leveldb_driver.go b/bin/v0.34.x/db/heleveldb/leveldb_driver.go → db/heleveldb/leveldb_driver.go
@@ -5,8 +5,8 @@ import (
 	"math"
 
 	tmdb "github.com/tendermint/tm-db"
-	"github.com/terra-money/mantlemint-provider-v0.34.x/db/hld"
-	"github.com/terra-money/mantlemint-provider-v0.34.x/lib"
+	"github.com/terra-money/mantlemint/db/hld"
+	"github.com/terra-money/mantlemint/lib"
 )
 
 type Driver struct {

diff --git a/bin/v0.34.x/db/heleveldb/leveldb_iterator.go → db/heleveldb/leveldb_iterator.go b/bin/v0.34.x/db/heleveldb/leveldb_iterator.go → db/heleveldb/leveldb_iterator.go
@@ -4,7 +4,7 @@ import (
 	"bytes"
 
 	tmdb "github.com/tendermint/tm-db"
-	"github.com/terra-money/mantlemint-provider-v0.34.x/db/hld"
+	"github.com/terra-money/mantlemint/db/hld"
 )
 
 var _ hld.HeightLimitEnabledIterator = (*Iterator)(nil)

diff --git a/bin/v0.34.x/db/heleveldb/types.go → db/heleveldb/types.go b/bin/v0.34.x/db/heleveldb/types.go → db/heleveldb/types.go
@@ -4,7 +4,7 @@ import (
 	"encoding/binary"
 	"math"
 
-	"github.com/terra-money/mantlemint-provider-v0.34.x/lib"
+	"github.com/terra-money/mantlemint/lib"
 )
 
 const (

diff --git a/bin/v0.34.x/db/hld/cluster_utils.go → db/hld/cluster_utils.go b/bin/v0.34.x/db/hld/cluster_utils.go → db/hld/cluster_utils.go
diff --git a/bin/v0.34.x/db/hld/height_limited_db.go → db/hld/height_limited_db.go b/bin/v0.34.x/db/hld/height_limited_db.go → db/hld/height_limited_db.go
@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/terra-money/mantlemint-provider-v0.34.x/lib"
+	"github.com/terra-money/mantlemint/lib"
 
 	tmdb "github.com/tendermint/tm-db"
 )

diff --git a/...v0.34.x/db/hld/height_limited_iterator.go → db/hld/height_limited_iterator.go b/...v0.34.x/db/hld/height_limited_iterator.go → db/hld/height_limited_iterator.go
diff --git a/bin/v0.34.x/db/hld/types.go → db/hld/types.go b/bin/v0.34.x/db/hld/types.go → db/hld/types.go
diff --git a/...4.x/db/rollbackable/rollbackable_batch.go → db/rollbackable/rollbackable_batch.go b/...4.x/db/rollbackable/rollbackable_batch.go → db/rollbackable/rollbackable_batch.go
diff --git a/bin/v0.34.x/db/safe_batch/safe_batch.go → db/safe_batch/safe_batch.go b/bin/v0.34.x/db/safe_batch/safe_batch.go → db/safe_batch/safe_batch.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	tmdb "github.com/tendermint/tm-db"
-	"github.com/terra-money/mantlemint-provider-v0.34.x/db/rollbackable"
+	"github.com/terra-money/mantlemint/db/rollbackable"
 )
 
 var _ tmdb.DB = (*SafeBatchDB)(nil)

diff --git a/....34.x/db/safe_batch/safe_batch_nullify.go → db/safe_batch/safe_batch_nullify.go b/....34.x/db/safe_batch/safe_batch_nullify.go → db/safe_batch/safe_batch_nullify.go
diff --git a/bin/v0.34.x/db/snappy/snappy_batch.go → db/snappy/snappy_batch.go b/bin/v0.34.x/db/snappy/snappy_batch.go → db/snappy/snappy_batch.go
diff --git a/bin/v0.34.x/db/snappy/snappy_db.go → db/snappy/snappy_db.go b/bin/v0.34.x/db/snappy/snappy_db.go → db/snappy/snappy_db.go
diff --git a/bin/v0.34.x/db/snappy/snappy_db_test.go → db/snappy/snappy_db_test.go b/bin/v0.34.x/db/snappy/snappy_db_test.go → db/snappy/snappy_db_test.go
diff --git a/bin/v0.34.x/db/types.go → db/types.go b/bin/v0.34.x/db/types.go → db/types.go
diff --git a/bin/v0.34.x/go.mod → go.mod b/bin/v0.34.x/go.mod → go.mod
@@ -1,9 +1,8 @@
-module github.com/terra-money/mantlemint-provider-v0.34.x
+module github.com/terra-money/mantlemint
 
 go 1.17
 
 require (
-	github.com/CosmWasm/wasmvm v0.16.3 // indirect
 	github.com/cosmos/cosmos-sdk v0.44.5
 	github.com/cosmos/iavl v0.17.3
 	github.com/gogo/protobuf v1.3.3
@@ -12,17 +11,19 @@ require (
 	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/pkg/errors v0.9.1
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/tendermint/tendermint v0.34.14
 	github.com/tendermint/tm-db v0.6.6
-	github.com/terra-money/core v0.5.16
+	github.com/terra-money/core v0.5.18-0.20220304040324-70c1ef6f11b0
 )
 
 require (
 	filippo.io/edwards25519 v1.0.0-beta.2 // indirect
 	github.com/99designs/keyring v1.1.6 // indirect
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
+	github.com/CosmWasm/wasmvm v0.16.3 // indirect
 	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/armon/go-metrics v0.3.9 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -33,6 +34,7 @@ require (
 	github.com/confio/ics23/go v0.6.6 // indirect
 	github.com/cosmos/btcutil v1.0.4 // indirect
 	github.com/cosmos/go-bip39 v1.0.0 // indirect
+	github.com/cosmos/gorocksdb v1.2.0 // indirect
 	github.com/cosmos/ibc-go v1.1.5 // indirect
 	github.com/cosmos/ledger-cosmos-go v0.11.1 // indirect
 	github.com/cosmos/ledger-go v0.9.2 // indirect
@@ -85,10 +87,8 @@ require (
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/cobra v1.2.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect
-	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/tendermint/btcd v0.1.1 // indirect
 	github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 // indirect
 	github.com/tendermint/go-amino v0.16.0 // indirect
@@ -100,7 +100,7 @@ require (
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 // indirect
-	google.golang.org/grpc v1.42.0 // indirect
+	google.golang.org/grpc v1.44.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/ini.v1 v1.63.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -114,3 +114,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.33.2
 replace github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 
 replace github.com/99designs/keyring => github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76
+
+replace github.com/CosmWasm/wasmvm => github.com/terra-money/wasmvm v0.16.4
+
+replace github.com/tendermint/tm-db => github.com/terra-money/tm-db v0.6.5-0.20220307182415-c71e8b6e9f20