Panic with 1.4.0-rc4-testnet on chainID cronostestnet_338-3

[harshsingh-cs]

Describe the bug
Our testnet node is crashing with panic.

Expected behavior
node to sync

Config.toml

config.toml: |-
      # TCP or UNIX socket address of the ABCI application,
      # or the name of an ABCI application compiled in with the Tendermint binary
      proxy_app = "tcp://127.0.0.1:26658"

      # A custom human readable name for this node
      moniker = "000-000-000"

      # If this node is many blocks behind the tip of the chain, FastSync
      # allows them to catchup quickly by downloading blocks in parallel
      # and verifying their commits
      fast_sync = true
      db_backend = "goleveldb"
      db_dir = "/home/cronos/data"
      log_level = "info"
      log_format = "plain"

      ##### additional base config options #####

      # Path to the JSON file containing the initial validator set and other meta data
      genesis_file = "config/genesis.json"

      # Path to the JSON file containing the private key to use as a validator in the consensus protocol
      priv_validator_key_file = "config/priv_validator_key.json"

      # Path to the JSON file containing the last sign state of a validator
      priv_validator_state_file = "data/priv_validator_state.json"

      # TCP or UNIX socket address for Tendermint to listen on for
      # connections from an external PrivValidator process
      priv_validator_laddr = ""

      # Path to the JSON file containing the private key to use for node authentication in the p2p protocol
      node_key_file = "config/node_key.json"

      # Mechanism to connect to the ABCI application: socket | grpc
      abci = "socket"

      # If true, query the ABCI app on connecting to a new peer
      # so the app can decide if we should keep the connection or not
      filter_peers = false


      [rpc]
      laddr = "tcp://127.0.0.1:26657"
      cors_allowed_origins = []
      cors_allowed_methods = ["HEAD", "GET", "POST", ]
      cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]
      grpc_laddr = ""
      grpc_max_open_connections = 900
      unsafe = false
      max_open_connections = 900
      max_subscription_clients = 100
      max_subscriptions_per_client = 5
      experimental_subscription_buffer_size = 200
      experimental_websocket_write_buffer_size = 200
      experimental_close_on_slow_client = false
      # See https://github.com/tendermint/tendermint/issues/3435
      timeout_broadcast_tx_commit = "10s"
      # Maximum size of request body, in bytes
      max_body_bytes = 1000000
      # Maximum size of request header, in bytes
      max_header_bytes = 1048576
      tls_cert_file = ""
      tls_key_file = ""

      # pprof listen address (https://golang.org/pkg/net/http/pprof)
      pprof_laddr = "localhost:6060"

      [p2p]
      laddr = "tcp://0.0.0.0:26656"
      external_address = ""
      seeds =  "[email protected]:26656,[email protected]:26656,b8e6d6e16d236fa6a7101316d96de718200c500c@bd-cronos-testnet-seed-node-01.bdnodes.net:26656"
      persistent_peers = ""
      upnp = false
      # Path to address book
      addr_book_file = "config/addrbook.json"
      addr_book_strict = true
      max_num_inbound_peers = 40
      max_num_outbound_peers = 10
      unconditional_peer_ids = ""
      persistent_peers_max_dial_period = "0s"
      flush_throttle_timeout = "100ms"
      max_packet_msg_payload_size = 1024
      send_rate = 5120000
      recv_rate = 5120000
      # Set true to enable the peer-exchange reactor
      pex = true
      seed_mode = false
      private_peer_ids = ""
      allow_duplicate_ip = false
      # Peer connection configuration.
      handshake_timeout = "20s"
      dial_timeout = "3s"

      [mempool]
      # Mempool version to use:
      #   1) "v0" - (default) FIFO mempool.
      #   2) "v1" - prioritized mempool.
      version = "v1"
      recheck = true
      broadcast = true
      wal_dir = ""

      # Maximum number of transactions in the mempool
      size = 5000

      # Limit the total size of all txs in the mempool.
      # This only accounts for raw transactions (e.g. given 1MB transactions and
      # max_txs_bytes=5MB, mempool will only accept 5 transactions).
      max_txs_bytes = 1073741824

      # Size of the cache (used to filter transactions we saw earlier) in transactions
      cache_size = 10000

      # Do not remove invalid transactions from the cache (default: false)
      # Set to true if it's not possible for any invalid transaction to become valid
      # again in the future.
      keep-invalid-txs-in-cache = false

      # Maximum size of a single transaction.
      # NOTE: the max size of a tx transmitted over the network is {max_tx_bytes}.
      max_tx_bytes = 1048576

      # Maximum size of a batch of transactions to send to a peer
      # Including space needed by encoding (one varint per transaction).
      # XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796
      max_batch_bytes = 0
      ttl-duration = "0s"
      ttl-num-blocks = 0



      # Time to spend discovering snapshots before initiating a restore.
      discovery_time = "15s"

      # Temporary directory for state sync snapshot chunks, defaults to the OS tempdir (typically /tmp).
      # Will create a new, randomly named directory within, and remove it when done.
      temp_dir = ""

      # The timeout duration before re-requesting a chunk, possibly from a different
      # peer (default: 1 minute).
      chunk_request_timeout = "10s"

      # The number of concurrent chunk fetchers to run (default: 1).
      chunk_fetchers = "4"

      [fastsync]

      # Fast Sync version to use:
      #   1) "v0" (default) - the legacy fast sync implementation
      #   2) "v1" - refactor of v0 version for better testability
      #   2) "v2" - complete redesign of v0, optimized for testability & readability
      version = "v0"

      [consensus]

      wal_file = "data/cs.wal/wal"

      # How long we wait for a proposal block before prevoting nil
      timeout_propose = "3s"
      # How much timeout_propose increases with each round
      timeout_propose_delta = "500ms"
      # How long we wait after receiving +2/3 prevotes for “anything” (ie. not a single block or nil)
      timeout_prevote = "1s"
      # How much the timeout_prevote increases with each round
      timeout_prevote_delta = "500ms"
      # How long we wait after receiving +2/3 precommits for “anything” (ie. not a single block or nil)
      timeout_precommit = "1s"
      # How much the timeout_precommit increases with each round
      timeout_precommit_delta = "500ms"
      # How long we wait after committing a block, before starting on the new
      # height (this gives us a chance to receive some more precommits, even
      # though we already have +2/3).
      timeout_commit = "5s"

      # How many blocks to look back to check existence of the node's consensus votes before joining consensus
      # When non-zero, the node will panic upon restart
      # if the same consensus key was used to sign {double_sign_check_height} last blocks.
      # So, validators should stop the state machine, wait for some blocks, and then restart the state machine to avoid panic.
      double_sign_check_height = 0

      # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0)
      skip_timeout_commit = false

      # EmptyBlocks mode and possible interval between empty blocks
      create_empty_blocks = true
      create_empty_blocks_interval = "0s"

      # Reactor sleep duration parameters
      peer_gossip_sleep_duration = "100ms"
      peer_query_maj23_sleep_duration = "2s"


      [tx_index]

      # What indexer to use for transactions
      #
      # The application will set which txs to index. In some cases a node operator will be able
      # to decide which txs to index based on configuration set in the application.
      #
      # Options:
      #   1) "null"
      #   2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
      # 		- When "kv" is chosen "tx.height" and "tx.hash" will always be indexed.
      #   3) "psql" - the indexer services backed by PostgreSQL.
      # When "kv" or "psql" is chosen "tx.height" and "tx.hash" will always be indexed.
      indexer = "kv"

      # The PostgreSQL connection configuration, the connection format:
      #   postgresql://<user>:<password>@<host>:<port>/<db>?<opts>
      psql-conn = ""

      [instrumentation]

      # When true, Prometheus metrics are served under /metrics on
      # PrometheusListenAddr.
      # Check out the documentation for the list of available metrics.
      prometheus = true

      # Address to listen for Prometheus collector(s) connections
      prometheus_listen_addr = ":26660"

      # Maximum number of simultaneous connections.
      # If you want to accept a larger number than the default, make sure
      # you increase your OS limits.
      # 0 - unlimited.
      max_open_connections = 3

      # Instrumentation namespace
      namespace = "tendermint"

app.toml

app.toml: |-
      # config file template https://raw.githubusercontent.com/crypto-org-chain/cronos-testnets/main/cronostestnet_338-3/app.toml
      minimum-gas-prices = "5000000000000basecro"

      # default: the last 100 states are kept in addition to every 500th state; pruning at 10 block intervals
      # nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node)
      # everything: all saved states will be deleted, storing only the current and previous state; pruning at 10 block intervals
      # custom: allow pruning options to be manually specified through 'pruning-keep-recent', 'pruning-keep-every', and 'pruning-interval'
      pruning = "default"

      # These are applied if and only if the pruning strategy is custom.
      [versiondb]
      # Enable defines if the versiondb should be enabled.
      enable = true

      pruning-keep-recent = "0"
      pruning-keep-every = "0"
      pruning-interval = "0"

      halt-height = 0
      halt-time = 0
      min-retain-blocks = 0

      inter-block-cache = true
      index-events = []

      # IavlCacheSize set the size of the iavl tree cache.
      # Default cache size is 50mb.
      iavl-cache-size = 781250

      [telemetry]

      # Prefixed with keys to separate services.
      service-name = ""

      # Enabled enables the application telemetry functionality. When enabled,
      # an in-memory sink is also enabled by default. Operators may also enabled
      # other sinks such as Prometheus.
      enabled = false

      # Enable prefixing gauge values with hostname.
      enable-hostname = false

      # Enable adding hostname to labels.
      enable-hostname-label = false

      # Enable adding service to labels.
      enable-service-label = false

      # PrometheusRetentionTime, when positive, enables a Prometheus metrics sink.
      prometheus-retention-time = 0

      # GlobalLabels defines a global set of name/value label tuples applied to all
      # metrics emitted using the wrapper functions defined in telemetry package.
      #
      # Example:
      # [["chain_id", "cosmoshub-1"]]
      global-labels = [
      ]

      [api]
      # Enable defines if the API server should be enabled.
      enable = false
      # Swagger defines if swagger documentation should automatically be registered.
      swagger = false
      # Address defines the API server to listen on.
      address = "tcp://0.0.0.0:1317"
      # MaxOpenConnections defines the number of maximum open connections.
      max-open-connections = 1000
      # RPCReadTimeout defines the Tendermint RPC read timeout (in seconds).
      rpc-read-timeout = 10
      # RPCWriteTimeout defines the Tendermint RPC write timeout (in seconds).
      rpc-write-timeout = 0
      # RPCMaxBodyBytes defines the Tendermint maximum response body (in bytes).
      rpc-max-body-bytes = 1000000
      # EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk).
      enabled-unsafe-cors = false

      [rosetta]
      enable = false
      address = ":8080"
      blockchain = "app"
      network = "network"
      retries = 3
      offline = false

      [grpc]
      enable = false
      address = "0.0.0.0:9090"

      # State sync snapshots allow other nodes to rapidly join the network without replaying historical
      # blocks, instead downloading and applying a snapshot of the application state at a given height.
      [state-sync]
      snapshot-interval = 0
      snapshot-keep-recent = 2

      [evm]

      # Tracer defines the 'vm.Tracer' type that the EVM will use when the node is run in
      # debug mode. To enable tracing use the '--trace' flag when starting your node.
      # Valid types are: json|struct|access_list|markdown
      tracer = ""

      # MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.
      max-tx-gas-wanted = 500000

      [json-rpc]

      # Enable defines if the gRPC server should be enabled.
      enable = true
      address = "0.0.0.0:8545"
      ws-address = "0.0.0.0:8546"

      # API defines a list of JSON-RPC namespaces that should be enabled
      # Example: "eth,txpool,personal,net,debug,web3"
      api = "eth,net,web3"

      # GasCap sets a cap on gas that can be used in eth_call/estimateGas (0=infinite). Default: 25,000,000.
      gas-cap = 25000000

      # EVMTimeout is the global timeout for eth_call. Default: 5s.
      evm-timeout =  "4m0s"

      # TxFeeCap is the global tx-fee cap for send transaction. Default: 1eth.
      txfee-cap = 1
      # FilterCap sets the global cap for total number of filters that can be created
      filter-cap = 200
      # FeeHistoryCap sets the global cap for total number of blocks that can be fetched
      feehistory-cap = 100
      # LogsCap defines the max number of results can be returned from single 'eth_getLogs' query.
      logs-cap = 10000
      # BlockRangeCap defines the max block range allowed for 'eth_getLogs' query.
      block-range-cap = 2000
      # HTTPTimeout is the read/write timeout of http json-rpc server.
      http-timeout = "4m0s"
      # HTTPIdleTimeout is the idle timeout of http json-rpc server.
      http-idle-timeout = "4m0s"

      [tls]
      # Certificate path defines the cert.pem file path for the TLS configuration.
      certificate-path = ""
      # Key path defines the key.pem file path for the TLS configuration.
      key-path = ""

    client.toml: |-
      # The network chain ID
      chain-id = ""
      # The keyring's backend, where the keys are stored (os|file|kwallet|pass|test|memory)
      keyring-backend = "os"
      # CLI output format (text|json)
      output = "text"
      # <host>:<port> to Tendermint RPC interface for this chain
      node = "tcp://localhost:26657"
      # Transaction broadcasting mode (sync|async|block)
      broadcast-mode = "sync"

Any suggestion pleas

[harshsingh-cs]

Logs

7:58AM INF Unlocking keyring module=server
7:58AM INF starting ABCI with CometBFT module=server
7:58AM ERR Setting ante handler without blacklist module=server
7:58AM INF starting node with ABCI CometBFT in-process module=server
7:58AM INF service start impl=multiAppConn module=proxy msg="Starting multiAppConn service" server=node
7:58AM INF service start connection=query impl=localClient module=abci-client msg="Starting localClient service" server=node
7:58AM INF service start connection=snapshot impl=localClient module=abci-client msg="Starting localClient service" server=node
7:58AM INF service start connection=mempool impl=localClient module=abci-client msg="Starting localClient service" server=node
7:58AM INF service start connection=consensus impl=localClient module=abci-client msg="Starting localClient service" server=node
7:58AM INF service start impl=EventBus module=events msg="Starting EventBus service" server=node
7:58AM INF service start impl=PubSub module=pubsub msg="Starting PubSub service" server=node
7:58AM INF service start impl=IndexerService module=txindex msg="Starting IndexerService service" server=node
7:58AM INF ABCI Handshake App Info hash=E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855 height=0 module=consensus protocol-version=0 server=node software-version=v1.4.0
7:58AM INF ABCI Replay Blocks appHeight=0 module=consensus server=node stateHeight=0 storeHeight=0
7:58AM INF InitChain chainID=cronostestnet_338-3 initialHeight=1 module=server
7:58AM INF initializing blockchain state from genesis.json module=server
7:58AM INF Closing application.db module=server
7:58AM INF Closing snapshots/metadata.db module=server
7:58AM INF Application gracefully shutdown module=server
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x26eb4f3]

goroutine 1 [running]:
github.com/cosmos/cosmos-sdk/x/gov.InitGenesis({{0x640e648, 0x8baf600}, {0x642c920, 0xc0023963f0}, {{0x0, 0x0}, {0xc001cc6918, 0x13}, 0x0, {0x22792648, ...}, ...}, ...}, ...)
	github.com/cosmos/[email protected]/x/gov/genesis.go:21 +0x153
github.com/cosmos/cosmos-sdk/x/gov.AppModule.InitGenesis({{{0x6459d18, 0xc002480070}, {0x0, 0x0, 0x0}, {0x63e0c68, 0xc002480780}}, 0xc0014077a0, {0x6414780, 0xc000842c80}, ...}, ...)
	github.com/cosmos/[email protected]/x/gov/module.go:302 +0x109
github.com/cosmos/cosmos-sdk/types/module.(*Manager).InitGenesis(_, {{0x640e648, 0x8baf600}, {0x642c920, 0xc0023963f0}, {{0x0, 0x0}, {0xc001cc6918, 0x13}, 0x0, ...}, ...}, ...)
	github.com/cosmos/[email protected]/types/module/module.go:505 +0x694

[harshsingh-cs]

@yihuang can you help

[yihuang]

to sync from genesis, you can't start with 1.4 directly, please check out this doc: https://docs.cronos.org/for-node-hosts/running-nodes/cronos-testnet#step-0-notes-on-testnet-network-upgrade