UTxO-HD targeting main

.github/workflows/haskell.yml

@@ -97,6 +97,19 @@ jobs:
       with:
         use-sodium-vrf: true # default is true
 
+    - name: Linux install lmdb
+      if: matrix.sys.os == 'ubuntu-latest'
+      run: sudo apt install liblmdb-dev
+
+    - name: Mac install lmdb
+      if: matrix.sys.os == 'macos-13'
+      run: brew install lmdb
+
+    - name: Windows install lmdb
+      if: matrix.sys.os == 'windows-latest'
+      shell: 'C:/msys64/usr/bin/bash.exe -e {0}'
+      run: /usr/bin/pacman --noconfirm -S mingw-w64-x86_64-lmdb
+
     - uses: actions/checkout@v4
 
     - name: Cabal update

.gitignore

@@ -7,9 +7,7 @@
 /cabal.project.old
 configuration/defaults/simpleview/genesis/
 configuration/defaults/liveview/genesis/
-dist-newstyle
-dist-newstyle/
-dist-profiled/
+dist-*
 dist/
 *~
 \#*
@@ -20,12 +18,13 @@ dist/
 result*
 /launch-*
 stack.yaml.lock
+.ghcid
 
 /.cache
 /db
 /db-[0-9]
 /logs
-/mainnet
+/mainnet*
 /profile
 /launch_*
 /state-*

cabal.project

@@ -13,7 +13,7 @@ repository cardano-haskell-packages
 -- See CONTRIBUTING for information about these, including some Nix commands
 -- you need to run if you change them
 index-state:
-  , hackage.haskell.org 2025-01-01T23:24:19Z
+  , hackage.haskell.org 2025-01-14T03:16:27Z
   , cardano-haskell-packages 2025-01-22T16:50:05Z
 
 packages:
@@ -64,3 +64,78 @@ allow-newer:
 -- IMPORTANT
 -- Do NOT add more source-repository-package stanzas here unless they are strictly
 -- temporary! Please read the section in CONTRIBUTING about updating dependencies.
+
+-- UTxO-HD for 10.2
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  tag: d5232aa5d237f9ab6f92cb256180e57d979cf873
+  --sha256: sha256-Y0XGp4+F0j1jFBoDYR3RIvY2LzsHchrmIzLCnRxkSzo=
+  subdir:
+    ouroboros-consensus
+    ouroboros-consensus-cardano
+    ouroboros-consensus-diffusion
+    ouroboros-consensus-protocol
+    sop-extras
+    strict-sop-core
+
+-- mempack support
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-base.git
+  tag: 297efc43b74291a24344ea7ea3b66e91a25af5a8
+  --sha256: sha256-Kv8a3VSQLsqdEEynjyJqxKS8xQjbuqau3lS57GvT21o=
+  subdir:
+    cardano-crypto-class
+
+-- mempack support
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-ledger
+  tag: 5e9799940b05af8b04812bc828b50a4848e17c93
+  --sha256: sha256-G0pz2z1hvg32OBnrrhEgAomAs3liPrOsaslpSGR9nWM=
+  subdir:
+    eras/allegra/impl
+    eras/alonzo/impl
+    eras/alonzo/test-suite
+    eras/babbage/impl
+    eras/babbage/test-suite
+    eras/conway/impl
+    eras/conway/test-suite
+    eras/mary/impl
+    eras/shelley/impl
+    eras/shelley/test-suite
+    eras/shelley-ma/test-suite
+    libs/cardano-ledger-api
+    libs/cardano-ledger-core
+    libs/cardano-ledger-binary
+    libs/cardano-protocol-tpraos
+    libs/non-integral
+    libs/small-steps
+    libs/cardano-data
+    libs/set-algebra
+    libs/vector-map
+    eras/byron/chain/executable-spec
+    eras/byron/ledger/executable-spec
+    eras/byron/ledger/impl
+    eras/byron/ledger/impl/test
+    eras/byron/crypto
+    eras/byron/crypto/test
+
+-- UTxO-HD for 10.2
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-api.git
+  tag: 4605bee8f27017d1dcf6c1fe81530f62617b77ae
+  --sha256: sha256-ukjNDPVbB62a5G7U7TMgvgDJYIqX2wllu+lUXoIrGgM=
+  subdir:
+    cardano-api
+
+-- UTxO-HD for 10.2
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-cli.git
+  tag: d4e549b2481a176a34db7c7c1c5244fcc23507ca
+  --sha256: sha256-EhHoHeKTRrwOJn28wnE8WP9EVrsGlD2AG65SoSCQrco=
+  subdir:
+    cardano-cli

cardano-node/app/cardano-node.hs

@@ -35,8 +35,6 @@ main = do
     case cmd of
       RunCmd args -> do
         warnIfSet args pncMaybeMempoolCapacityOverride "mempool-capacity-override" "MempoolCapacityBytesOverride"
-        warnIfSet args pncNumOfDiskSnapshots "num-of-disk-snapshots" "NumOfDiskSnapshots"
-        warnIfSet args pncSnapshotInterval "snapshot-interval" "SnapshotInterval"
         runNode args
       TraceDocumentation tdc -> runTraceDocumentationCmd tdc
       VersionCmd  -> runVersionCommand
@@ -45,15 +43,15 @@ main = do
       p = Opt.prefs Opt.showHelpOnEmpty
 
       warnIfSet :: PartialNodeConfiguration -> (PartialNodeConfiguration -> Last a) -> String -> String -> IO ()
-      warnIfSet args f name key = 
-          maybe 
-            (pure ()) 
-            (\_ -> hPutStrLn stderr $ "WARNING: Option --" ++ name ++ " was set via CLI flags.\ 
+      warnIfSet args f name key =
+          maybe
+            (pure ())
+            (\_ -> hPutStrLn stderr $ "WARNING: Option --" ++ name ++ " was set via CLI flags.\
             \ This CLI flag will be removed in upcoming node releases.\
-            \ Please, set this configuration option in the configuration file instead with key " ++ key ++ ".") 
+            \ Please, set this configuration option in the configuration file instead with key " ++ key ++ ".")
         $ getLast
         $ f args
-        
+
       opts :: Opt.ParserInfo Command
       opts =
         Opt.info (fmap RunCmd nodeCLIParser

cardano-node/cardano-node.cabal

@@ -70,6 +70,7 @@ library
   exposed-modules:      Cardano.Node.Configuration.Logging
                         Cardano.Node.Configuration.NodeAddress
                         Cardano.Node.Configuration.POM
+                        Cardano.Node.Configuration.LedgerDB
                         Cardano.Node.Configuration.Socket
                         Cardano.Node.Configuration.Topology
                         Cardano.Node.Configuration.TopologyP2P
@@ -206,6 +207,9 @@ library
                       , stm <2.5.2 || >=2.5.3
                       , strict-sop-core
                       , strict-stm
+                      , sop-core
+                      , sop-extras
+                      , text >= 2.0
                       , time
                       , trace-dispatcher ^>= 2.8.0
                       , trace-forward ^>= 2.2.9
@@ -269,7 +273,6 @@ test-suite cardano-node-test
                       , ouroboros-network-api
                       , strict-sop-core
                       , text
-                      , time
                       , transformers
                       , vector
                       , yaml

cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs

@@ -0,0 +1,127 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE GeneralisedNewtypeDeriving #-}
+
+{-# OPTIONS_GHC -Wno-orphans #-}
+
+module Cardano.Node.Configuration.LedgerDB (
+    LedgerDbConfiguration (..)
+  , LedgerDbSelectorFlag(..)
+  , Gigabytes
+  , selectorToArgs
+  ) where
+
+import           Ouroboros.Consensus.Storage.LedgerDB.Args
+import           Ouroboros.Consensus.Storage.LedgerDB.Snapshots
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Args as V1
+import           Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB (LMDBLimits (..))
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Args as V2
+import           Ouroboros.Consensus.Util.Args
+
+import qualified Data.Aeson.Types as Aeson (FromJSON)
+import           Data.Maybe (fromMaybe)
+import           Data.SOP.Dict
+
+-- | Choose the LedgerDB Backend
+--
+-- As of UTxO-HD, the LedgerDB now uses either an in-memory backend or LMDB to
+-- keep track of differences in the UTxO set.
+--
+-- - 'V2InMemory': uses more memory than the minimum requirements but is somewhat
+--   faster.
+--
+-- - 'V1LMDB': uses less memory but is somewhat slower.
+--
+-- - 'V1InMemory': Not intended for production. It is an in-memory reproduction
+--   of the LMDB implementation.
+data LedgerDbSelectorFlag =
+    V1LMDB
+      V1.FlushFrequency
+      -- ^ The frequency at which changes are flushed to the disk.
+      (Maybe FilePath)
+      -- ^ Path for the live tables.
+      (Maybe Gigabytes)
+      -- ^ A map size can be specified, this is the maximum disk space the LMDB
+      -- database can fill. If not provided, the default of 16GB will be used.
+  | V1InMemory V1.FlushFrequency
+  | V2InMemory
+  deriving (Eq, Show)
+
+data LedgerDbConfiguration =
+    LedgerDbConfiguration
+      NumOfDiskSnapshots
+      SnapshotInterval
+      QueryBatchSize
+      LedgerDbSelectorFlag
+      (Flag "DoDiskSnapshotChecksum")
+  deriving (Eq, Show)
+
+-- | A number of gigabytes.
+newtype Gigabytes = Gigabytes Int
+  deriving stock (Eq, Show)
+  deriving newtype (Read, Aeson.FromJSON)
+
+-- | Convert a number of Gigabytes to the equivalent number of bytes.
+toBytes :: Gigabytes -> Int
+toBytes (Gigabytes x) = x * 1024 * 1024 * 1024
+
+-- | Recommended settings for the LMDB backing store.
+--
+-- === @'lmdbMapSize'@
+-- The default @'LMDBLimits'@ uses an @'lmdbMapSize'@ of @1024 * 1024 * 1024 * 16@
+-- bytes, or 16 Gigabytes. @'lmdbMapSize'@ sets the size of the memory map
+-- that is used internally by the LMDB backing store, and is also the
+-- maximum size of the on-disk database. 16 GB should be sufficient for the
+-- medium term, i.e., it is sufficient until a more performant alternative to
+-- the LMDB backing store is implemented, which will probably replace the LMDB
+-- backing store altogether.
+--
+-- Note(jdral): It is recommended not to set the @'lmdbMapSize'@ to a value
+-- that is much smaller than 16 GB through manual configuration: the node will
+-- die with a fatal error as soon as the database size exceeds the
+-- @'lmdbMapSize'@. If this fatal error were to occur, we would expect that
+-- the node can continue normal operation if it is restarted with a higher
+-- @'lmdbMapSize'@ configured. Nonetheless, this situation should be avoided.
+--
+-- === @'lmdbMaxDatabases'@
+-- The @'lmdbMaxDatabases'@ is set to 10, which means that the LMDB backing
+-- store will allow up @<= 10@ internal databases. We say /internal/
+-- databases, since they are not exposed outside the backing store interface,
+-- such that from the outside view there is just one /logical/ database.
+-- Two of these internal databases are reserved for normal operation of the
+-- backing store, while the remaining databases will be used to store ledger
+-- tables. At the moment, there is at most one ledger table that will be
+-- stored in an internal database: the UTxO. Nonetheless, we set
+-- @'lmdbMaxDatabases'@ to @10@ in order to future-proof these limits.
+--
+-- === @'lmdbMaxReaders'@
+-- The @'lmdbMaxReaders'@ limit sets the maximum number of threads that can
+-- read from the LMDB database. Currently, there should only be a single reader
+-- active. Again, we set @'lmdbMaxReaders'@ to @16@ in order to future-proof
+-- these limits.
+--
+-- === References
+-- For more information about LMDB limits, one should inspect:
+-- * The @lmdb-simple@ and @haskell-lmdb@ forked repositories.
+-- * The official LMDB API documentation at
+--    <http://www.lmdb.tech/doc/group__mdb.html>.
+defaultLMDBLimits :: LMDBLimits
+defaultLMDBLimits = LMDBLimits {
+    lmdbMapSize = 16 * 1024 * 1024 * 1024
+  , lmdbMaxDatabases = 10
+  , lmdbMaxReaders = 16
+  }
+
+defaultLMDBPath :: FilePath
+defaultLMDBPath = "mainnet/db/lmdb"
+
+selectorToArgs :: LedgerDbSelectorFlag -> Complete LedgerDbFlavorArgs IO
+selectorToArgs (V1InMemory a)  = LedgerDbFlavorArgsV1 $ V1.V1Args a V1.InMemoryBackingStoreArgs
+selectorToArgs V2InMemory      = LedgerDbFlavorArgsV2 $ V2.V2Args V2.InMemoryHandleArgs
+selectorToArgs (V1LMDB a fp l) =
+    LedgerDbFlavorArgsV1
+  $ V1.V1Args a
+  $ V1.LMDBBackingStoreArgs
+      (fromMaybe defaultLMDBPath fp)
+      (maybe id (\ll lim -> lim { lmdbMapSize = toBytes ll }) l defaultLMDBLimits)
+      Dict