Update to LSM

jasagredo · jasagredo · commit 4926d76484df · 2025-10-31T12:16:17.000+01:00
diff --git a/cabal.project b/cabal.project
@@ -13,7 +13,7 @@ repository cardano-haskell-packages
 -- See CONTRIBUTING for information about these, including some Nix commands
 -- you need to run if you change them
 index-state:
-  , hackage.haskell.org 2025-10-17T00:26:22Z
+  , hackage.haskell.org 2025-10-23T13:39:53Z
   , cardano-haskell-packages 2025-10-24T13:50:12Z
 
 packages:
@@ -61,14 +61,70 @@ package plutus-scripts-bench
 allow-newer:
   , katip:Win32
 
-if impl (ghc >= 9.12)
-  allow-newer:
-    -- https://github.com/kapralVV/Unique/issues/11
-    , Unique:hashable
-
-    -- https://github.com/Gabriella439/Haskell-Pipes-Safe-Library/pull/70
-    , pipes-safe:base
-
 -- IMPORTANT
 -- Do NOT add more source-repository-package stanzas here unless they are strictly
 -- temporary! Please read the section in CONTRIBUTING about updating dependencies.
+source-repository-package
+  type: git
+  location: https://github.com/intersectmbo/cardano-cli.git
+  tag: 9dbc9cfa1a862386f854c1eceb0305695c4784a2
+  --sha256: sha256-IrgvEB1299aldN/BPdabljxyf4JjrwzKcO6yuO87E20=
+  subdir: cardano-cli
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-api
+  tag: 565fb92f5e0653cb8f740bd8f150283dd2760c4a
+  --sha256: sha256-quT9DcwFqJM50PHtsSfQ88EE62HeajTk7QBWq0Ld7KY=
+  subdir:
+    cardano-api
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  tag: 414f51d729de85c4c867781503d862a1f92bbc15
+  --sha256: sha256-zUMDepYkkLeIqFMa/bN3senAei/idOSqnjv1QCDqjhQ=
+  subdir:
+    ouroboros-consensus
+    ouroboros-consensus-cardano
+    ouroboros-consensus-diffusion
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-ledger
+  tag: fb09078fa55015c881303a2ddb609c024cec258f
+  --sha256: sha256-9Y9CRiyMn0AWD+C4aNVMaJgrj3FDAYfCX4VrLvtoMaI=
+  subdir:
+    eras/allegra/impl
+    eras/alonzo/impl
+    eras/alonzo/test-suite
+    eras/babbage/impl
+    eras/conway/impl
+    eras/dijkstra/impl
+    eras/mary/impl
+    eras/shelley/impl
+    eras/shelley/test-suite
+    eras/shelley-ma/test-suite
+    libs/cardano-ledger-api
+    libs/cardano-ledger-core
+    libs/cardano-ledger-binary
+    libs/cardano-protocol-tpraos
+    libs/non-integral
+    libs/small-steps
+    libs/cardano-data
+    libs/set-algebra
+    libs/vector-map
+    eras/byron/chain/executable-spec
+    eras/byron/ledger/executable-spec
+    eras/byron/ledger/impl
+    eras/byron/crypto
+
+-- Backported version of https://github.com/IntersectMBO/ouroboros-network/pull/5161
+source-repository-package
+   type: git
+   location: https://github.com/IntersectMBO/ouroboros-network
+   tag: 1385b53cefb81e79553b6b0252537455833ea9c4
+   --sha256: sha256-zZ7WsMfRs1fG16bmvI5vIh4fhQ8RGyEvYGLSWlrxpg0=
+   subdir:
+     ouroboros-network-api
+     ouroboros-network
diff --git a/cardano-node-chairman/cardano-node-chairman.cabal b/cardano-node-chairman/cardano-node-chairman.cabal
@@ -44,7 +44,7 @@ executable cardano-node-chairman
   build-depends:        cardano-api
                       , cardano-crypto-class
                       , cardano-git-rev ^>= 0.2.2
-                      , cardano-ledger-core ^>= 1.18
+                      , cardano-ledger-core ^>= 1.19
                       , cardano-node ^>= 10.5
                       , cardano-prelude
                       , containers
diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal
@@ -188,7 +188,7 @@ library
                       , network-mux >= 0.8
                       , nothunks
                       , optparse-applicative-fork >= 0.18.1
-                      , ouroboros-consensus ^>= 0.28
+                      , ouroboros-consensus:{ouroboros-consensus, ouroboros-consensus-lmdb, ouroboros-consensus-lsm} ^>= 0.28
                       , ouroboros-consensus-cardano ^>= 0.26
                       , ouroboros-consensus-diffusion ^>= 0.24
                       , ouroboros-consensus-protocol
diff --git a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs
@@ -1,28 +1,35 @@
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE GeneralisedNewtypeDeriving #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeApplications #-}
 
 {-# OPTIONS_GHC -Wno-orphans #-}
 
 module Cardano.Node.Configuration.LedgerDB (
-    DeprecatedOptions (..)
-  , LedgerDbConfiguration (..)
-  , LedgerDbSelectorFlag(..)
-  , Gigabytes
-  , noDeprecatedOptions
-  , selectorToArgs
-  ) where
+    DeprecatedOptions (..),
+    LedgerDbConfiguration (..),
+    LedgerDbSelectorFlag (..),
+    Gigabytes,
+    noDeprecatedOptions,
+    selectorToArgs,
+) where
 
+import           Ouroboros.Consensus.Ledger.SupportsProtocol
+import           Ouroboros.Consensus.Storage.LedgerDB.API
 import           Ouroboros.Consensus.Storage.LedgerDB.Args
 import           Ouroboros.Consensus.Storage.LedgerDB.Snapshots
 import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Args as V1
-import           Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB (LMDBLimits (..))
-import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Args as V2
-import           Ouroboros.Consensus.Util.Args
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB as LMDB
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM
 
 import qualified Data.Aeson.Types as Aeson (FromJSON)
 import           Data.Maybe (fromMaybe)
-import           Data.SOP.Dict
+import           Data.Proxy
+import           System.FilePath
+import           System.Random (StdGen)
 
 -- | Choose the LedgerDB Backend
 --
@@ -34,21 +41,25 @@ import           Data.SOP.Dict
 --
 -- - 'V1LMDB': uses less memory but is somewhat slower.
 --
--- - 'V1InMemory': Not intended for production. It is an in-memory reproduction
---   of the LMDB implementation.
+-- - 'V2LSM': Uses the LSM backend.
 data LedgerDbSelectorFlag =
     V1LMDB
       V1.FlushFrequency
       -- ^ The frequency at which changes are flushed to the disk.
       (Maybe FilePath)
-      -- ^ Path for the live tables.
+      -- ^ Path for the live tables. If not provided the default will be used
+      -- (@<fast-storage>/lmdb@).
       (Maybe Gigabytes)
       -- ^ A map size can be specified, this is the maximum disk space the LMDB
       -- database can fill. If not provided, the default of 16GB will be used.
       (Maybe Int)
       -- ^ An override to the max number of readers.
-  | V1InMemory V1.FlushFrequency
   | V2InMemory
+  | V2LSM
+      (Maybe FilePath)
+      -- ^ Maybe a custom path to the LSM database. If not provided the default
+      -- will be used (@<fast-storage>/lsm@).
+
   deriving (Eq, Show)
 
 -- | Some options that existed in the TopLevel were now moved to a
@@ -118,24 +129,23 @@ toBytes (Gigabytes x) = x * 1024 * 1024 * 1024
 -- * The @lmdb-simple@ and @haskell-lmdb@ forked repositories.
 -- * The official LMDB API documentation at
 --    <http://www.lmdb.tech/doc/group__mdb.html>.
-defaultLMDBLimits :: LMDBLimits
-defaultLMDBLimits = LMDBLimits {
-    lmdbMapSize = 16 * 1024 * 1024 * 1024
-  , lmdbMaxDatabases = 10
-  , lmdbMaxReaders = 16
+defaultLMDBLimits :: LMDB.LMDBLimits
+defaultLMDBLimits = LMDB.LMDBLimits {
+    LMDB.lmdbMapSize = 16 * 1024 * 1024 * 1024
+  , LMDB.lmdbMaxDatabases = 10
+  , LMDB.lmdbMaxReaders = 16
   }
 
-defaultLMDBPath :: FilePath
-defaultLMDBPath = "mainnet/db/lmdb"
+defaultLMDBPath :: FilePath -> FilePath
+defaultLMDBPath = (</> "lmdb")
 
-selectorToArgs :: LedgerDbSelectorFlag -> Complete LedgerDbFlavorArgs IO
-selectorToArgs (V1InMemory ff)  = LedgerDbFlavorArgsV1 $ V1.V1Args ff V1.InMemoryBackingStoreArgs
-selectorToArgs V2InMemory      = LedgerDbFlavorArgsV2 $ V2.V2Args V2.InMemoryHandleArgs
-selectorToArgs (V1LMDB ff fp l mxReaders) =
-    LedgerDbFlavorArgsV1
-  $ V1.V1Args ff
-  $ V1.LMDBBackingStoreArgs
-      (fromMaybe defaultLMDBPath fp)
-      (maybe id (\overrideMaxReaders lim -> lim { lmdbMaxReaders = overrideMaxReaders }) mxReaders
-       $ maybe id (\ll lim -> lim { lmdbMapSize = toBytes ll }) l defaultLMDBLimits)
-      Dict
+selectorToArgs :: forall blk. (LedgerSupportsProtocol blk, LedgerSupportsLedgerDB blk) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen)
+selectorToArgs V2InMemory _ = InMemory.mkInMemoryArgs
+selectorToArgs (V1LMDB ff fp l mxReaders) fastStoragePath =
+    LMDB.mkLMDBArgs
+        ff
+        (fromMaybe (defaultLMDBPath fastStoragePath) fp)
+        ( maybe id (\overrideMaxReaders lim -> lim{LMDB.lmdbMaxReaders = overrideMaxReaders}) mxReaders $
+            maybe id (\ll lim -> lim{LMDB.lmdbMapSize = toBytes ll}) l defaultLMDBLimits
+        )
+selectorToArgs (V2LSM fp) fastStoragePath = LSM.mkLSMArgs (Proxy @blk) (fromMaybe "lsm" fp) fastStoragePath
diff --git a/cardano-node/src/Cardano/Node/Configuration/POM.hs b/cardano-node/src/Cardano/Node/Configuration/POM.hs
@@ -466,16 +466,16 @@ instance FromJSON PartialNodeConfiguration where
              qsize           <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize
              backend         <- o .:? "Backend" .!= "V2InMemory"
              selector        <- case backend of
-               "V1InMemory" -> do
-                 flush <- (fmap RequestedFlushFrequency <$> o .:? "FlushFrequency")       .!= DefaultFlushFrequency
-                 return $ V1InMemory flush
                "V1LMDB"     -> do
                  flush <- (fmap RequestedFlushFrequency <$> o .:? "FlushFrequency")       .!= DefaultFlushFrequency
                  mapSize :: Maybe Gigabytes <- o .:? "MapSize"
                  lmdbPath :: Maybe FilePath <- o .:? "LiveTablesPath"
                  mxReaders :: Maybe Int <- o .:? "MaxReaders"
                  return $ V1LMDB flush lmdbPath mapSize mxReaders
                "V2InMemory" -> return V2InMemory
+               "V2LSM" -> do
+                 lsmPath :: Maybe FilePath <- o .:? "LSMDatabasePath"
+                 pure $ V2LSM lsmPath
                _ -> fail $ "Malformed LedgerDB Backend: " <> backend
              pure $ Just $ LedgerDbConfiguration ldbSnapNum ldbSnapInterval qsize selector deprecatedOpts
 
diff --git a/cardano-node/src/Cardano/Node/Run.hs b/cardano-node/src/Cardano/Node/Run.hs
@@ -68,7 +68,7 @@ import           Cardano.Tracing.Tracers
 import qualified Ouroboros.Consensus.Config as Consensus
 import           Ouroboros.Consensus.Config.SupportsNode (ConfigSupportsNode (..))
 import           Ouroboros.Consensus.Node (SnapshotPolicyArgs (..),
-                   NodeDatabasePaths (..), RunNodeArgs (..), StdRunNodeArgs (..))
+                   NodeDatabasePaths (..), nonImmutableDbPath, RunNodeArgs (..), StdRunNodeArgs (..))
 import           Ouroboros.Consensus.Protocol.Praos.AgentClient (KESAgentClientTrace)
 import           Ouroboros.Consensus.Ledger.SupportsMempool (GenTxId)
 import           Ouroboros.Consensus.Node (RunNodeArgs (..),
@@ -79,7 +79,6 @@ import           Ouroboros.Consensus.Node.NetworkProtocolVersion
 import           Ouroboros.Consensus.Node.ProtocolInfo
 import qualified Ouroboros.Consensus.Node.Tracers as Consensus
 import qualified Ouroboros.Consensus.Storage.LedgerDB.Args as LDBArgs
-import           Ouroboros.Consensus.Storage.LedgerDB.V2.Args
 import           Ouroboros.Consensus.Util.Args
 import           Ouroboros.Consensus.Util.Orphans ()
 
@@ -560,7 +559,7 @@ handleSimpleNode blockType runP tracers nc onKernel = do
           , srnChainSyncIdleTimeout         = customizeChainSyncTimeout
           , srnSnapshotPolicyArgs           = snapshotPolicyArgs
           , srnQueryBatchSize               = queryBatchSize
-          , srnLdbFlavorArgs                = selectorToArgs ldbBackend
+          , srnLedgerDbBackendArgs          = selectorToArgs ldbBackend (nonImmutableDbPath dbPath)
           }
  where
   customizeChainSyncTimeout :: ChainSyncIdleTimeout
diff --git a/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs b/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs
@@ -1073,6 +1073,8 @@ instance
   , LogFormatting (PredicateFailure (Ledger.EraRule "CERTS" era))
   ) => LogFormatting (Conway.ConwayLedgerPredFailure era) where
   forMachine v (Conway.ConwayUtxowFailure f) = forMachine v f
+  forMachine _verb (Conway.ConwayWithdrawalsMissingAccounts _) = mconcat [ "todo" .= String "todo"]
+  forMachine _verb (Conway.ConwayIncompleteWithdrawals _) = mconcat [ "todo" .= String "todo"]
   forMachine _ (Conway.ConwayTxRefScriptsSizeTooBig  Mismatch {mismatchSupplied, mismatchExpected}) =
     mconcat [ "kind" .= String "ConwayTxRefScriptsSizeTooBig"
             , "actual" .= mismatchSupplied
diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs
diff --git a/cardano-node/src/Cardano/Tracing/OrphanInstances/Consensus.hs b/cardano-node/src/Cardano/Tracing/OrphanInstances/Consensus.hs
diff --git a/cardano-node/src/Cardano/Tracing/OrphanInstances/Shelley.hs b/cardano-node/src/Cardano/Tracing/OrphanInstances/Shelley.hs
