Delete readAll functionality in LedgerDB backends

Author: jasagredo

ouroboros-consensus/changelog.d/20250904_111212_javier.sagredo_lsm_sc_da.md

@@ -17,9 +17,7 @@ For top level release notes, leave all the headers commented out.
 - A bullet item for the Non-Breaking category.
 
 -->
-<!--
-### Breaking
 
-- A bullet item for the Breaking category.
+### Breaking
 
--->
+- Removed the `readAll` functionality from the LedgerDB backends now that it became unnecessary as tables conversion is now streamed.

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/V1/BackingStore/API.hs

@@ -54,7 +54,6 @@ module Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.API
 
     -- * 🧪 Testing
   , bsRead
-  , bsReadAll
   ) where
 
 import Cardano.Slotting.Slot (SlotNo, WithOrigin (..))
@@ -169,9 +168,6 @@ data BackingStoreValueHandle m keys key values = BackingStoreValueHandle
   -- itself is idempotent.
   , bsvhRangeRead :: !(ReadHint values -> RangeQuery keys -> m (values, Maybe key))
   -- ^ See 'RangeQuery'
-  , bsvhReadAll :: !(ReadHint values -> m values)
-  -- ^ Costly read all operation, not to be used in Consensus but only in
-  -- snapshot-converter executable.
   , bsvhRead :: !(ReadHint values -> keys -> m values)
   -- ^ Read the given keys from the handle
   --
@@ -206,7 +202,6 @@ castBackingStoreValueHandle f g h bsvh =
   BackingStoreValueHandle
     { bsvhAtSlot
     , bsvhClose
-    , bsvhReadAll = \rhint -> f <$> bsvhReadAll rhint
     , bsvhRangeRead = \rhint (RangeQuery prev count) ->
         fmap (second (fmap h) . first f) . bsvhRangeRead rhint $ RangeQuery (fmap g prev) count
     , bsvhRead = \rhint -> fmap f . bsvhRead rhint . g
@@ -215,7 +210,6 @@ castBackingStoreValueHandle f g h bsvh =
  where
   BackingStoreValueHandle
     { bsvhClose
-    , bsvhReadAll
     , bsvhAtSlot
     , bsvhRangeRead
     , bsvhRead
@@ -233,13 +227,6 @@ bsRead store rhint keys = withBsValueHandle store $ \vh -> do
   values <- bsvhRead vh rhint keys
   pure (bsvhAtSlot vh, values)
 
-bsReadAll ::
-  MonadThrow m =>
-  BackingStore m keys key values diff ->
-  ReadHint values ->
-  m values
-bsReadAll store rhint = withBsValueHandle store $ \vh -> bsvhReadAll vh rhint
-
 -- | A 'IOLike.bracket'ed 'bsValueHandle'
 withBsValueHandle ::
   MonadThrow m =>

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/V1/BackingStore/Impl/InMemory.hs

@@ -184,11 +184,6 @@ newInMemoryBackingStore tracer (SnapshotsFS (SomeHasFS fs)) initialization = do
                           pure $ rangeRead rq values
                         traceWith tracer $ BSValueHandleTrace Nothing BSVHRangeRead
                         pure r
-                    , bsvhReadAll = \_ ->
-                        atomically $ do
-                          guardClosed ref
-                          guardHandleClosed refHandleClosed
-                          pure values
                     , bsvhRead = \_ keys -> do
                         traceWith tracer $ BSValueHandleTrace Nothing BSVHReading
                         r <- atomically $ do

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/V1/BackingStore/Impl/LMDB.hs

@@ -9,7 +9,6 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE PatternSynonyms #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE TypeOperators #-}
 
 -- | A 'BackingStore' implementation based on [LMDB](http://www.lmdb.tech/doc/).
@@ -42,7 +41,6 @@ import Data.Functor.Contravariant ((>$<))
 import Data.Map (Map)
 import qualified Data.Map.Strict as Map
 import Data.MemPack
-import Data.Proxy
 import qualified Data.Set as Set
 import qualified Data.Text as Strict
 import qualified Database.LMDB.Simple as LMDB
@@ -158,19 +156,6 @@ getDb ::
   LMDB.Transaction mode (LMDBMK k v)
 getDb (K2 name) = LMDBMK name <$> LMDB.getDatabase (Just name)
 
-readAll ::
-  (Ord (TxIn l), MemPack (TxIn l), IndexedMemPack idx (TxOut l)) =>
-  Proxy l ->
-  idx ->
-  LMDBMK (TxIn l) (TxOut l) ->
-  LMDB.Transaction mode (ValuesMK (TxIn l) (TxOut l))
-readAll _ st (LMDBMK _ dbMK) =
-  ValuesMK
-    <$> Bridge.runCursorAsTransaction'
-      st
-      LMDB.Cursor.cgetAll
-      dbMK
-
 -- | @'rangeRead' rq dbMK@ performs a range read of @rqCount rq@
 -- values from database @dbMK@, starting from some key depending on @rqPrev rq@.
 --
@@ -659,25 +644,11 @@ mkLMDBBackingStoreValueHandle db = do
           Trace.traceWith tracer API.BSVHStatted
           pure res
 
-    bsvhReadAll :: l EmptyMK -> m (LedgerTables l ValuesMK)
-    bsvhReadAll st =
-      Status.withReadAccess dbStatusLock (throwIO LMDBErrClosed) $ do
-        Status.withReadAccess vhStatusLock (throwIO (LMDBErrNoValueHandle vhId)) $ do
-          Trace.traceWith tracer API.BSVHRangeReading
-          res <-
-            liftIO $
-              TrH.submitReadOnly trh $
-                let dbMK = getLedgerTables dbBackingTables
-                 in LedgerTables <$> readAll (Proxy @l) st dbMK
-          Trace.traceWith tracer API.BSVHRangeRead
-          pure res
-
     bsvh =
       API.BackingStoreValueHandle
         { API.bsvhAtSlot = initSlot
         , API.bsvhClose = bsvhClose
         , API.bsvhRead = bsvhRead
-        , API.bsvhReadAll = bsvhReadAll
         , API.bsvhRangeRead = bsvhRangeRead
         , API.bsvhStat = bsvhStat
         }

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/V2/InMemory.hs

@@ -122,9 +122,6 @@ newInMemoryLedgerTablesHandle tracer someFS@(SomeHasFS hasFS) l = do
                 let m' = Map.take t . (maybe id (\g -> snd . Map.split g) f) $ m
                  in pure (LedgerTables (ValuesMK m'), fst <$> Map.lookupMax m')
             )
-      , readAll = \_ -> do
-          hs <- readTVarIO tv
-          guardClosed hs pure
       , pushDiffs = \st0 !diffs ->
           atomically $
             modifyTVar

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/V2/LSM.hs

@@ -283,11 +283,6 @@ newLSMLedgerTablesHandle tracer rr (resKey, t) = do
               Map.empty
             $ V.zip vec' res
       , readRange = implReadRange t
-      , readAll = \st ->
-          let readAll' m = do
-                (v, n) <- implReadRange t st (m, 100000)
-                maybe (pure v) (fmap (ltliftA2 unionValues v) . readAll' . Just) n
-           in readAll' Nothing
       , pushDiffs = const (implPushDiffs t)
       , takeHandleSnapshot = \_ snapshotName -> do
           LSM.saveSnapshot

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Storage/LedgerDB/V2/LedgerSeq.hs

@@ -106,10 +106,6 @@ data LedgerTablesHandle m l = LedgerTablesHandle
   -- back into the next iteration of the range read. If the function returns
   -- Nothing, it means the read returned no results, or in other words, we
   -- reached the end of the ledger tables.
-  , readAll :: !(l EmptyMK -> m (LedgerTables l ValuesMK))
-  -- ^ Costly read all operation, not to be used in Consensus but only in
-  -- snapshot-converter executable. The values will be read as if they were from
-  -- the same era as the given ledger state.
   , pushDiffs :: !(forall mk. l mk -> l DiffMK -> m ())
   -- ^ Push some diffs into the ledger tables handle.
   --