Merge AristoTxRef/KvtTxRef with their LayerRef (#3078)

* Merge AristoTxRef/KvtTxRef with their LayerRef

also clean up unused files

* rm unused test_rocksdb

* Fix tests

* lint

Author: jangko

Makefile

@@ -241,7 +241,6 @@ endif
 
 # builds and runs the nimbus test suite
 test: | build deps rocksdb
-	$(ENV_SCRIPT) nim test_rocksdb $(NIM_PARAMS) nimbus.nims
 	$(ENV_SCRIPT) nim test $(NIM_PARAMS) nimbus.nims
 
 test_import: nimbus_execution_client

execution_chain/db/aristo.nim

@@ -1,5 +1,5 @@
 # nimbus-eth1
-# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Copyright (c) 2023-2025 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -19,10 +19,10 @@ export
   aristo_api, aristo_constants
 
 import
-  aristo/aristo_init
+  aristo/aristo_init/memory_only,
+  aristo/aristo_init/init_common
 export
   MemBackendRef,
-  VoidBackendRef,
   finish,
   init
 

execution_chain/db/aristo/aristo_api.nim

@@ -18,7 +18,9 @@ import
   results,
   ./aristo_desc/desc_backend,
   ./aristo_init/memory_db,
-  "."/[aristo_delete, aristo_desc, aristo_fetch, aristo_init, aristo_merge,
+  ./aristo_init/memory_only,
+  ./aristo_init/init_common,
+  "."/[aristo_delete, aristo_desc, aristo_fetch, aristo_merge,
        aristo_part, aristo_path, aristo_persist, aristo_profile, aristo_tx_frame]
 
 export
@@ -379,9 +381,6 @@ proc dup(be: BackendRef): BackendRef =
     when AristoPersistentBackendOk:
       return RdbBackendRef(be).dup
 
-  of BackendVoid:
-    discard
-
 # ------------------------------------------------------------------------------
 # Public API constuctors
 # ------------------------------------------------------------------------------

execution_chain/db/aristo/aristo_check.nim

@@ -17,7 +17,7 @@ import
   eth/common/hashes,
   results,
   ./aristo_walk/persistent,
-  "."/[aristo_desc, aristo_get, aristo_init],
+  "."/[aristo_desc, aristo_get, aristo_init/memory_only],
   ./aristo_check/[check_be, check_top, check_twig]
 
 # ------------------------------------------------------------------------------
@@ -71,9 +71,6 @@ proc checkBE*(
     return MemBackendRef.checkBE db
   of BackendRocksDB:
     return RdbBackendRef.checkBE db
-  of BackendVoid:
-    return VoidBackendRef.checkBE db
-
 
 proc check*(
     db: AristoTxRef;                   # Database

execution_chain/db/aristo/aristo_check/check_be.nim

@@ -17,13 +17,13 @@ import
   stew/interval_set,
   ../../aristo,
   ../aristo_walk/persistent,
-  ".."/[aristo_desc, aristo_get, aristo_layers]
+  ".."/[aristo_desc, aristo_get]
 
 # ------------------------------------------------------------------------------
 # Public functions
 # ------------------------------------------------------------------------------
 
-proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
+proc checkBE*[T: RdbBackendRef|MemBackendRef](
     _: type T;
     db: AristoDbRef;                   # Database, top layer
       ): Result[void,(VertexID,AristoError)] =

execution_chain/db/aristo/aristo_check/check_top.nim

@@ -75,7 +75,7 @@ proc checkTopCommon*(
   let
     kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len
     kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount
-    vTop = db.layer.vTop
+    vTop = db.vTop
   var
     topVid = VertexID(0)
     stoRoots: HashSet[VertexID]

execution_chain/db/aristo/aristo_debug.nim

@@ -25,7 +25,7 @@ import
 # ------------------------------------------------------------------------------
 
 func orDefault(db: AristoTxRef): AristoTxRef =
-  if db.isNil: AristoTxRef(layer: LayerRef()) else: db
+  if db.isNil: AristoTxRef() else: db
 
 # --------------------------
 
@@ -359,7 +359,7 @@ proc ppXMap*(
 
 
 proc ppBalancer(
-    fl: LayerRef;
+    fl: AristoTxRef;
     db: AristoTxRef;
     indent: int;
       ): string =
@@ -434,7 +434,7 @@ proc ppBe[T](be: T; db: AristoTxRef; limit: int; indent: int): string =
     result &= "[]"
 
 proc ppLayer(
-    layer: LayerRef;
+    layer: AristoTxRef;
     db: AristoTxRef;
     vTopOk: bool;
     sTabOk: bool;
@@ -663,7 +663,7 @@ func pp*(wp: VidVtxPair; db: AristoTxRef): string =
 
 
 proc pp*(
-    layer: LayerRef;
+    layer: AristoTxRef;
     db: AristoTxRef;
     indent = 4;
     sTabOk = true,
@@ -679,14 +679,12 @@ proc pp*(
   limit = 100;
   indent = 4;
     ): string =
-  result = db.layer.ppBalancer(db, indent+1) & indent.toPfx
+  result = db.ppBalancer(db, indent+1) & indent.toPfx
   case be.kind:
   of BackendMemory:
     result &= be.MemBackendRef.ppBe(db, limit, indent+1)
   of BackendRocksDB:
     result &= be.RdbBackendRef.ppBe(db, limit, indent+1)
-  of BackendVoid:
-    result &= "<NoBackend>"
 
 proc pp*(
     db: AristoTxRef;

execution_chain/db/aristo/aristo_desc.nim

@@ -41,10 +41,40 @@ export
 type
   AristoTxRef* = ref object
     ## Transaction descriptor
-    db*: AristoDbRef                  ## Database descriptor
-    parent*: AristoTxRef              ## Previous transaction
-    layer*: LayerRef
-    blockNumber*: Opt[uint64] ## Block number set when freezing the frame
+    ##
+    ## Delta layers are stacked implying a tables hierarchy. Table entries on
+    ## a higher level take precedence over lower layer table entries. So an
+    ## existing key-value table entry of a layer on top supersedes same key
+    ## entries on all lower layers. A missing entry on a higher layer indicates
+    ## that the key-value pair might be fond on some lower layer.
+    ##
+    ## A zero value (`nil`, empty hash etc.) is considered am missing key-value
+    ## pair. Tables on the `LayerDelta` may have stray zero key-value pairs for
+    ## missing entries due to repeated transactions while adding and deleting
+    ## entries. There is no need to purge redundant zero entries.
+    ##
+    ## As for `kMap[]` entries, there might be a zero value entriy relating
+    ## (i.e. indexed by the same vertex ID) to an `sMap[]` non-zero value entry
+    ## (of the same layer or a lower layer whatever comes first.) This entry
+    ## is kept as a reminder that the hash value of the `kMap[]` entry needs
+    ## to be re-compiled.
+    ##
+    ## The reasoning behind the above scenario is that every vertex held on the
+    ## `sTab[]` tables must correspond to a hash entry held on the `kMap[]`
+    ## tables. So a corresponding zero value or missing entry produces an
+    ## inconsistent state that must be resolved.
+    db*: AristoDbRef                       ## Database descriptor
+    parent*: AristoTxRef                   ## Previous transaction
+
+    sTab*: Table[RootedVertexID,VertexRef] ## Structural vertex table
+    kMap*: Table[RootedVertexID,HashKey]   ## Merkle hash key mapping
+    vTop*: VertexID                        ## Last used vertex ID
+
+    accLeaves*: Table[Hash32, VertexRef]   ## Account path -> VertexRef
+    stoLeaves*: Table[Hash32, VertexRef]   ## Storage path -> VertexRef
+
+    cTop*: VertexID                        ## Last committed vertex ID
+    blockNumber*: Opt[uint64]              ## Block number set when freezing the frame
 
   AristoDbRef* = ref object
     ## Three tier database object supporting distributed instances.
@@ -120,8 +150,8 @@ func isValid*(nd: NodeRef): bool =
 func isValid*(pid: PathID): bool =
   pid != VOID_PATH_ID
 
-func isValid*(layer: LayerRef): bool =
-  layer != LayerRef(nil)
+func isValid*(tx: AristoTxRef): bool =
+  tx != AristoTxRef(nil)
 
 func isValid*(root: Hash32): bool =
   root != emptyRoot
@@ -163,21 +193,21 @@ iterator stack*(tx: AristoTxRef): AristoTxRef =
   while frames.len > 0:
     yield frames.pop()
 
-iterator rstack*(tx: AristoTxRef): (LayerRef, int) =
+iterator rstack*(tx: AristoTxRef): (AristoTxRef, int) =
   # Stack in reverse order, ie going from tx to base
   var tx = tx
 
   var i = 0
   while tx != nil:
     let level = if tx.parent == nil: -1 else: i
-    yield (tx.layer, level)
+    yield (tx, level)
     tx = tx.parent
 
-proc deltaAtLevel*(db: AristoTxRef, level: int): LayerRef =
+proc deltaAtLevel*(db: AristoTxRef, level: int): AristoTxRef =
   if level == -2:
     nil
   elif level == -1:
-    db.db.txRef.layer
+    db.db.txRef
   else:
     var
       frame = db
@@ -187,7 +217,7 @@ proc deltaAtLevel*(db: AristoTxRef, level: int): LayerRef =
       frame = frame.parent
       level -= 1
 
-    frame.layer
+    frame
 
 # ------------------------------------------------------------------------------
 # End

execution_chain/db/aristo/aristo_desc/desc_error.nim

@@ -12,17 +12,6 @@ type
   AristoError* = enum
     NothingSerious = 0
 
-    # Miscelaneous/unclassified handy helpers
-    GenericError
-
-
-    # Data record transcoders, `blobify()` from `blobify.nim`
-    BlobifyBranchMissingRefs
-    BlobifyExtMissingRefs
-    BlobifyExtPathOverflow
-    BlobifyLeafPathOverflow
-    BlobifyNilVertex
-
 
     # Cache checker `checkCache()`
     CheckAnyVidDeadStorageRoot
@@ -79,17 +68,6 @@ type
     DelStoRootNotAccepted
     DelVidStaleVtx
 
-    # Functions from `aristo_desc.nim`
-    DescMustBeOnCentre
-    DescNotAllowedOnCentre
-    DescStaleDescriptor
-
-
-    # Functions from  `aristo_delta.nim`
-    FilBackendRoMode
-    FilSiblingsCommitUnfinshed
-
-
     # Fetch functions from `aristo_fetch.nim`
     FetchAccInaccessible
     FetchAccPathWithoutLeaf
@@ -205,7 +183,6 @@ type
     PartVtxSlotWasNotModified
 
     # RocksDB backend
-    RdbBeCantCreateDataDir
     RdbBeCantCreateTmpDir
     RdbBeDriverDelAdmError
     RdbBeDriverDelKeyError

execution_chain/db/aristo/aristo_desc/desc_structural.nim

@@ -94,37 +94,6 @@ type
     key*: Hash32                     ## Some state hash (if any)
     serial*: uint64                  ## Generic identifier from application
 
-  LayerRef* = ref Layer
-  Layer* = object
-    ## Delta layers are stacked implying a tables hierarchy. Table entries on
-    ## a higher level take precedence over lower layer table entries. So an
-    ## existing key-value table entry of a layer on top supersedes same key
-    ## entries on all lower layers. A missing entry on a higher layer indicates
-    ## that the key-value pair might be fond on some lower layer.
-    ##
-    ## A zero value (`nil`, empty hash etc.) is considered am missing key-value
-    ## pair. Tables on the `LayerDelta` may have stray zero key-value pairs for
-    ## missing entries due to repeated transactions while adding and deleting
-    ## entries. There is no need to purge redundant zero entries.
-    ##
-    ## As for `kMap[]` entries, there might be a zero value entriy relating
-    ## (i.e. indexed by the same vertex ID) to an `sMap[]` non-zero value entry
-    ## (of the same layer or a lower layer whatever comes first.) This entry
-    ## is kept as a reminder that the hash value of the `kMap[]` entry needs
-    ## to be re-compiled.
-    ##
-    ## The reasoning behind the above scenario is that every vertex held on the
-    ## `sTab[]` tables must correspond to a hash entry held on the `kMap[]`
-    ## tables. So a corresponding zero value or missing entry produces an
-    ## inconsistent state that must be resolved.
-    ##
-    sTab*: Table[RootedVertexID,VertexRef] ## Structural vertex table
-    kMap*: Table[RootedVertexID,HashKey]   ## Merkle hash key mapping
-    vTop*: VertexID                        ## Last used vertex ID
-
-    accLeaves*: Table[Hash32, VertexRef]   ## Account path -> VertexRef
-    stoLeaves*: Table[Hash32, VertexRef]   ## Storage path -> VertexRef
-
   GetVtxFlag* = enum
     PeekCache
       ## Peek into, but don't update cache - useful on work loads that are