Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions core/blockchain.go
Original file line number Diff line number Diff line change
Expand Up @@ -2011,7 +2011,10 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
// If we are past Byzantium, enable prefetching to pull in trie node paths
// while processing transactions. Before Byzantium the prefetcher is mostly
// useless due to the intermediate root hashing after each transaction.
var witness *stateless.Witness
var (
witness *stateless.Witness
witnessStats *stateless.WitnessStats
)
if bc.chainConfig.IsByzantium(block.Number()) {
// Generate witnesses either if we're self-testing, or if it's the
// only block being inserted. A bit crude, but witnesses are huge,
Expand All @@ -2021,8 +2024,11 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
if err != nil {
return nil, err
}
if bc.cfg.VmConfig.EnableWitnessStats {
witnessStats = stateless.NewWitnessStats()
}
}
statedb.StartPrefetcher("chain", witness)
statedb.StartPrefetcher("chain", witness, witnessStats)
defer statedb.StopPrefetcher()
}

Expand Down Expand Up @@ -2083,6 +2089,7 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
return nil, fmt.Errorf("stateless self-validation receipt root mismatch (cross: %x local: %x)", crossReceiptRoot, block.ReceiptHash())
}
}

xvtime := time.Since(xvstart)
proctime := time.Since(startTime) // processing + validation + cross validation

Expand Down Expand Up @@ -2118,6 +2125,11 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s
if err != nil {
return nil, err
}
// Report the collected witness statistics
if witnessStats != nil {
witnessStats.ReportMetrics()
}

// Update the metrics touched during block commit
accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them
storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them
Expand Down
2 changes: 1 addition & 1 deletion core/state/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ type Trie interface {

// Witness returns a set containing all trie nodes that have been accessed.
// The returned map could be nil if the witness is empty.
Witness() map[string]struct{}
Witness() map[string][]byte

// NodeIterator returns an iterator that returns nodes of the trie. Iteration
// starts at the key after the given start key. And error will be returned
Expand Down
36 changes: 29 additions & 7 deletions core/state/statedb.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ type StateDB struct {
journal *journal

// State witness if cross validation is needed
witness *stateless.Witness
witness *stateless.Witness
witnessStats *stateless.WitnessStats

// Measurements gathered during execution for debugging purposes
AccountReads time.Duration
Expand Down Expand Up @@ -191,12 +192,13 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro
// StartPrefetcher initializes a new trie prefetcher to pull in nodes from the
// state trie concurrently while the state is mutated so that when we reach the
// commit phase, most of the needed data is already hot.
func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness) {
func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness, witnessStats *stateless.WitnessStats) {
// Terminate any previously running prefetcher
s.StopPrefetcher()

// Enable witness collection if requested
s.witness = witness
s.witnessStats = witnessStats

// With the switch to the Proof-of-Stake consensus algorithm, block production
// rewards are now handled at the consensus layer. Consequently, a block may
Expand Down Expand Up @@ -858,9 +860,17 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
continue
}
if trie := obj.getPrefetchedTrie(); trie != nil {
s.witness.AddState(trie.Witness())
witness := trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
} else if obj.trie != nil {
s.witness.AddState(obj.trie.Witness())
witness := obj.trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
}
}
// Pull in only-read and non-destructed trie witnesses
Expand All @@ -874,9 +884,17 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
continue
}
if trie := obj.getPrefetchedTrie(); trie != nil {
s.witness.AddState(trie.Witness())
witness := trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
} else if obj.trie != nil {
s.witness.AddState(obj.trie.Witness())
witness := obj.trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, obj.addrHash)
}
}
}
}
Expand Down Expand Up @@ -942,7 +960,11 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {

// If witness building is enabled, gather the account trie witness
if s.witness != nil {
s.witness.AddState(s.trie.Witness())
witness := s.trie.Witness()
s.witness.AddState(witness)
if s.witnessStats != nil {
s.witnessStats.Add(witness, common.Hash{})
}
}
return hash
}
Expand Down
108 changes: 108 additions & 0 deletions core/stateless/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package stateless

import (
"maps"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/metrics"
)

var (
accountTrieDepthAvg = metrics.NewRegisteredGauge("witness/trie/account/depth/avg", nil)
accountTrieDepthMin = metrics.NewRegisteredGauge("witness/trie/account/depth/min", nil)
accountTrieDepthMax = metrics.NewRegisteredGauge("witness/trie/account/depth/max", nil)

storageTrieDepthAvg = metrics.NewRegisteredGauge("witness/trie/storage/depth/avg", nil)
storageTrieDepthMin = metrics.NewRegisteredGauge("witness/trie/storage/depth/min", nil)
storageTrieDepthMax = metrics.NewRegisteredGauge("witness/trie/storage/depth/max", nil)
)

// depthStats tracks min/avg/max statistics for trie access depths.
type depthStats struct {
totalDepth int64
samples int64
minDepth int64
maxDepth int64
}

// newDepthStats creates a new depthStats with default values.
func newDepthStats() *depthStats {
return &depthStats{minDepth: -1}
}

// add records a new depth sample.
func (d *depthStats) add(n int64) {
if n < 0 {
return
}
d.totalDepth += n
d.samples++

if d.minDepth == -1 || n < d.minDepth {
d.minDepth = n
}
if n > d.maxDepth {
d.maxDepth = n
}
}

// report uploads the collected statistics into the provided gauges.
func (d *depthStats) report(maxGauge, minGauge, avgGauge *metrics.Gauge) {
if d.samples == 0 {
return
}
maxGauge.Update(d.maxDepth)
minGauge.Update(d.minDepth)
avgGauge.Update(d.totalDepth / d.samples)
}

// WitnessStats aggregates statistics for account and storage trie accesses.
type WitnessStats struct {
accountTrie *depthStats
storageTrie *depthStats
}

// NewWitnessStats creates a new WitnessStats collector.
func NewWitnessStats() *WitnessStats {
return &WitnessStats{
accountTrie: newDepthStats(),
storageTrie: newDepthStats(),
}
}

// Add records trie access depths from the given node paths.
// If `owner` is the zero hash, accesses are attributed to the account trie;
// otherwise, they are attributed to the storage trie of that account.
func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) {
if owner == (common.Hash{}) {
for path := range maps.Keys(nodes) {
s.accountTrie.add(int64(len(path)))
}
} else {
for path := range maps.Keys(nodes) {
s.storageTrie.add(int64(len(path)))
}
}
}

// ReportMetrics reports the collected statistics to the global metrics registry.
func (s *WitnessStats) ReportMetrics() {
s.accountTrie.report(accountTrieDepthMax, accountTrieDepthMin, accountTrieDepthAvg)
s.storageTrie.report(storageTrieDepthMax, storageTrieDepthMin, storageTrieDepthAvg)
}
8 changes: 5 additions & 3 deletions core/stateless/witness.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func NewWitness(context *types.Header, chain HeaderReader) (*Witness, error) {
}
headers = append(headers, parent)
}
// Create the wtness with a reconstructed gutted out block
// Create the witness with a reconstructed gutted out block
return &Witness{
context: context,
Headers: headers,
Expand Down Expand Up @@ -88,14 +88,16 @@ func (w *Witness) AddCode(code []byte) {
}

// AddState inserts a batch of MPT trie nodes into the witness.
func (w *Witness) AddState(nodes map[string]struct{}) {
func (w *Witness) AddState(nodes map[string][]byte) {
if len(nodes) == 0 {
return
}
w.lock.Lock()
defer w.lock.Unlock()

maps.Copy(w.State, nodes)
for _, value := range nodes {
w.State[string(value)] = struct{}{}
}
}

// Copy deep-copies the witness object. Witness.Block isn't deep-copied as it
Expand Down
1 change: 1 addition & 0 deletions core/vm/interpreter.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type Config struct {
ExtraEips []int // Additional EIPS that are to be enabled

StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose)
EnableWitnessStats bool // Whether trie access statistics collection is enabled
}

// ScopeContext contains the things that are per-call, such as stack and memory,
Expand Down
2 changes: 1 addition & 1 deletion miner/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase
if err != nil {
return nil, err
}
state.StartPrefetcher("miner", bundle)
state.StartPrefetcher("miner", bundle, nil)
}
// Note the passed coinbase may be different with header.Coinbase.
return &environment{
Expand Down
2 changes: 1 addition & 1 deletion trie/secure_trie.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte {
}

// Witness returns a set containing all trie nodes that have been accessed.
func (t *StateTrie) Witness() map[string]struct{} {
func (t *StateTrie) Witness() map[string][]byte {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume that Gary okay'd this. I wonder what the ram usage impact will be though. We'll have to check that... with metrics 😁

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe Gary changed this so that everything is tracked in stats now, but would be curious to see how much it impacts memory.

return t.trie.Witness()
}

Expand Down
5 changes: 2 additions & 3 deletions trie/tracer.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package trie

import (
"maps"
"slices"
"sync"
)

Expand Down Expand Up @@ -147,11 +146,11 @@ func (t *prevalueTracer) hasList(list [][]byte) []bool {
}

// values returns a list of values of the cached trie nodes.
func (t *prevalueTracer) values() [][]byte {
func (t *prevalueTracer) values() map[string][]byte {
t.lock.RLock()
defer t.lock.RUnlock()

return slices.Collect(maps.Values(t.data))
return maps.Clone(t.data)
}

// reset resets the cached content in the prevalueTracer.
Expand Down
2 changes: 1 addition & 1 deletion trie/transition.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,6 @@ func (t *TransitionTrie) UpdateContractCode(addr common.Address, codeHash common
}

// Witness returns a set containing all trie nodes that have been accessed.
func (t *TransitionTrie) Witness() map[string]struct{} {
func (t *TransitionTrie) Witness() map[string][]byte {
panic("not implemented")
}
12 changes: 2 additions & 10 deletions trie/trie.go
Original file line number Diff line number Diff line change
Expand Up @@ -752,16 +752,8 @@ func (t *Trie) hashRoot() []byte {
}

// Witness returns a set containing all trie nodes that have been accessed.
func (t *Trie) Witness() map[string]struct{} {
values := t.prevalueTracer.values()
if len(values) == 0 {
return nil
}
witness := make(map[string]struct{}, len(values))
for _, val := range values {
witness[string(val)] = struct{}{}
}
return witness
func (t *Trie) Witness() map[string][]byte {
return t.prevalueTracer.values()
}

// Reset drops the referenced root node and cleans all internal state.
Expand Down
2 changes: 1 addition & 1 deletion trie/verkle.go
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,6 @@ func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) {
}

// Witness returns a set containing all trie nodes that have been accessed.
func (t *VerkleTrie) Witness() map[string]struct{} {
func (t *VerkleTrie) Witness() map[string][]byte {
panic("not implemented")
}