Block sync significantly slowed down due to BlockValidatorUpdates #1844
Description
In the last few days I have observed that my nodes frequently not catching up with the chain.
While not catching up, CPU usage stays at 100% of a single core. Disk I/O and network are under-utilized.
Pprof shows ~81% cpu time are spent on the following method:
1 @ 0x530d0b 0x1071745 0x10a0605 0x10a0551 0x10c7a4a 0x10c7405 0x10c427e 0x154894c 0x153d287 0x152820d 0x1527f65 0x1541d67 0x152d585 0x154a86f 0x151e63b 0x152018c 0x11a2cc2 0x11a4cac 0x1b590bb 0x26fed5f 0x26fef4e 0x26f4d34 0x26c2b9e 0x2758a5d 0x1958096 0x33a94fd 0x190c9b5 0x19020d5 0x1902aa5 0x1b0844c 0x129b1c7 0x129d53e 0x169b545 0x1732e14 0x1732ca6 0x1732168 0x1735232 0x172a747 0x172a031 0x540d21
# 0x1071744 github.com/linxGnu/grocksdb._Cfunc_rocksdb_iter_seek+0x44 _cgo_gotypes.go:3606
# 0x10a0604 github.com/linxGnu/grocksdb.(*Iterator).Seek.func1+0x64 github.com/linx!gnu/grocksdb@v1.9.7/iterator.go:108
# 0x10a0550 github.com/linxGnu/grocksdb.(*Iterator).Seek+0x30 github.com/linx!gnu/grocksdb@v1.9.7/iterator.go:108
# 0x10c7a49 github.com/cosmos/cosmos-db.newRocksDBIterator+0xc9 github.com/cosmos/cosmos-db@v1.0.3-0.20240408151834-e75f6e4b28d8/rocksdb_iterator.go:40
# 0x10c7404 github.com/cosmos/cosmos-db.(*RocksDB).Iterator+0xa4 github.com/cosmos/cosmos-db@v1.0.3-0.20240408151834-e75f6e4b28d8/rocksdb.go:215
# 0x10c427d github.com/cosmos/cosmos-db.(*PrefixDB).Iterator+0x2dd github.com/cosmos/cosmos-db@v1.0.3-0.20240408151834-e75f6e4b28d8/prefixdb.go:112
# 0x154894b cosmossdk.io/store/wrapper.(*DBWrapper).Iterator+0x2b github.com/crypto-org-chain/cosmos-sdk/store@v0.0.0-20241205052910-142c0afa4948/wrapper/wrapper.go:29
# 0x153d286 github.com/cosmos/iavl.(*nodeDB).getFastIterator+0x146 github.com/cosmos/iavl@v1.2.2/nodedb.go:1023
# 0x152820c github.com/cosmos/iavl.(*FastIterator).Next+0x4c github.com/cosmos/iavl@v1.2.2/fast_iterator.go:105
# 0x1527f64 github.com/cosmos/iavl.NewFastIterator+0xe4 github.com/cosmos/iavl@v1.2.2/fast_iterator.go:45
# 0x1541d66 github.com/cosmos/iavl.NewUnsavedFastIterator+0x66 github.com/cosmos/iavl@v1.2.2/unsaved_fast_iterator.go:53
# 0x152d584 github.com/cosmos/iavl.(*MutableTree).Iterator+0x104 github.com/cosmos/iavl@v1.2.2/mutable_tree.go:242
# 0x154a86e cosmossdk.io/store/iavl.(*Store).Iterator+0x2e github.com/crypto-org-chain/cosmos-sdk/store@v0.0.0-20241205052910-142c0afa4948/iavl/store.go:248
# 0x151e63a cosmossdk.io/store/cachekv.(*GStore[...]).iterator+0xba github.com/crypto-org-chain/cosmos-sdk/store@v0.0.0-20241205052910-142c0afa4948/cachekv/store.go:153
# 0x152018b cosmossdk.io/store/cachekv.(*GStore[...]).Iterator+0x4b github.com/crypto-org-chain/cosmos-sdk/store@v0.0.0-20241205052910-142c0afa4948/cachekv/store.go:136
# 0x11a2cc1 cosmossdk.io/store/gaskv.(*GStore[...]).iterator+0x61 github.com/crypto-org-chain/cosmos-sdk/store@v0.0.0-20241205052910-142c0afa4948/gaskv/store.go:121
# 0x11a4cab cosmossdk.io/store/gaskv.(*GStore[...]).Iterator+0x4b github.com/crypto-org-chain/cosmos-sdk/store@v0.0.0-20241205052910-142c0afa4948/gaskv/store.go:102
# 0x1b590ba github.com/cosmos/cosmos-sdk/runtime.coreKVStore.Iterator+0x3a github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/runtime/store.go:89
# 0x26fed5e github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.ValidatorQueueIterator+0xde github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/x/staking/keeper/validator.go:536
# 0x26fef4d github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.UnbondAllMatureValidators+0x16d github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/x/staking/keeper/validator.go:551
# 0x26f4d33 github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.BlockValidatorUpdates+0xb3 github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/x/staking/keeper/val_state_change.go:37
# 0x26c2b9d github.com/cosmos/cosmos-sdk/x/staking/keeper.(*Keeper).EndBlocker+0x13d github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/x/staking/keeper/abci.go:22
# 0x2758a5c github.com/cosmos/cosmos-sdk/x/staking.AppModule.EndBlock+0x5c github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/x/staking/module.go:187
# 0x1958095 github.com/cosmos/cosmos-sdk/types/module.(*Manager).EndBlock+0x3b5 github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/types/module/module.go:803
# 0x33a94fc github.com/crypto-org-chain/cronos/v2/app.(*App).EndBlocker+0x7c cro/app/app.go:1201
# 0x190c9b4 github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).endBlock+0xf4 github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/baseapp/baseapp.go:820
# 0x19020d4 github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).internalFinalizeBlock+0x1454 github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/baseapp/abci.go:830
# 0x1902aa4 github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).FinalizeBlock+0x164 github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/baseapp/abci.go:934
# 0x1b0844b github.com/cosmos/cosmos-sdk/server.cometABCIWrapper.FinalizeBlock+0x2b github.com/crypto-org-chain/cosmos-sdk@v0.43.0-beta1.0.20241205052910-142c0afa4948/server/cmt_abci.go:44
# 0x129b1c6 github.com/cometbft/cometbft/abci/client.(*localClient).FinalizeBlock+0xc6 cometbft/abci/client/local_client.go:189
# 0x129d53d github.com/cometbft/cometbft/proxy.(*appConnConsensus).FinalizeBlock+0x15d cometbft/proxy/app_conn.go:104
# 0x169b544 github.com/cometbft/cometbft/state.(*BlockExecutor).applyBlock+0x524 cometbft/state/execution.go:236
# 0x1732e13 github.com/cometbft/cometbft/state.(*BlockExecutor).ApplyVerifiedBlock+0xbb3 cometbft/state/execution.go:214
# 0x1732ca5 github.com/cometbft/cometbft/consensus.(*State).finalizeCommit+0xa45 cometbft/consensus/state.go:1776
# 0x1732167 github.com/cometbft/cometbft/consensus.(*State).tryFinalizeCommit+0x2e7 cometbft/consensus/state.go:1686
# 0x1735231 github.com/cometbft/cometbft/consensus.(*State).handleCompleteProposal+0x371 cometbft/consensus/state.go:2065
# 0x172a746 github.com/cometbft/cometbft/consensus.(*State).handleMsg+0x166 cometbft/consensus/state.go:914
# 0x172a030 github.com/cometbft/cometbft/consensus.(*State).receiveRoutine+0x3f0 cometbft/consensus/state.go:840