Problem: go-block-stm is not integrated for parallel tx execution

- integrate go-block-stm as tx executor
- small optimiations

CHANGELOG.md

@@ -37,6 +37,15 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## Unreleased
 
+### Features
+
+* (evm) [#414](https://github.com/crypto-org-chain/ethermint/pull/414) Integrate go-block-stm for parallel tx execution.
+
+### State Machine Breaking
+
+* (rpc) [#443](https://github.com/crypto-org-chain/ethermint/pull/443) Keep behavior of random opcode as before.
+* (app) [#451](https://github.com/crypto-org-chain/ethermint/pull/451) Disable block gas meter, it's not compatible with parallel tx execution. It's safe to do as long as we checks total gas-wanted against block gas limit in process proposal, which we do in default handler.
+
 ### Bug Fixes
 
 - (ante) [#422](https://github.com/crypto-org-chain/ethermint/pull/422) vendor `NewDeductFeeDecorator` to re-use the custom `checkTxFeeWithValidatorMinGasPrices` method, so it'll repsect the `DefaultPriorityReduction` config.
@@ -52,13 +61,6 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (evm) [#450](https://github.com/crypto-org-chain/ethermint/pull/450) Refactor transient stores to be compatible with parallel tx execution.
 * (evm) [#454](https://github.com/crypto-org-chain/ethermint/pull/454) Migrate transient stores to object stores.
 
-### State Machine Breaking
-
-* (rpc) [#443](https://github.com/crypto-org-chain/ethermint/pull/443) Keep behavior of random opcode as before.
-* (app) [#451](https://github.com/crypto-org-chain/ethermint/pull/451) Disable block gas meter, it's not compatible with parallel tx execution. It's safe to do as long as we checks total gas-wanted against block gas limit in process proposal, which we do in default handler.
-
-### Features
-
 ## v0.21.x-cronos
 
 ### Features

app/app.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sort"
 
 	autocliv1 "cosmossdk.io/api/cosmos/autocli/v1"
 	reflectionv1 "cosmossdk.io/api/cosmos/reflection/v1"
@@ -354,6 +355,17 @@ func NewEthermintApp(
 		okeys:             okeys,
 	}
 
+	app.SetDisableBlockGasMeter(true)
+
+	executor := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor))
+	if executor == "block-stm" {
+		sdk.SetAddrCacheEnabled(false)
+		workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers))
+		app.SetTxExecutor(STMTxExecutor(app.GetStoreKeys(), workers))
+	} else {
+		app.SetTxExecutor(DefaultTxExecutor)
+	}
+
 	// init params keeper and subspaces
 	app.ParamsKeeper = initParamsKeeper(appCodec, cdc, keys[paramstypes.StoreKey], tkeys[paramstypes.TStoreKey])
 
@@ -981,6 +993,25 @@ func (app *EthermintApp) GetMemKey(storeKey string) *storetypes.MemoryStoreKey {
 	return app.memKeys[storeKey]
 }
 
+// GetStoreKeys returns all the stored store keys.
+func (app *EthermintApp) GetStoreKeys() []storetypes.StoreKey {
+	keys := make([]storetypes.StoreKey, 0, len(app.keys))
+	for _, key := range app.keys {
+		keys = append(keys, key)
+	}
+	for _, key := range app.tkeys {
+		keys = append(keys, key)
+	}
+	for _, key := range app.memKeys {
+		keys = append(keys, key)
+	}
+	for _, key := range app.okeys {
+		keys = append(keys, key)
+	}
+	sort.SliceStable(keys, func(i, j int) bool { return keys[i].Name() < keys[j].Name() })
+	return keys
+}
+
 // GetSubspace returns a param subspace for a given module name.
 //
 // NOTE: This is solely to be used for testing purposes.

app/executor.go

@@ -2,10 +2,16 @@ package app
 
 import (
 	"context"
+	"io"
 
+	"cosmossdk.io/store/cachemulti"
 	storetypes "cosmossdk.io/store/types"
 	abci "github.com/cometbft/cometbft/abci/types"
 	evmtypes "github.com/evmos/ethermint/x/evm/types"
+
+	"github.com/cosmos/cosmos-sdk/baseapp"
+
+	block_stm "github.com/yihuang/go-block-stm"
 )
 
 func DefaultTxExecutor(_ context.Context,
@@ -19,3 +25,129 @@ func DefaultTxExecutor(_ context.Context,
 	}
 	return evmtypes.PatchTxResponses(results), nil
 }
+
+func STMTxExecutor(stores []storetypes.StoreKey, workers int) baseapp.TxExecutor {
+	index := make(map[storetypes.StoreKey]int, len(stores))
+	for i, k := range stores {
+		index[k] = i
+	}
+	return func(
+		ctx context.Context,
+		blockSize int,
+		ms storetypes.MultiStore,
+		deliverTxWithMultiStore func(int, storetypes.MultiStore) *abci.ExecTxResult,
+	) ([]*abci.ExecTxResult, error) {
+		if blockSize == 0 {
+			return nil, nil
+		}
+		results := make([]*abci.ExecTxResult, blockSize)
+		if err := block_stm.ExecuteBlock(
+			ctx,
+			blockSize,
+			index,
+			stmMultiStoreWrapper{ms},
+			workers,
+			func(txn block_stm.TxnIndex, ms block_stm.MultiStore) {
+				result := deliverTxWithMultiStore(int(txn), newMultiStoreWrapper(ms, stores))
+				results[txn] = result
+			},
+		); err != nil {
+			return nil, err
+		}
+
+		return evmtypes.PatchTxResponses(results), nil
+	}
+}
+
+type msWrapper struct {
+	block_stm.MultiStore
+	stores     []storetypes.StoreKey
+	keysByName map[string]storetypes.StoreKey
+}
+
+var _ storetypes.MultiStore = msWrapper{}
+
+func newMultiStoreWrapper(ms block_stm.MultiStore, stores []storetypes.StoreKey) msWrapper {
+	keysByName := make(map[string]storetypes.StoreKey, len(stores))
+	for _, k := range stores {
+		keysByName[k.Name()] = k
+	}
+	return msWrapper{ms, stores, keysByName}
+}
+
+func (ms msWrapper) getCacheWrapper(key storetypes.StoreKey) storetypes.CacheWrapper {
+	return ms.GetStore(key)
+}
+
+func (ms msWrapper) GetStore(key storetypes.StoreKey) storetypes.Store {
+	return ms.MultiStore.GetStore(key)
+}
+
+func (ms msWrapper) GetKVStore(key storetypes.StoreKey) storetypes.KVStore {
+	return ms.MultiStore.GetKVStore(key)
+}
+
+func (ms msWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes.ObjKVStore {
+	return ms.MultiStore.GetObjKVStore(key)
+}
+
+func (ms msWrapper) CacheMultiStore() storetypes.CacheMultiStore {
+	return cachemulti.NewFromParent(ms.getCacheWrapper, nil, nil)
+}
+
+func (ms msWrapper) CacheMultiStoreWithVersion(_ int64) (storetypes.CacheMultiStore, error) {
+	panic("cannot branch cached multi-store with a version")
+}
+
+// Implements CacheWrapper.
+func (ms msWrapper) CacheWrap() storetypes.CacheWrap {
+	return ms.CacheMultiStore().(storetypes.CacheWrap)
+}
+
+// CacheWrapWithTrace implements the CacheWrapper interface.
+func (ms msWrapper) CacheWrapWithTrace(_ io.Writer, _ storetypes.TraceContext) storetypes.CacheWrap {
+	return ms.CacheWrap()
+}
+
+// GetStoreType returns the type of the store.
+func (ms msWrapper) GetStoreType() storetypes.StoreType {
+	return storetypes.StoreTypeMulti
+}
+
+// LatestVersion returns the branch version of the store
+func (ms msWrapper) LatestVersion() int64 {
+	panic("cannot get latest version from branch cached multi-store")
+}
+
+// Implements interface MultiStore
+func (ms msWrapper) SetTracer(io.Writer) storetypes.MultiStore {
+	return nil
+}
+
+// Implements interface MultiStore
+func (ms msWrapper) SetTracingContext(storetypes.TraceContext) storetypes.MultiStore {
+	return nil
+}
+
+// Implements interface MultiStore
+func (ms msWrapper) TracingEnabled() bool {
+	return false
+}
+
+type stmMultiStoreWrapper struct {
+	inner storetypes.MultiStore
+}
+
+var _ block_stm.MultiStore = stmMultiStoreWrapper{}
+
+func (ms stmMultiStoreWrapper) GetStore(key storetypes.StoreKey) storetypes.Store {
+	return ms.inner.GetStore(key)
+}
+
+func (ms stmMultiStoreWrapper) GetKVStore(key storetypes.StoreKey) storetypes.KVStore {
+	return ms.inner.GetKVStore(key)
+}
+
+func (ms stmMultiStoreWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes.ObjKVStore {
+	return ms.inner.GetObjKVStore(key)
+}

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/tidwall/gjson v1.14.4
 	github.com/tidwall/sjson v1.2.5
 	github.com/tyler-smith/go-bip39 v1.1.0
+	github.com/yihuang/go-block-stm v0.0.0-20240404183025-a4e4c8046423
 	golang.org/x/net v0.21.0
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
@@ -242,7 +243,7 @@ require (
 
 replace (
 	// release/v0.50.x
-	cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240403085742-29d3142f7596
+	cosmossdk.io/store => github.com/yihuang/cosmos-sdk/store v0.0.0-20240404193924-d4f38fd057d6
 	// use cosmos keyring
 	github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0
 	github.com/cockroachdb/pebble => github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811
@@ -253,4 +254,5 @@ replace (
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.7.0
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/tidwall/btree => github.com/yihuang/btree v0.0.0-20240215071918-6726a9b22e40
 )

go.sum

@@ -415,8 +415,6 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240403085742-29d3142f7596 h1:0GboNOKFdX+T9VmuMa/qW+s/XCjwgcqjomQ9Si6bm28=
 github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240403085742-29d3142f7596/go.mod h1:nRk8EA8/fEG4zSme2i/Rq5z3k7TrlsHkOYhrY79hhD8=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240403085742-29d3142f7596 h1:NUjIepL9YhwwhpyefOiJIU2lm5dI+qhsJ9sPyTrMFCY=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240403085742-29d3142f7596/go.mod h1:lfuLI1f4o+0SGtlHQS4x5qsjRcZZfYqG8bp3k8hM0M8=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20231207063621-43cf32d91c3e h1:vnyepPQ/m25+19xcTuBUdRxmltZ/EjVWNqEjhg7Ummk=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20231207063621-43cf32d91c3e/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y=
 github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE=
@@ -1093,8 +1091,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E=
 github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
-github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
-github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
+github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
 github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -1127,6 +1125,12 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/yihuang/btree v0.0.0-20240215071918-6726a9b22e40 h1:FPcNbZSQgK2d/Oj5tD4TwJM8ealdG4DLYvETLV9D5EE=
+github.com/yihuang/btree v0.0.0-20240215071918-6726a9b22e40/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/yihuang/cosmos-sdk/store v0.0.0-20240404193924-d4f38fd057d6 h1:7z30s8j7xhOoGs6Y3Zlt751JkwzTYGES21efEU8Jh9c=
+github.com/yihuang/cosmos-sdk/store v0.0.0-20240404193924-d4f38fd057d6/go.mod h1:lfuLI1f4o+0SGtlHQS4x5qsjRcZZfYqG8bp3k8hM0M8=
+github.com/yihuang/go-block-stm v0.0.0-20240404183025-a4e4c8046423 h1:3fiPoGYWkXgUa22WHCKbrjWgX1/kixGFqVPx0dmYsW4=
+github.com/yihuang/go-block-stm v0.0.0-20240404183025-a4e4c8046423/go.mod h1:U56qBapBYN86mr25QoPfL3pJyF/9a+AzL5TMlo7agWs=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

gomod2nix.toml

@@ -41,9 +41,9 @@ schema = 3
     version = "v1.3.0"
     hash = "sha256-EEFK43Cr0g0ndhQhkIKher0FqV3mvkmE9z0sP7uVSHg="
   [mod."cosmossdk.io/store"]
-    version = "v0.0.0-20240403085742-29d3142f7596"
-    hash = "sha256-wV4pdty1onr51ZtRWS1MSR4sH3rptsLL+Zc3dPuvlIA="
-    replaced = "github.com/crypto-org-chain/cosmos-sdk/store"
+    version = "v0.0.0-20240404193924-d4f38fd057d6"
+    hash = "sha256-UF7viH+TYLp4eqXub4DHzQn43G34RwRAgcG7Xepsugg="
+    replaced = "github.com/yihuang/cosmos-sdk/store"
   [mod."cosmossdk.io/tools/confix"]
     version = "v0.1.1"
     hash = "sha256-/Et2FFhb4XfakbLFvGQK3QxN5Y7alzO+DGfi2/EWbxo="
@@ -566,8 +566,9 @@ schema = 3
     version = "v0.16.0"
     hash = "sha256-JW4zO/0vMzf1dXLePOqaMtiLUZgNbuIseh9GV+jQlf0="
   [mod."github.com/tidwall/btree"]
-    version = "v1.7.0"
-    hash = "sha256-bnr6c7a0nqo2HyGqxHk0kEZCEsjLYkPbAVY9WzaZ30o="
+    version = "v0.0.0-20240215071918-6726a9b22e40"
+    hash = "sha256-s0olXV7w23o3UHCxlndyXFilL0tzbibxVecylN5a+LQ="
+    replaced = "github.com/yihuang/btree"
   [mod."github.com/tidwall/gjson"]
     version = "v1.14.4"
     hash = "sha256-3DS2YNL95wG0qSajgRtIABD32J+oblaKVk8LIw+KSOc="
@@ -592,6 +593,9 @@ schema = 3
   [mod."github.com/ulikunitz/xz"]
     version = "v0.5.11"
     hash = "sha256-SUyrjc2wyN3cTGKe5JdBEXjtZC1rJySRxJHVUZ59row="
+  [mod."github.com/yihuang/go-block-stm"]
+    version = "v0.0.0-20240404183025-a4e4c8046423"
+    hash = "sha256-WrtgRKXasiWUh+fSX2FHcYMDpjoArKwGxLoMJ35GOEU="
   [mod."github.com/zondax/hid"]
     version = "v0.9.2"
     hash = "sha256-9h1gEJ/loyaJvu9AsmslztiA8U9ixDTC6TBw9lCU2BE="

server/config/config.go

@@ -124,6 +124,10 @@ type EVMConfig struct {
 	Tracer string `mapstructure:"tracer"`
 	// MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.
 	MaxTxGasWanted uint64 `mapstructure:"max-tx-gas-wanted"`
+	// BlockExecutor set block executor type, "block-stm" for parallel execution, "default" for sequential execution.
+	BlockExecutor string `mapstructure:"block-executor"`
+	// BlockSTMWorkers is the number of workers for block-stm execution, `0` means using all available CPUs.
+	BlockSTMWorkers int `mapstructure:"block-stm-workers"`
 }
 
 // JSONRPCConfig defines configuration for the EVM RPC server.
@@ -238,6 +242,7 @@ func DefaultEVMConfig() *EVMConfig {
 	return &EVMConfig{
 		Tracer:         DefaultEVMTracer,
 		MaxTxGasWanted: DefaultMaxTxGasWanted,
+		BlockExecutor:  "sequential",
 	}
 }
 

server/config/toml.go

@@ -31,6 +31,11 @@ tracer = "{{ .EVM.Tracer }}"
 # MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.
 max-tx-gas-wanted = {{ .EVM.MaxTxGasWanted }}
 
+# BlockExecutor set block executor type, "block-stm" for parallel execution, "default" for sequential execution.
+block-executor = "{{ .EVM.BlockExecutor }}"
+# BlockSTMWorkers is the number of workers for block-stm execution, 0 means using all available CPUs.
+block-stm-workers = {{ .EVM.BlockSTMWorkers }}
+
 ###############################################################################
 ###                           JSON RPC Configuration                        ###
 ###############################################################################

server/flags/flags.go

@@ -76,8 +76,10 @@ const (
 
 // EVM flags
 const (
-	EVMTracer         = "evm.tracer"
-	EVMMaxTxGasWanted = "evm.max-tx-gas-wanted"
+	EVMTracer          = "evm.tracer"
+	EVMMaxTxGasWanted  = "evm.max-tx-gas-wanted"
+	EVMBlockExecutor   = "evm.block-executor"
+	EVMBlockSTMWorkers = "evm.block-stm-workers"
 )
 
 // TLS flags

tests/importer/importer_test.go

@@ -145,7 +145,6 @@ func (suite *ImporterTestSuite) TestImportBlocks() {
 		}
 
 		for i, tx := range block.Transactions() {
-
 			receipt, gas, err := applyTransaction(
 				ctx, chainConfig, chainContext, nil, gp, suite.app.EvmKeeper, vmdb, header, tx, usedGas, vmConfig, uint(i),
 			)
@@ -263,7 +262,7 @@ func applyTransaction(
 
 	// if the transaction created a contract, store the creation address in the receipt.
 	if msg.To == nil {
-		receipt.ContractAddress = crypto.CreateAddress(vmenv.TxContext.Origin, tx.Nonce())
+		receipt.ContractAddress = crypto.CreateAddress(vmenv.Origin, tx.Nonce())
 	}
 
 	// Set the receipt logs and create a bloom for filtering

tests/integration_tests/configs/default.jsonnet

@@ -6,6 +6,9 @@
     'app-config': {
       'minimum-gas-prices': '0aphoton',
       'index-events': ['ethereum_tx.ethereumTxHash'],
+      evm: {
+        'block-executor': 'block-stm',
+      },
       'json-rpc': {
         address: '127.0.0.1:{EVMRPC_PORT}',
         'ws-address': '127.0.0.1:{EVMRPC_PORT_WS}',