Problem: parallel transaction execution is not supported

CHANGELOG.md

@@ -37,6 +37,15 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## Unreleased
 
+### Features
+
+* (evm) [#414](https://github.com/crypto-org-chain/ethermint/pull/414) Integrate go-block-stm for parallel tx execution.
+
+### State Machine Breaking
+
+* (rpc) [#443](https://github.com/crypto-org-chain/ethermint/pull/443) Keep behavior of random opcode as before.
+* (app) [#451](https://github.com/crypto-org-chain/ethermint/pull/451) Disable block gas meter, it's not compatible with parallel tx execution. It's safe to do as long as we checks total gas-wanted against block gas limit in process proposal, which we do in default handler.
+
 ### Bug Fixes
 
 - (ante) [#422](https://github.com/crypto-org-chain/ethermint/pull/422) vendor `NewDeductFeeDecorator` to re-use the custom `checkTxFeeWithValidatorMinGasPrices` method, so it'll repsect the `DefaultPriorityReduction` config.
@@ -52,13 +61,6 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (evm) [#450](https://github.com/crypto-org-chain/ethermint/pull/450) Refactor transient stores to be compatible with parallel tx execution.
 * (evm) [#454](https://github.com/crypto-org-chain/ethermint/pull/454) Migrate transient stores to object stores.
 
-### State Machine Breaking
-
-* (rpc) [#443](https://github.com/crypto-org-chain/ethermint/pull/443) Keep behavior of random opcode as before.
-* (app) [#451](https://github.com/crypto-org-chain/ethermint/pull/451) Disable block gas meter, it's not compatible with parallel tx execution. It's safe to do as long as we checks total gas-wanted against block gas limit in process proposal, which we do in default handler.
-
-### Features
-
 ## v0.21.x-cronos
 
 ### Features

app/app.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sort"
 
 	autocliv1 "cosmossdk.io/api/cosmos/autocli/v1"
 	reflectionv1 "cosmossdk.io/api/cosmos/reflection/v1"
@@ -314,7 +315,6 @@ func NewEthermintApp(
 	bApp.SetVersion(version.Version)
 	bApp.SetInterfaceRegistry(interfaceRegistry)
 	bApp.SetTxEncoder(txConfig.TxEncoder())
-	bApp.SetTxExecutor(DefaultTxExecutor)
 	bApp.SetDisableBlockGasMeter(true)
 
 	keys := storetypes.NewKVStoreKeys(
@@ -354,6 +354,15 @@ func NewEthermintApp(
 		okeys:             okeys,
 	}
 
+	executor := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor))
+	if executor == "block-stm" {
+		sdk.SetAddrCacheEnabled(false)
+		workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers))
+		app.SetTxExecutor(STMTxExecutor(app.GetStoreKeys(), workers))
+	} else {
+		app.SetTxExecutor(DefaultTxExecutor)
+	}
+
 	// init params keeper and subspaces
 	app.ParamsKeeper = initParamsKeeper(appCodec, cdc, keys[paramstypes.StoreKey], tkeys[paramstypes.TStoreKey])
 
@@ -981,6 +990,25 @@ func (app *EthermintApp) GetMemKey(storeKey string) *storetypes.MemoryStoreKey {
 	return app.memKeys[storeKey]
 }
 
+// GetStoreKeys returns all the stored store keys.
+func (app *EthermintApp) GetStoreKeys() []storetypes.StoreKey {
+	keys := make([]storetypes.StoreKey, 0, len(app.keys))
+	for _, key := range app.keys {
+		keys = append(keys, key)
+	}
+	for _, key := range app.tkeys {
+		keys = append(keys, key)
+	}
+	for _, key := range app.memKeys {
+		keys = append(keys, key)
+	}
+	for _, key := range app.okeys {
+		keys = append(keys, key)
+	}
+	sort.SliceStable(keys, func(i, j int) bool { return keys[i].Name() < keys[j].Name() })
+	return keys
+}
+
 // GetSubspace returns a param subspace for a given module name.
 //
 // NOTE: This is solely to be used for testing purposes.

app/executor.go

@@ -2,10 +2,16 @@ package app
 
 import (
 	"context"
+	"io"
 
+	"cosmossdk.io/store/cachemulti"
 	storetypes "cosmossdk.io/store/types"
 	abci "github.com/cometbft/cometbft/abci/types"
 	evmtypes "github.com/evmos/ethermint/x/evm/types"
+
+	"github.com/cosmos/cosmos-sdk/baseapp"
+
+	blockstm "github.com/crypto-org-chain/go-block-stm"
 )
 
 func DefaultTxExecutor(_ context.Context,
@@ -19,3 +25,105 @@ func DefaultTxExecutor(_ context.Context,
 	}
 	return evmtypes.PatchTxResponses(results), nil
 }
+
+func STMTxExecutor(stores []storetypes.StoreKey, workers int) baseapp.TxExecutor {
+	index := make(map[storetypes.StoreKey]int, len(stores))
+	for i, k := range stores {
+		index[k] = i
+	}
+	return func(
+		ctx context.Context,
+		blockSize int,
+		ms storetypes.MultiStore,
+		deliverTxWithMultiStore func(int, storetypes.MultiStore) *abci.ExecTxResult,
+	) ([]*abci.ExecTxResult, error) {
+		if blockSize == 0 {
+			return nil, nil
+		}
+		results := make([]*abci.ExecTxResult, blockSize)
+		if err := blockstm.ExecuteBlock(
+			ctx,
+			blockSize,
+			index,
+			stmMultiStoreWrapper{ms},
+			workers,
+			func(txn blockstm.TxnIndex, ms blockstm.MultiStore) {
+				result := deliverTxWithMultiStore(int(txn), msWrapper{ms})
+				results[txn] = result
+			},
+		); err != nil {
+			return nil, err
+		}
+
+		return evmtypes.PatchTxResponses(results), nil
+	}
+}
+
+type msWrapper struct {
+	blockstm.MultiStore
+}
+
+var _ storetypes.MultiStore = msWrapper{}
+
+func (ms msWrapper) getCacheWrapper(key storetypes.StoreKey) storetypes.CacheWrapper {
+	return ms.GetStore(key)
+}
+
+func (ms msWrapper) GetStore(key storetypes.StoreKey) storetypes.Store {
+	return ms.MultiStore.GetStore(key)
+}
+
+func (ms msWrapper) GetKVStore(key storetypes.StoreKey) storetypes.KVStore {
+	return ms.MultiStore.GetKVStore(key)
+}
+
+func (ms msWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes.ObjKVStore {
+	return ms.MultiStore.GetObjKVStore(key)
+}
+
+func (ms msWrapper) CacheMultiStore() storetypes.CacheMultiStore {
+	return cachemulti.NewFromParent(ms.getCacheWrapper, nil, nil)
+}
+
+// Implements CacheWrapper.
+func (ms msWrapper) CacheWrap() storetypes.CacheWrap {
+	return ms.CacheMultiStore().(storetypes.CacheWrap)
+}
+
+// GetStoreType returns the type of the store.
+func (ms msWrapper) GetStoreType() storetypes.StoreType {
+	return storetypes.StoreTypeMulti
+}
+
+// Implements interface MultiStore
+func (ms msWrapper) SetTracer(io.Writer) storetypes.MultiStore {
+	return nil
+}
+
+// Implements interface MultiStore
+func (ms msWrapper) SetTracingContext(storetypes.TraceContext) storetypes.MultiStore {
+	return nil
+}
+
+// Implements interface MultiStore
+func (ms msWrapper) TracingEnabled() bool {
+	return false
+}
+
+type stmMultiStoreWrapper struct {
+	storetypes.MultiStore
+}
+
+var _ blockstm.MultiStore = stmMultiStoreWrapper{}
+
+func (ms stmMultiStoreWrapper) GetStore(key storetypes.StoreKey) storetypes.Store {
+	return ms.MultiStore.GetStore(key)
+}
+
+func (ms stmMultiStoreWrapper) GetKVStore(key storetypes.StoreKey) storetypes.KVStore {
+	return ms.MultiStore.GetKVStore(key)
+}
+
+func (ms stmMultiStoreWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes.ObjKVStore {
+	return ms.MultiStore.GetObjKVStore(key)
+}

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/cosmos/ibc-go/modules/capability v1.0.0
 	github.com/cosmos/ibc-go/v8 v8.1.0
 	github.com/cosmos/rosetta v0.50.3-1
+	github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/gogo/protobuf v1.3.2
@@ -242,15 +243,16 @@ require (
 
 replace (
 	// release/v0.50.x
-	cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240403085742-29d3142f7596
+	cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240408014408-56c165536d0a
 	// use cosmos keyring
 	github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0
 	github.com/cockroachdb/pebble => github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811
 	// release/v0.50.x
-	github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240403085742-29d3142f7596
+	github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240408014408-56c165536d0a
 	github.com/ethereum/go-ethereum => github.com/crypto-org-chain/go-ethereum v1.10.20-0.20231207063621-43cf32d91c3e
 	// Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.7.0
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/tidwall/btree => github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c
 )

go.sum

@@ -413,10 +413,14 @@ github.com/creachadair/tomledit v0.0.24 h1:5Xjr25R2esu1rKCbQEmjZYlrhFkDspoAbAKb6
 github.com/creachadair/tomledit v0.0.24/go.mod h1:9qHbShRWQzSCcn617cMzg4eab1vbLCOjOshAWSzWr8U=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240403085742-29d3142f7596 h1:0GboNOKFdX+T9VmuMa/qW+s/XCjwgcqjomQ9Si6bm28=
-github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240403085742-29d3142f7596/go.mod h1:nRk8EA8/fEG4zSme2i/Rq5z3k7TrlsHkOYhrY79hhD8=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240403085742-29d3142f7596 h1:NUjIepL9YhwwhpyefOiJIU2lm5dI+qhsJ9sPyTrMFCY=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240403085742-29d3142f7596/go.mod h1:lfuLI1f4o+0SGtlHQS4x5qsjRcZZfYqG8bp3k8hM0M8=
+github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c h1:MOgfS4+FBB8cMkDE2j2VBVsbY+HCkPIu0YsJ/9bbGeQ=
+github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240408014408-56c165536d0a h1:3IiFpiRRiF0lFCJcKGEu6UafcqzRUwUUl75aXan9pJI=
+github.com/crypto-org-chain/cosmos-sdk v0.46.0-beta2.0.20240408014408-56c165536d0a/go.mod h1:nRk8EA8/fEG4zSme2i/Rq5z3k7TrlsHkOYhrY79hhD8=
+github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240408014408-56c165536d0a h1:YHeIe/VPKf7xK7xxLWUzvspFZJY6fDT6Epj+MH9dLOc=
+github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240408014408-56c165536d0a/go.mod h1:lfuLI1f4o+0SGtlHQS4x5qsjRcZZfYqG8bp3k8hM0M8=
+github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde h1:sQIHTJfVt5VTrF7po9eZiFkZiPjlHbFvnXtGCOoBjNM=
+github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20231207063621-43cf32d91c3e h1:vnyepPQ/m25+19xcTuBUdRxmltZ/EjVWNqEjhg7Ummk=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20231207063621-43cf32d91c3e/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y=
 github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE=
@@ -1093,8 +1097,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E=
 github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
-github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
-github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
+github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
 github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=

gomod2nix.toml

@@ -41,8 +41,8 @@ schema = 3
     version = "v1.3.0"
     hash = "sha256-EEFK43Cr0g0ndhQhkIKher0FqV3mvkmE9z0sP7uVSHg="
   [mod."cosmossdk.io/store"]
-    version = "v0.0.0-20240403085742-29d3142f7596"
-    hash = "sha256-wV4pdty1onr51ZtRWS1MSR4sH3rptsLL+Zc3dPuvlIA="
+    version = "v0.0.0-20240408014408-56c165536d0a"
+    hash = "sha256-5fFhveNdA4rEwtTVNE4MIzK7udgNF80q9fkiDcw7T/Q="
     replaced = "github.com/crypto-org-chain/cosmos-sdk/store"
   [mod."cosmossdk.io/tools/confix"]
     version = "v0.1.1"
@@ -161,8 +161,8 @@ schema = 3
     version = "v1.0.0-beta.4"
     hash = "sha256-5Kn82nsZfiEtuwhhLZqmMxdAY1tX/Fi3HJ0/MEaRohw="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.46.0-beta2.0.20240403085742-29d3142f7596"
-    hash = "sha256-qIvDAFwGrFIOxPGUazpfR3mu9X3eJqkbBJWQUwhP2hY="
+    version = "v0.46.0-beta2.0.20240408014408-56c165536d0a"
+    hash = "sha256-xjZvl0pxsEkSW2X8Q37mPHXAHnMeHiwCtX2FmDZVUHQ="
     replaced = "github.com/crypto-org-chain/cosmos-sdk"
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"
@@ -200,6 +200,9 @@ schema = 3
   [mod."github.com/creachadair/tomledit"]
     version = "v0.0.24"
     hash = "sha256-4vUukHONOjNn0qfQr4esK6TWfPWsIp+rbdz65og84lw="
+  [mod."github.com/crypto-org-chain/go-block-stm"]
+    version = "v0.0.0-20240408011717-9f11af197bde"
+    hash = "sha256-+UTSUoh4DxkWPQEc/j0atak1Oxq3tdP8r6ZMmB0k5KE="
   [mod."github.com/danieljoos/wincred"]
     version = "v1.2.0"
     hash = "sha256-LHcvTJCc8++bFndbd8ZgMSTe4L5h2C4rN+cSWHCz54Y="
@@ -566,8 +569,9 @@ schema = 3
     version = "v0.16.0"
     hash = "sha256-JW4zO/0vMzf1dXLePOqaMtiLUZgNbuIseh9GV+jQlf0="
   [mod."github.com/tidwall/btree"]
-    version = "v1.7.0"
-    hash = "sha256-bnr6c7a0nqo2HyGqxHk0kEZCEsjLYkPbAVY9WzaZ30o="
+    version = "v0.0.0-20240406140148-2687063b042c"
+    hash = "sha256-8eDLGHhw4qXG6MEa7w5Q9KLwOobXr8Vn5qqyQhuipQw="
+    replaced = "github.com/crypto-org-chain/btree"
   [mod."github.com/tidwall/gjson"]
     version = "v1.14.4"
     hash = "sha256-3DS2YNL95wG0qSajgRtIABD32J+oblaKVk8LIw+KSOc="

server/config/config.go

@@ -124,6 +124,10 @@ type EVMConfig struct {
 	Tracer string `mapstructure:"tracer"`
 	// MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.
 	MaxTxGasWanted uint64 `mapstructure:"max-tx-gas-wanted"`
+	// BlockExecutor set block executor type, "block-stm" for parallel execution, "sequential" for sequential execution.
+	BlockExecutor string `mapstructure:"block-executor"`
+	// BlockSTMWorkers is the number of workers for block-stm execution, `0` means using all available CPUs.
+	BlockSTMWorkers int `mapstructure:"block-stm-workers"`
 }
 
 // JSONRPCConfig defines configuration for the EVM RPC server.
@@ -238,6 +242,7 @@ func DefaultEVMConfig() *EVMConfig {
 	return &EVMConfig{
 		Tracer:         DefaultEVMTracer,
 		MaxTxGasWanted: DefaultMaxTxGasWanted,
+		BlockExecutor:  "sequential",
 	}
 }
 

server/config/toml.go

@@ -31,6 +31,11 @@ tracer = "{{ .EVM.Tracer }}"
 # MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.
 max-tx-gas-wanted = {{ .EVM.MaxTxGasWanted }}
 
+# BlockExecutor set block executor type, "block-stm" for parallel execution, "default" for sequential execution.
+block-executor = "{{ .EVM.BlockExecutor }}"
+# BlockSTMWorkers is the number of workers for block-stm execution, 0 means using all available CPUs.
+block-stm-workers = {{ .EVM.BlockSTMWorkers }}
+
 ###############################################################################
 ###                           JSON RPC Configuration                        ###
 ###############################################################################

server/flags/flags.go

@@ -76,8 +76,10 @@ const (
 
 // EVM flags
 const (
-	EVMTracer         = "evm.tracer"
-	EVMMaxTxGasWanted = "evm.max-tx-gas-wanted"
+	EVMTracer          = "evm.tracer"
+	EVMMaxTxGasWanted  = "evm.max-tx-gas-wanted"
+	EVMBlockExecutor   = "evm.block-executor"
+	EVMBlockSTMWorkers = "evm.block-stm-workers"
 )
 
 // TLS flags

tests/importer/importer_test.go

@@ -145,7 +145,6 @@ func (suite *ImporterTestSuite) TestImportBlocks() {
 		}
 
 		for i, tx := range block.Transactions() {
-
 			receipt, gas, err := applyTransaction(
 				ctx, chainConfig, chainContext, nil, gp, suite.app.EvmKeeper, vmdb, header, tx, usedGas, vmConfig, uint(i),
 			)
@@ -263,7 +262,7 @@ func applyTransaction(
 
 	// if the transaction created a contract, store the creation address in the receipt.
 	if msg.To == nil {
-		receipt.ContractAddress = crypto.CreateAddress(vmenv.TxContext.Origin, tx.Nonce())
+		receipt.ContractAddress = crypto.CreateAddress(vmenv.Origin, tx.Nonce())
 	}
 
 	// Set the receipt logs and create a bloom for filtering

tests/integration_tests/configs/default.jsonnet

@@ -6,6 +6,9 @@
     'app-config': {
       'minimum-gas-prices': '0aphoton',
       'index-events': ['ethereum_tx.ethereumTxHash'],
+      evm: {
+        'block-executor': 'block-stm',
+      },
       'json-rpc': {
         address: '127.0.0.1:{EVMRPC_PORT}',
         'ws-address': '127.0.0.1:{EVMRPC_PORT_WS}',