Merge branch 'zkevm' into feature/hashing-poseidongold

cmd/cdk-erigon/main.go

@@ -8,7 +8,6 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/gateway-fm/vectorized-poseidon-gold/src/vectorizedposeidongold"
 	"github.com/ledgerwatch/erigon/common"
 	zkutils "github.com/ledgerwatch/erigon/zk/utils"
 	"github.com/ledgerwatch/log/v3"
@@ -61,7 +60,6 @@ func runErigon(cliCtx *cli.Context) error {
 
 	// initializing the node and providing the current git commit there
 	log.Info("Build info", "git_branch", params.GitBranch, "git_tag", params.GitTag, "git_commit", params.GitCommit)
-	log.Info("Poseidon hashing", "Accelerated", vectorizedposeidongold.UsingSimd || vectorizedposeidongold.UsingScalars)
 
 	logger := log.New()
 	nodeCfg := node.NewNodConfigUrfave(cliCtx, logger)

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -87,6 +87,7 @@ type HttpCfg struct {
 	ReturnDataLimit             int    // Maximum number of bytes returned from calls (like eth_call)
 	LogsMaxRange                uint64 // Maximum number of logs that can be requested in a single call
 	AllowUnprotectedTxs         bool   // Whether to allow non EIP-155 protected transactions  txs over RPC
+	DisableStateRootCheck       bool   // Disable state root check in GetProof
 	MaxGetProofRewindBlockCount int    //Max GetProof rewind block count
 	// Ots API
 	OtsMaxPageSize uint64

cmd/utils/flags.go

@@ -936,6 +936,11 @@ var (
 		Name:  "debug.step-after",
 		Usage: "Start incrementing by debug.step after this block",
 	}
+	DebugDisableStateRootCheck = cli.BoolFlag{
+		Name:  "debug.disable-state-root-check",
+		Usage: "Check the state root after each block",
+		Value: false,
+	}
 	RpcBatchConcurrencyFlag = cli.UintFlag{
 		Name:  "rpc.batch.concurrency",
 		Usage: "Does limit amount of goroutines to process 1 batch request. Means 1 bach request can't overload server. 1 batch still can have unlimited amount of request",

core/vm/zk_counters.go

@@ -124,6 +124,35 @@ func (c Counters) UsedAsMap() map[string]int {
 	}
 }
 
+func (c Counters) OverflownAsString() string {
+	var res string
+	if c[SHA].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[SHA], c[SHA].remaining)
+	}
+	if c[A].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[A], c[A].remaining)
+	}
+	if c[B].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[B], c[B].remaining)
+	}
+	if c[K].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[K], c[K].remaining)
+	}
+	if c[M].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[M], c[M].remaining)
+	}
+	if c[P].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[P], c[P].remaining)
+	}
+	if c[S].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[S], c[S].remaining)
+	}
+	if c[D].remaining < 0 {
+		res += fmt.Sprintf("[%s: %v]", CounterKeyNames[D], c[D].remaining)
+	}
+	return res
+}
+
 func (c *Counters) GetArithmetics() *Counter {
 	return (*c)[A]
 }

eth/ethconfig/config_zkevm.go

@@ -88,11 +88,12 @@ type Zk struct {
 	SyncLimitUnverifiedCount uint64
 	Gasless                  bool
 
-	DebugTimers    bool
-	DebugNoSync    bool
-	DebugLimit     uint64
-	DebugStep      uint64
-	DebugStepAfter uint64
+	DebugTimers                bool
+	DebugNoSync                bool
+	DebugLimit                 uint64
+	DebugStep                  uint64
+	DebugStepAfter             uint64
+	DebugDisableStateRootCheck bool
 
 	PoolManagerUrl              string
 	DisableVirtualCounters      bool

eth/stagedsync/stage_execute_zkevm.go

@@ -49,8 +49,10 @@ func SpawnExecuteBlocksStageZk(s *StageState, u Unwinder, tx kv.RwTx, toBlock ui
 	defer func() {
 		if cfg.zk.DebugLimit > 0 {
 			if err != nil {
-				log.Error("Execution Failed", "err", err, "block", highestBlockExecuted)
-				os.Exit(2)
+				if !errors.Is(err, common.ErrStopped) {
+					log.Error("Execution Failed", "err", err, "block", highestBlockExecuted)
+					os.Exit(2)
+				}
 			}
 		}
 	}()

go.mod

@@ -43,7 +43,6 @@ require (
 	github.com/edsrzf/mmap-go v1.1.0
 	github.com/emicklei/dot v1.6.1
 	github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c
-	github.com/gateway-fm/vectorized-poseidon-gold v1.0.0
 	github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35
 	github.com/gfx-labs/sse v0.0.0-20231226060816-f747e26a9baa
 	github.com/go-chi/chi/v5 v5.0.12
@@ -63,6 +62,7 @@ require (
 	github.com/hashicorp/golang-lru/arc/v2 v2.0.6
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/holiman/uint256 v1.3.1
+	github.com/huandu/xstrings v1.4.0
 	github.com/huin/goupnp v1.2.0
 	github.com/iden3/go-iden3-crypto v0.0.17
 	github.com/jackc/pgx/v4 v4.18.2
@@ -186,7 +186,6 @@ require (
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd // indirect
 	github.com/hermeznetwork/tracerr v0.3.2 // indirect
-	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/ianlancetaylor/cgosymbolizer v0.0.0-20220405231054-a1ae3e4bba26 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

smt/pkg/utils/util_test.go

@@ -2,8 +2,6 @@ package utils
 
 import (
 	"bytes"
-	"crypto/rand"
-	"encoding/hex"
 	"fmt"
 	"math/big"
 	"reflect"
@@ -55,13 +53,7 @@ func BenchmarkConvertBigIntToHex(b *testing.B) {
 }
 
 func BenchmarkHashContractBytecode(b *testing.B) {
-	str := strings.Repeat("7e", 500)
-	h1 := HashContractBytecodeBigIntV1(str)
-	h2 := HashContractBytecodeBigInt(str)
-	if h1.Cmp(h2) != 0 {
-		panic("hashes do not match")
-	}
-	b.ResetTimer()
+	str := strings.Repeat("e", 1000)
 	b.Run("1", func(b *testing.B) {
 		for n := 0; n < b.N; n++ {
 			HashContractBytecodeBigIntV1(str)

turbo/cli/default_flags.go

@@ -251,6 +251,7 @@ var DefaultFlags = []cli.Flag{
 	&utils.DebugLimit,
 	&utils.DebugStep,
 	&utils.DebugStepAfter,
+	&utils.DebugDisableStateRootCheck,
 	&utils.OtsSearchMaxCapFlag,
 	&utils.PanicOnReorg,
 	&utils.ShadowSequencer,

turbo/cli/flags.go

@@ -521,6 +521,7 @@ func setEmbeddedRpcDaemon(ctx *cli.Context, cfg *nodecfg.Config, logger log.Logg
 		DataStreamInactivityTimeout:       ctx.Duration(utils.DataStreamInactivityTimeout.Name),
 		DataStreamInactivityCheckInterval: ctx.Duration(utils.DataStreamInactivityCheckInterval.Name),
 		L2RpcUrl:                          ctx.String(utils.L2RpcUrlFlag.Name),
+		DisableStateRootCheck:             ctx.Bool(utils.DebugDisableStateRootCheck.Name),
 	}
 
 	if ctx.IsSet(utils.HttpCompressionFlag.Name) {

turbo/cli/flags_zkevm.go

@@ -248,6 +248,7 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) {
 		DebugLimit:                             ctx.Uint64(utils.DebugLimit.Name),
 		DebugStep:                              ctx.Uint64(utils.DebugStep.Name),
 		DebugStepAfter:                         ctx.Uint64(utils.DebugStepAfter.Name),
+		DebugDisableStateRootCheck:             ctx.Bool(utils.DebugDisableStateRootCheck.Name),
 		PoolManagerUrl:                         ctx.String(utils.PoolManagerUrl.Name),
 		TxPoolRejectSmartContractDeployments:   ctx.Bool(utils.TxPoolRejectSmartContractDeployments.Name),
 		DisableVirtualCounters:                 ctx.Bool(utils.DisableVirtualCounters.Name),

turbo/engineapi/engine_server.go

@@ -92,7 +92,7 @@ func (e *EngineServer) Start(
 ) {
 	base := jsonrpc.NewBaseApi(filters, stateCache, blockReader, agg, httpConfig.WithDatadir, httpConfig.EvmCallTimeout, engineReader, httpConfig.Dirs)
 
-	ethImpl := jsonrpc.NewEthAPI(base, db, eth, txPool, mining, httpConfig.Gascap, httpConfig.Feecap, httpConfig.ReturnDataLimit, &ethconfig.Defaults, httpConfig.AllowUnprotectedTxs, httpConfig.MaxGetProofRewindBlockCount, httpConfig.WebsocketSubscribeLogsChannelSize, e.logger, gasTracker, httpConfig.LogsMaxRange)
+	ethImpl := jsonrpc.NewEthAPI(base, db, eth, txPool, mining, httpConfig.Gascap, httpConfig.Feecap, httpConfig.ReturnDataLimit, &ethconfig.Defaults, httpConfig.AllowUnprotectedTxs, httpConfig.MaxGetProofRewindBlockCount, httpConfig.WebsocketSubscribeLogsChannelSize, e.logger, gasTracker, httpConfig.LogsMaxRange, httpConfig.DisableStateRootCheck)
 
 	// engineImpl := NewEngineAPI(base, db, engineBackend)
 	// e.startEngineMessageHandler()

turbo/jsonrpc/daemon.go

@@ -39,7 +39,7 @@ func APIList(db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, r
 	base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs)
 	base.SetL2RpcUrl(ethCfg.Zk.L2RpcUrl)
 	base.SetGasless(ethCfg.AllowFreeTransactions)
-	ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.Feecap, cfg.ReturnDataLimit, ethCfg, cfg.AllowUnprotectedTxs, cfg.MaxGetProofRewindBlockCount, cfg.WebsocketSubscribeLogsChannelSize, logger, gasTracker, cfg.LogsMaxRange)
+	ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.Feecap, cfg.ReturnDataLimit, ethCfg, cfg.AllowUnprotectedTxs, cfg.MaxGetProofRewindBlockCount, cfg.WebsocketSubscribeLogsChannelSize, logger, gasTracker, cfg.LogsMaxRange, ethCfg.DebugDisableStateRootCheck)
 	erigonImpl := NewErigonAPI(base, db, eth)
 	txpoolImpl := NewTxPoolAPI(base, db, txPool, rawPool, rpcUrl)
 	netImpl := NewNetAPIImpl(eth)

turbo/jsonrpc/eth_api.go

@@ -383,10 +383,12 @@ type APIImpl struct {
 	BadTxAllowance                uint64
 	SenderLocks                   *SenderLock
 	LogsMaxRange                  uint64
+	DisableStateRootCheck         bool
+	DisableVirtualCounters        bool
 }
 
 // NewEthAPI returns APIImpl instance
-func NewEthAPI(base *BaseAPI, db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient, gascap uint64, feecap float64, returnDataLimit int, ethCfg *ethconfig.Config, allowUnprotectedTxs bool, maxGetProofRewindBlockCount int, subscribeLogsChannelSize int, logger log.Logger, gasTracker RpcL1GasPriceTracker, LogsMaxRange uint64) *APIImpl {
+func NewEthAPI(base *BaseAPI, db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient, gascap uint64, feecap float64, returnDataLimit int, ethCfg *ethconfig.Config, allowUnprotectedTxs bool, maxGetProofRewindBlockCount int, subscribeLogsChannelSize int, logger log.Logger, gasTracker RpcL1GasPriceTracker, LogsMaxRange uint64, disableStateRootCheck bool) *APIImpl {
 	if gascap == 0 {
 		gascap = uint64(math.MaxUint64 / 2)
 	}
@@ -423,6 +425,8 @@ func NewEthAPI(base *BaseAPI, db kv.RoDB, eth rpchelper.ApiBackend, txPool txpoo
 		BadTxAllowance:                ethCfg.BadTxAllowance,
 		SenderLocks:                   NewSenderLock(),
 		LogsMaxRange:                  LogsMaxRange,
+		DisableStateRootCheck:         disableStateRootCheck,
+		DisableVirtualCounters:        ethCfg.DisableVirtualCounters,
 	}
 }
 

turbo/jsonrpc/eth_call.go

@@ -250,11 +250,13 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs
 	}
 	header := block.HeaderNoCopy()
 
-	caller, err := transactions.NewReusableCaller(engine, stateReader, nil, header, args, api.GasCap, latestNumOrHash, dbtx, api._blockReader, chainConfig, api.evmCallTimeout, api.VirtualCountersSmtReduction, false)
+	useCounters := !api.DisableVirtualCounters
+	caller, err := transactions.NewReusableCaller(engine, stateReader, nil, header, args, api.GasCap, latestNumOrHash, dbtx, api._blockReader, chainConfig, api.evmCallTimeout, api.VirtualCountersSmtReduction, useCounters)
 	if err != nil {
 		return 0, err
 	}
 
+	countersChecked := false
 	// Create a helper to check if a gas allowance results in an executable transaction
 	executable := func(gas uint64) (bool, *core.ExecutionResult, error) {
 		result, err := caller.DoCallWithNewGas(ctx, gas)
@@ -267,6 +269,14 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs
 			// Bail out
 			return true, nil, err
 		}
+
+		if useCounters && !countersChecked {
+			if overflow, err := caller.CheckCountersOverflow(result); overflow {
+				return true, nil, err
+			}
+			countersChecked = true
+		}
+
 		return result.Failed(), result, nil
 	}
 
@@ -366,7 +376,7 @@ func (api *APIImpl) GetProof(ctx context.Context, address libcommon.Address, sto
 			return nil, err
 		}
 
-		interHashStageCfg := stagedsync.StageTrieCfg(nil, false, false, false, api.dirs.Tmp, api._blockReader, nil, api.historyV3(batch), api._agg)
+		interHashStageCfg := stagedsync.StageTrieCfg(nil, !api.DisableStateRootCheck, false, false, api.dirs.Tmp, api._blockReader, nil, api.historyV3(batch), api._agg)
 		loader, err = stagedsync.UnwindIntermediateHashesForTrieLoader("eth_getProof", rl, unwindState, stageState, batch, interHashStageCfg, nil, nil, ctx.Done(), api.logger)
 		if err != nil {
 			return nil, err

turbo/jsonrpc/eth_callMany_test.go

@@ -87,7 +87,7 @@ func TestCallMany(t *testing.T) {
 	db := contractBackend.DB()
 	engine := contractBackend.Engine()
 	api := NewEthAPI(NewBaseApi(nil, stateCache, contractBackend.BlockReader(), contractBackend.Agg(), false, rpccfg.DefaultEvmCallTimeout, engine,
-		datadir.New(t.TempDir())), db, nil, nil, nil, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, log.New(), nil, 1000)
+		datadir.New(t.TempDir())), db, nil, nil, nil, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, log.New(), nil, 1000, false)
 
 	callArgAddr1 := ethapi.CallArgs{From: &address, To: &tokenAddr, Nonce: &nonce,
 		MaxPriorityFeePerGas: (*hexutil.Big)(big.NewInt(1e9)),

turbo/jsonrpc/eth_mining_test.go

@@ -30,7 +30,7 @@ func TestPendingBlock(t *testing.T) {
 	stateCache := kvcache.New(kvcache.DefaultCoherentConfig)
 	engine := ethash.NewFaker()
 	api := NewEthAPI(NewBaseApi(ff, stateCache, m.BlockReader, nil, false, rpccfg.DefaultEvmCallTimeout, engine,
-		m.Dirs), nil, nil, nil, mining, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, log.New(), nil, 1000)
+		m.Dirs), nil, nil, nil, mining, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, log.New(), nil, 1000, false)
 	expect := uint64(12345)
 	b, err := rlp.EncodeToBytes(types.NewBlockWithHeader(&types.Header{Number: big.NewInt(int64(expect))}))
 	require.NoError(t, err)

turbo/jsonrpc/get_logs_test.go

@@ -27,7 +27,7 @@ func TestGetLogsWithRange(t *testing.T) {
 	db := contractBackend.DB()
 	agg := contractBackend.Agg()
 	baseApi := NewBaseApi(nil, stateCache, contractBackend.BlockReader(), agg, false, rpccfg.DefaultEvmCallTimeout, contractBackend.Engine(), datadir.New(t.TempDir()))
-	ethImpl := NewEthAPI(baseApi, db, nil, nil, nil, 5000000, 100_000, 100_000, &ethconfig.Defaults, false, 100, 100, log.New(), defaultL1GasPriceTracker, logsMaxRange)
+	ethImpl := NewEthAPI(baseApi, db, nil, nil, nil, 5000000, 100_000, 100_000, &ethconfig.Defaults, false, 100, 100, log.New(), defaultL1GasPriceTracker, logsMaxRange, false)
 
 	scenarios := []struct {
 		name          string

turbo/jsonrpc/send_transaction_test.go

@@ -89,7 +89,7 @@ func TestSendRawTransaction(t *testing.T) {
 	ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, mockSentry)
 	txPool := txpool.NewTxpoolClient(conn)
 	ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}, mockSentry.Log)
-	api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, logger, nil, 1000)
+	api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, logger, nil, 1000, false)
 	api.BadTxAllowance = 1
 
 	buf := bytes.NewBuffer(nil)
@@ -149,7 +149,7 @@ func TestSendRawTransactionUnprotected(t *testing.T) {
 	ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, mockSentry)
 	txPool := txpool.NewTxpoolClient(conn)
 	ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}, mockSentry.Log)
-	api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, logger, nil, 1000)
+	api := jsonrpc.NewEthAPI(newBaseApiForTest(mockSentry), mockSentry.DB, nil, txPool, nil, 5000000, 1e18, 100_000, &ethconfig.Defaults, false, 100_000, 128, logger, nil, 1000, false)
 	api.BadTxAllowance = 1
 
 	// Enable unproteced txs flag

turbo/jsonrpc/zkevm_api.go

@@ -1634,7 +1634,7 @@ func (zkapi *ZkEvmAPIImpl) GetProof(ctx context.Context, address common.Address,
 		unwindState := &stagedsync.UnwindState{UnwindPoint: blockNr}
 		stageState := &stagedsync.StageState{BlockNumber: latestBlock}
 
-		interHashStageCfg := zkStages.StageZkInterHashesCfg(nil, true, true, false, api.dirs.Tmp, api._blockReader, nil, api.historyV3(tx), api._agg, nil)
+		interHashStageCfg := zkStages.StageZkInterHashesCfg(nil, !api.DisableStateRootCheck, true, false, api.dirs.Tmp, api._blockReader, nil, api.historyV3(tx), api._agg, nil)
 
 		if err = zkStages.UnwindZkIntermediateHashesStage(unwindState, stageState, batch, interHashStageCfg, ctx, true); err != nil {
 			return nil, fmt.Errorf("unwind intermediate hashes: %w", err)