diff --git a/cmd/rpcdaemon/commands/call_traces_test.go b/cmd/rpcdaemon/commands/call_traces_test.go index 730410db4ea..d94c24ebb2c 100644 --- a/cmd/rpcdaemon/commands/call_traces_test.go +++ b/cmd/rpcdaemon/commands/call_traces_test.go @@ -8,6 +8,7 @@ import ( "github.com/holiman/uint256" jsoniter "github.com/json-iterator/go" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -56,7 +57,7 @@ func TestCallTraceOneByOne(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) api := NewTraceAPI( - NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), + NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) // Insert blocks 1 by 1, to tirgget possible "off by one" errors for i := 0; i < chain.Length(); i++ { @@ -104,7 +105,8 @@ func TestCallTraceUnwind(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) + if err = m.InsertChain(chainA); err != nil { t.Fatalf("inserting chainA: %v", err) } @@ -166,7 +168,7 @@ func TestFilterNoAddresses(t *testing.T) { } agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) // Insert blocks 1 by 1, to tirgget possible "off by one" errors for i := 0; i < chain.Length(); i++ { if err = m.InsertChain(chain.Slice(i, i+1)); err != nil { @@ -192,7 +194,7 @@ func TestFilterAddressIntersection(t *testing.T) { m := stages.Mock(t) agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) toAddress1, toAddress2, other := common.Address{1}, common.Address{2}, common.Address{3} diff --git a/cmd/rpcdaemon/commands/corner_cases_support_test.go b/cmd/rpcdaemon/commands/corner_cases_support_test.go index 52b3f0872df..4ad27e6cf83 100644 --- a/cmd/rpcdaemon/commands/corner_cases_support_test.go +++ b/cmd/rpcdaemon/commands/corner_cases_support_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/require" @@ -23,7 +24,7 @@ func TestNotFoundMustReturnNil(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) api := NewEthAPI( - NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), + NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) ctx := context.Background() diff --git a/cmd/rpcdaemon/commands/daemon.go b/cmd/rpcdaemon/commands/daemon.go index 87f30d6c909..0d62fe51656 100644 --- a/cmd/rpcdaemon/commands/daemon.go +++ b/cmd/rpcdaemon/commands/daemon.go @@ -17,7 +17,7 @@ func APIList(db kv.RoDB, borDb kv.RoDB, eth rpchelper.ApiBackend, txPool txpool. filters *rpchelper.Filters, stateCache kvcache.Cache, blockReader services.FullBlockReader, agg *libstate.AggregatorV3, cfg httpcfg.HttpCfg, engine consensus.EngineReader, ) (list []rpc.API) { - base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine) + base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs) ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.ReturnDataLimit) erigonImpl := NewErigonAPI(base, db, eth) txpoolImpl := NewTxPoolAPI(base, db, txPool) @@ -138,7 +138,7 @@ func AuthAPIList(db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClien agg *libstate.AggregatorV3, cfg httpcfg.HttpCfg, engine consensus.EngineReader, ) (list []rpc.API) { - base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine) + base := NewBaseApi(filters, stateCache, blockReader, agg, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs) ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.ReturnDataLimit) engineImpl := NewEngineAPI(base, db, eth, cfg.InternalCL) diff --git a/cmd/rpcdaemon/commands/debug_api_test.go b/cmd/rpcdaemon/commands/debug_api_test.go index 4d72128b375..cb65c4b3ac6 100644 --- a/cmd/rpcdaemon/commands/debug_api_test.go +++ b/cmd/rpcdaemon/commands/debug_api_test.go @@ -9,6 +9,7 @@ import ( "github.com/davecgh/go-spew/spew" jsoniter "github.com/json-iterator/go" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/iter" "github.com/ledgerwatch/erigon-lib/kv/kvcache" @@ -55,7 +56,7 @@ func TestTraceBlockByNumber(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000) api := NewPrivateDebugAPI(baseApi, m.DB, 0) for _, tt := range debugTraceTransactionTests { @@ -104,7 +105,7 @@ func TestTraceBlockByHash(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000) api := NewPrivateDebugAPI(baseApi, m.DB, 0) for _, tt := range debugTraceTransactionTests { @@ -140,7 +141,7 @@ func TestTraceTransaction(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) api := NewPrivateDebugAPI(base, m.DB, 0) for _, tt := range debugTraceTransactionTests { var buf bytes.Buffer @@ -173,7 +174,7 @@ func TestTraceTransactionNoRefund(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) agg := m.HistoryV3Components() api := NewPrivateDebugAPI( - NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), + NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, 0) for _, tt := range debugTraceTransactionNoRefundTests { var buf bytes.Buffer @@ -207,7 +208,7 @@ func TestStorageRangeAt(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) agg := m.HistoryV3Components() api := NewPrivateDebugAPI( - NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), + NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, 0) t.Run("invalid addr", func(t *testing.T) { var block4 *types.Block @@ -304,7 +305,7 @@ func TestAccountRange(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) agg := m.HistoryV3Components() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) api := NewPrivateDebugAPI(base, m.DB, 0) t.Run("valid account", func(t *testing.T) { @@ -367,7 +368,7 @@ func TestGetModifiedAccountsByNumber(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) agg := m.HistoryV3Components() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) api := NewPrivateDebugAPI(base, m.DB, 0) t.Run("correct input", func(t *testing.T) { @@ -470,7 +471,7 @@ func TestAccountAt(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + base := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) api := NewPrivateDebugAPI(base, m.DB, 0) var blockHash0, blockHash1, blockHash3, blockHash10, blockHash12 common.Hash diff --git a/cmd/rpcdaemon/commands/erigon_receipts_test.go b/cmd/rpcdaemon/commands/erigon_receipts_test.go index 09f13604b8f..156ea819531 100644 --- a/cmd/rpcdaemon/commands/erigon_receipts_test.go +++ b/cmd/rpcdaemon/commands/erigon_receipts_test.go @@ -8,6 +8,7 @@ import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/assert" @@ -32,7 +33,7 @@ func TestGetLogs(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) agg := m.HistoryV3Components() - baseApi := NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + baseApi := NewBaseApi(nil, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) { ethApi := NewEthAPI(baseApi, m.DB, nil, nil, nil, 5000000, 100_000) @@ -67,7 +68,7 @@ func TestErigonGetLatestLogs(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) db := m.DB agg := m.HistoryV3Components() - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), db, nil) expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) expectedErigonLogs := make([]*types.ErigonLog, 0) @@ -102,7 +103,7 @@ func TestErigonGetLatestLogsIgnoreTopics(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) db := m.DB agg := m.HistoryV3Components() - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), db, nil) expectedLogs, _ := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(rpc.LatestBlockNumber.Int64())}) expectedErigonLogs := make([]*types.ErigonLog, 0) @@ -192,7 +193,7 @@ func TestGetBlockReceiptsByBlockHash(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil) expect := map[uint64]string{ 0: `[]`, diff --git a/cmd/rpcdaemon/commands/eth_api.go b/cmd/rpcdaemon/commands/eth_api.go index 9550abc8ac2..081fc485d88 100644 --- a/cmd/rpcdaemon/commands/eth_api.go +++ b/cmd/rpcdaemon/commands/eth_api.go @@ -14,6 +14,7 @@ import ( "github.com/ledgerwatch/erigon-lib/chain" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" @@ -120,9 +121,10 @@ type BaseAPI struct { _engine consensus.EngineReader evmCallTimeout time.Duration + dirs datadir.Dirs } -func NewBaseApi(f *rpchelper.Filters, stateCache kvcache.Cache, blockReader services.FullBlockReader, agg *libstate.AggregatorV3, singleNodeMode bool, evmCallTimeout time.Duration, engine consensus.EngineReader) *BaseAPI { +func NewBaseApi(f *rpchelper.Filters, stateCache kvcache.Cache, blockReader services.FullBlockReader, agg *libstate.AggregatorV3, singleNodeMode bool, evmCallTimeout time.Duration, engine consensus.EngineReader, dirs datadir.Dirs) *BaseAPI { blocksLRUSize := 128 // ~32Mb if !singleNodeMode { blocksLRUSize = 512 @@ -132,7 +134,7 @@ func NewBaseApi(f *rpchelper.Filters, stateCache kvcache.Cache, blockReader serv panic(err) } - return &BaseAPI{filters: f, stateCache: stateCache, blocksLRU: blocksLRU, _blockReader: blockReader, _txnReader: blockReader, _agg: agg, evmCallTimeout: evmCallTimeout, _engine: engine} + return &BaseAPI{filters: f, stateCache: stateCache, blocksLRU: blocksLRU, _blockReader: blockReader, _txnReader: blockReader, _agg: agg, evmCallTimeout: evmCallTimeout, _engine: engine, dirs: dirs} } func (api *BaseAPI) chainConfig(tx kv.Tx) (*chain.Config, error) { diff --git a/cmd/rpcdaemon/commands/eth_api_test.go b/cmd/rpcdaemon/commands/eth_api_test.go index 733a28c4294..e2965214111 100644 --- a/cmd/rpcdaemon/commands/eth_api_test.go +++ b/cmd/rpcdaemon/commands/eth_api_test.go @@ -7,6 +7,7 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/stretchr/testify/assert" "github.com/ledgerwatch/erigon/common/hexutil" @@ -29,7 +30,7 @@ func TestGetBalanceChangesInBlock(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) db := m.DB agg := m.HistoryV3Components() - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), db, nil) balances, err := api.GetBalanceChangesInBlock(context.Background(), myBlockNum) if err != nil { t.Errorf("calling GetBalanceChangesInBlock resulted in an error: %v", err) @@ -52,7 +53,7 @@ func TestGetTransactionReceipt(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), db, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), db, nil, nil, nil, 5000000, 100_000) // Call GetTransactionReceipt for transaction which is not in the database if _, err := api.GetTransactionReceipt(context.Background(), common.Hash{}); err != nil { t.Errorf("calling GetTransactionReceipt with empty hash: %v", err) @@ -64,7 +65,7 @@ func TestGetTransactionReceiptUnprotected(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) // Call GetTransactionReceipt for un-protected transaction if _, err := api.GetTransactionReceipt(context.Background(), common.HexToHash("0x3f3cb8a0e13ed2481f97f53f7095b9cbc78b6ffb779f2d3e565146371a8830ea")); err != nil { t.Errorf("calling GetTransactionReceipt for unprotected tx: %v", err) @@ -79,7 +80,7 @@ func TestGetStorageAt_ByBlockNumber_WithRequireCanonicalDefault(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithNumber(0)) @@ -96,7 +97,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), false)) @@ -113,7 +114,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") result, err := api.GetStorageAt(context.Background(), addr, "0x0", rpc.BlockNumberOrHashWithHash(m.Genesis.Hash(), true)) @@ -129,7 +130,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_BlockNotFoundError agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *core.BlockGen) { @@ -153,7 +154,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_BlockNotFoundError(t agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") offChain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 1, func(i int, block *core.BlockGen) { @@ -178,7 +179,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock( agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -200,7 +201,7 @@ func TestGetStorageAt_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t * agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) addr := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") orphanedBlock := orphanedChain[0].Blocks[0] @@ -219,7 +220,7 @@ func TestCall_ByBlockHash_WithRequireCanonicalDefault_NonCanonicalBlock(t *testi agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") @@ -245,7 +246,7 @@ func TestCall_ByBlockHash_WithRequireCanonicalTrue_NonCanonicalBlock(t *testing. agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) from := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") to := common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") diff --git a/cmd/rpcdaemon/commands/eth_block_test.go b/cmd/rpcdaemon/commands/eth_block_test.go index 7de1b60a299..f000128ec37 100644 --- a/cmd/rpcdaemon/commands/eth_block_test.go +++ b/cmd/rpcdaemon/commands/eth_block_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/assert" @@ -28,7 +29,7 @@ func TestGetBlockByNumberWithLatestTag(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) b, err := api.GetBlockByNumber(context.Background(), rpc.LatestBlockNumber, false) expected := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") if err != nil { @@ -61,7 +62,7 @@ func TestGetBlockByNumberWithLatestTag_WithHeadHashInDb(t *testing.T) { } tx.Commit() - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) block, err := api.GetBlockByNumber(ctx, rpc.LatestBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -93,7 +94,7 @@ func TestGetBlockByNumberWithPendingTag(t *testing.T) { RplBlock: rlpBlock, }) - api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) b, err := api.GetBlockByNumber(context.Background(), rpc.PendingBlockNumber, false) if err != nil { t.Errorf("error getting block number with pending tag: %s", err) @@ -107,7 +108,7 @@ func TestGetBlockByNumber_WithFinalizedTag_NoFinalizedBlockInDb(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) ctx := context.Background() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) if _, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false); err != nil { assert.ErrorIs(t, rpchelper.UnknownBlockError, err) } @@ -137,7 +138,7 @@ func TestGetBlockByNumber_WithFinalizedTag_WithFinalizedBlockInDb(t *testing.T) } tx.Commit() - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) block, err := api.GetBlockByNumber(ctx, rpc.FinalizedBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -152,7 +153,7 @@ func TestGetBlockByNumber_WithSafeTag_NoSafeBlockInDb(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) ctx := context.Background() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) if _, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false); err != nil { assert.ErrorIs(t, rpchelper.UnknownBlockError, err) } @@ -182,7 +183,7 @@ func TestGetBlockByNumber_WithSafeTag_WithSafeBlockInDb(t *testing.T) { } tx.Commit() - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) block, err := api.GetBlockByNumber(ctx, rpc.SafeBlockNumber, false) if err != nil { t.Errorf("error retrieving block by number: %s", err) @@ -198,7 +199,7 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { ctx := context.Background() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -234,7 +235,7 @@ func TestGetBlockTransactionCountByHash_ZeroTx(t *testing.T) { ctx := context.Background() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") tx, err := m.DB.BeginRw(ctx) @@ -269,7 +270,7 @@ func TestGetBlockTransactionCountByNumber(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) ctx := context.Background() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) blockHash := common.HexToHash("0x6804117de2f3e6ee32953e78ced1db7b20214e0d8c745a03b8fecf7cc8ee76ef") tx, err := m.DB.BeginRw(ctx) @@ -304,7 +305,7 @@ func TestGetBlockTransactionCountByNumber_ZeroTx(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) ctx := context.Background() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) blockHash := common.HexToHash("0x5883164d4100b95e1d8e931b8b9574586a1dea7507941e6ad3c1e3a2591485fd") diff --git a/cmd/rpcdaemon/commands/eth_call.go b/cmd/rpcdaemon/commands/eth_call.go index 62b83510f8e..79cbcb5eb0b 100644 --- a/cmd/rpcdaemon/commands/eth_call.go +++ b/cmd/rpcdaemon/commands/eth_call.go @@ -11,6 +11,7 @@ import ( "github.com/ledgerwatch/erigon-lib/gointerfaces" txpool_proto "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/memdb" types2 "github.com/ledgerwatch/erigon-lib/types" "github.com/ledgerwatch/log/v3" "google.golang.org/grpc" @@ -23,6 +24,7 @@ import ( "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/crypto" + "github.com/ledgerwatch/erigon/eth/stagedsync" "github.com/ledgerwatch/erigon/eth/tracers/logger" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rpc" @@ -303,7 +305,19 @@ func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi2.CallArgs return hexutil.Uint64(hi), nil } -// GetProof is partially implemented; no Storage proofs; only for the latest block +// maxGetProofRewindBlockCount limits the number of blocks into the past that +// GetProof will allow computing proofs. Because we must rewind the hash state +// and re-compute the state trie, the further back in time the request, the more +// computationally intensive the operation becomes. The staged sync code +// assumes that if more than 100_000 blocks are skipped, that the entire trie +// should be re-computed. Re-computing the entire trie will currently take ~15 +// minutes on mainnet. The current limit has been chosen arbitrarily as +// 'useful' without likely being overly computationally intense. This parameter +// could possibly be made configurable in the future if needed. +var maxGetProofRewindBlockCount uint64 = 1_000 + +// GetProof is partially implemented; no Storage proofs, and proofs must be for +// blocks within maxGetProofRewindBlockCount blocks of the head. func (api *APIImpl) GetProof(ctx context.Context, address libcommon.Address, storageKeys []libcommon.Hash, blockNrOrHash rpc.BlockNumberOrHash) (*accounts.AccProofResult, error) { tx, err := api.db.BeginRo(ctx) @@ -317,54 +331,89 @@ func (api *APIImpl) GetProof(ctx context.Context, address libcommon.Address, sto return nil, err } + header, err := api._blockReader.HeaderByNumber(ctx, tx, blockNr) + if err != nil { + return nil, err + } + latestBlock, err := rpchelper.GetLatestBlockNumber(tx) if err != nil { return nil, err - } else if blockNr != latestBlock { - return nil, fmt.Errorf(NotImplemented, "eth_getProof for block != latest") - } else if len(storageKeys) != 0 { + } + + if latestBlock < blockNr { + // shouldn't happen, but check anyway + return nil, fmt.Errorf("block number is in the future latest=%d requested=%d", latestBlock, blockNr) + } + + if len(storageKeys) != 0 { return nil, fmt.Errorf(NotImplemented, "eth_getProof with storageKeys") - } else { - addrHash, err := common.HashData(address[:]) - if err != nil { - return nil, err - } + } + + addrHash, err := common.HashData(address[:]) + if err != nil { + return nil, err + } - rl := trie.NewRetainList(0) - rl.AddKey(addrHash[:]) + rl := trie.NewRetainList(0) + rl.AddKey(addrHash[:]) - loader := trie.NewFlatDBTrieLoader("getProof") - if err := loader.Reset(rl, nil, nil, false); err != nil { - return nil, err + var loader *trie.FlatDBTrieLoader + if blockNr < latestBlock { + if latestBlock-blockNr > maxGetProofRewindBlockCount { + return nil, fmt.Errorf("requested block is too old, block must be within %d blocks of the head block number (currently %d)", maxGetProofRewindBlockCount, latestBlock) } + batch := memdb.NewMemoryBatch(tx, api.dirs.Tmp) + defer batch.Rollback() - var accProof accounts.AccProofResult - accProof.Address = address + unwindState := &stagedsync.UnwindState{UnwindPoint: blockNr} + stageState := &stagedsync.StageState{BlockNumber: latestBlock} - // Fill in the Account fields here to reduce the code changes - // needed in turbo/trie/hashbuilder.go - reader, err := rpchelper.CreateStateReader(ctx, tx, blockNrOrHash, 0, api.filters, api.stateCache, api.historyV3(tx), "") - if err != nil { + hashStageCfg := stagedsync.StageHashStateCfg(nil, api.dirs, api.historyV3(batch), api._agg) + if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx); err != nil { return nil, err } - a, err := reader.ReadAccountData(address) - if err != nil { - return nil, err - } - if a != nil { - accProof.Balance = (*hexutil.Big)(a.Balance.ToBig()) - accProof.CodeHash = a.CodeHash - accProof.Nonce = hexutil.Uint64(a.Nonce) - accProof.StorageHash = a.Root - } - loader.SetProofReturn(&accProof) - _, err = loader.CalcTrieRoot(tx, nil, nil) + interHashStageCfg := stagedsync.StageTrieCfg(nil, false, false, false, api.dirs.Tmp, api._blockReader, nil, api.historyV3(batch), api._agg) + loader, err = stagedsync.UnwindIntermediateHashesForTrieLoader("eth_getProof", rl, unwindState, stageState, batch, interHashStageCfg, nil, nil, ctx.Done()) if err != nil { return nil, err } - return &accProof, nil + tx = batch + } else { + loader = trie.NewFlatDBTrieLoader("eth_getProof", rl, nil, nil, false) + } + + var accProof accounts.AccProofResult + accProof.Address = address + + // Fill in the Account fields here to reduce the code changes + // needed in turbo/trie/hashbuilder.go + reader, err := rpchelper.CreateStateReader(ctx, tx, blockNrOrHash, 0, api.filters, api.stateCache, api.historyV3(tx), "") + if err != nil { + return nil, err + } + a, err := reader.ReadAccountData(address) + if err != nil { + return nil, err + } + if a != nil { + accProof.Balance = (*hexutil.Big)(a.Balance.ToBig()) + accProof.CodeHash = a.CodeHash + accProof.Nonce = hexutil.Uint64(a.Nonce) + accProof.StorageHash = a.Root + } + + loader.SetProofReturn(&accProof) + root, err := loader.CalcTrieRoot(tx, nil) + if err != nil { + return nil, err + } + + if root != header.Root { + return nil, fmt.Errorf("mismatch in expected state root computed %v vs %v indicates bug in proof implementation", root, header.Root) } + return &accProof, nil } func (api *APIImpl) tryBlockFromLru(hash libcommon.Hash) *types.Block { diff --git a/cmd/rpcdaemon/commands/eth_callMany_test.go b/cmd/rpcdaemon/commands/eth_callMany_test.go index b86b04cac1a..6c7036f41d7 100644 --- a/cmd/rpcdaemon/commands/eth_callMany_test.go +++ b/cmd/rpcdaemon/commands/eth_callMany_test.go @@ -8,6 +8,7 @@ import ( "strconv" "testing" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon/accounts/abi/bind" "github.com/ledgerwatch/erigon/accounts/abi/bind/backends" @@ -79,7 +80,7 @@ func TestCallMany(t *testing.T) { db := contractBackend.DB() engine := contractBackend.Engine() - api := NewEthAPI(NewBaseApi(nil, stateCache, contractBackend.BlockReader(), contractBackend.Agg(), false, rpccfg.DefaultEvmCallTimeout, engine), db, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, contractBackend.BlockReader(), contractBackend.Agg(), false, rpccfg.DefaultEvmCallTimeout, engine, datadir.New(t.TempDir())), db, nil, nil, nil, 5000000, 100_000) callArgAddr1 := ethapi.CallArgs{From: &address, To: &tokenAddr, Nonce: &nonce, MaxPriorityFeePerGas: (*hexutil.Big)(big.NewInt(1e9)), diff --git a/cmd/rpcdaemon/commands/eth_call_test.go b/cmd/rpcdaemon/commands/eth_call_test.go index 4522eba1304..ec2a572f9db 100644 --- a/cmd/rpcdaemon/commands/eth_call_test.go +++ b/cmd/rpcdaemon/commands/eth_call_test.go @@ -8,6 +8,7 @@ import ( "time" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon/turbo/trie" @@ -47,7 +48,7 @@ func TestEstimateGas(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, stages.Mock(t)) mining := txpool.NewMiningClient(conn) ff := rpchelper.New(ctx, nil, nil, mining, func() {}) - api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) var from = libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = libcommon.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") if _, err := api.EstimateGas(context.Background(), ðapi.CallArgs{ @@ -63,7 +64,7 @@ func TestEthCallNonCanonical(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) var from = libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") var to = libcommon.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e") if _, err := api.Call(context.Background(), ethapi.CallArgs{ @@ -88,7 +89,7 @@ func TestEthCallToPrunedBlock(t *testing.T) { agg := m.HistoryV3Components() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) callData := hexutil.MustDecode("0x2e64cec1") callDataBytes := hexutil.Bytes(callData) @@ -250,17 +251,15 @@ func verifyStorageProof(t *testing.T, storageRoot libcommon.Hash, proof accounts } func TestGetProof(t *testing.T) { - pruneTo := uint64(3) + maxGetProofRewindBlockCount = 1 // Note, this is unsafe for parallel tests, but, this test is the only consumer for now m, bankAddress, _ := chainWithDeployedContract(t) br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - doPrune(t, m.DB, pruneTo) - agg := m.HistoryV3Components() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) tests := []struct { name string @@ -270,18 +269,22 @@ func TestGetProof(t *testing.T) { }{ { name: "currentBlock", - blockNum: 2, + blockNum: 3, }, { name: "withState", - blockNum: 2, + blockNum: 3, storageKeys: []libcommon.Hash{{1}}, expectedErr: "the method is currently not implemented: eth_getProof with storageKeys", }, { - name: "olderBlock", + name: "olderBlock", + blockNum: 2, + }, + { + name: "tooOldBlock", blockNum: 1, - expectedErr: "the method is currently not implemented: eth_getProof for block != latest", + expectedErr: "requested block is too old, block must be within 1 blocks of the head block number (currently 3)", }, } @@ -298,6 +301,8 @@ func TestGetProof(t *testing.T) { require.Nil(t, proof) return } + require.NoError(t, err) + require.NotNil(t, proof) tx, err := m.DB.BeginRo(context.Background()) assert.NoError(t, err) @@ -305,9 +310,6 @@ func TestGetProof(t *testing.T) { header, err := api.headerByRPCNumber(rpc.BlockNumber(tt.blockNum), tx) require.NoError(t, err) - require.NoError(t, err) - require.NotNil(t, proof) - require.Equal(t, bankAddress, proof.Address) verifyAccountProof(t, header.Root, proof) @@ -336,7 +338,7 @@ func TestGetBlockByTimestampLatestTime(t *testing.T) { defer tx.Rollback() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil) latestBlock := rawdb.ReadCurrentBlock(tx) response, err := ethapi.RPCMarshalBlockDeprecated(latestBlock, true, false) @@ -374,7 +376,7 @@ func TestGetBlockByTimestampOldestTime(t *testing.T) { defer tx.Rollback() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil) oldestBlock, err := rawdb.ReadBlockByNumber(tx, 0) if err != nil { @@ -416,7 +418,7 @@ func TestGetBlockByTimeHigherThanLatestBlock(t *testing.T) { defer tx.Rollback() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil) latestBlock := rawdb.ReadCurrentBlock(tx) @@ -455,7 +457,7 @@ func TestGetBlockByTimeMiddle(t *testing.T) { defer tx.Rollback() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil) currentHeader := rawdb.ReadCurrentHeader(tx) oldestHeader, err := api._blockReader.HeaderByNumber(ctx, tx, 0) @@ -489,7 +491,6 @@ func TestGetBlockByTimeMiddle(t *testing.T) { if err != nil { t.Errorf("couldn't retrieve block %v", err) } - if block["timestamp"] != response["timestamp"] || block["hash"] != response["hash"] { t.Errorf("Retrieved the wrong block.\nexpected block hash: %s expected timestamp: %d\nblock hash retrieved: %s timestamp retrieved: %d", response["hash"], response["timestamp"], block["hash"], block["timestamp"]) } @@ -507,7 +508,7 @@ func TestGetBlockByTimestamp(t *testing.T) { defer tx.Rollback() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil) + api := NewErigonAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil) highestBlockNumber := rawdb.ReadCurrentHeader(tx).Number pickedBlock, err := rawdb.ReadBlockByNumber(tx, highestBlockNumber.Uint64()/3) @@ -558,7 +559,7 @@ func chainWithDeployedContract(t *testing.T) (*stages.MockSentry, libcommon.Addr var contractAddr libcommon.Address - chain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 2, func(i int, block *core.BlockGen) { + chain, err := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 3, func(i int, block *core.BlockGen) { nonce := block.TxNonce(bankAddress) switch i { case 0: @@ -566,7 +567,7 @@ func chainWithDeployedContract(t *testing.T) (*stages.MockSentry, libcommon.Addr assert.NoError(t, err) block.AddTx(tx) contractAddr = crypto.CreateAddress(bankAddress, nonce) - case 1: + case 1, 2: txn, err := types.SignTx(types.NewTransaction(nonce, contractAddr, new(uint256.Int), 90000, new(uint256.Int), nil), *signer, bankKey) assert.NoError(t, err) block.AddTx(txn) diff --git a/cmd/rpcdaemon/commands/eth_filters_test.go b/cmd/rpcdaemon/commands/eth_filters_test.go index fd50a8274d6..624fe9c686c 100644 --- a/cmd/rpcdaemon/commands/eth_filters_test.go +++ b/cmd/rpcdaemon/commands/eth_filters_test.go @@ -7,6 +7,7 @@ import ( "time" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon/rpc/rpccfg" @@ -31,7 +32,7 @@ func TestNewFilters(t *testing.T) { ctx, conn := rpcdaemontest.CreateTestGrpcConn(t, stages.Mock(t)) mining := txpool.NewMiningClient(conn) ff := rpchelper.New(ctx, nil, nil, mining, func() {}) - api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, nil, nil, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(ff, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, nil, nil, 5000000, 100_000) ptf, err := api.NewPendingTransactionFilter(ctx) assert.Nil(err) diff --git a/cmd/rpcdaemon/commands/eth_mining_test.go b/cmd/rpcdaemon/commands/eth_mining_test.go index 1a782599d90..47e9235eaec 100644 --- a/cmd/rpcdaemon/commands/eth_mining_test.go +++ b/cmd/rpcdaemon/commands/eth_mining_test.go @@ -8,6 +8,7 @@ import ( "github.com/ledgerwatch/erigon/consensus/ethash" "github.com/ledgerwatch/erigon/rpc/rpccfg" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" @@ -26,7 +27,7 @@ func TestPendingBlock(t *testing.T) { ff := rpchelper.New(ctx, nil, nil, mining, func() {}) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) engine := ethash.NewFaker() - api := NewEthAPI(NewBaseApi(ff, stateCache, snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3), nil, false, rpccfg.DefaultEvmCallTimeout, engine), nil, nil, nil, mining, 5000000, 100_000) + api := NewEthAPI(NewBaseApi(ff, stateCache, snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3), nil, false, rpccfg.DefaultEvmCallTimeout, engine, datadir.New(t.TempDir())), nil, nil, nil, mining, 5000000, 100_000) expect := uint64(12345) b, err := rlp.EncodeToBytes(types.NewBlockWithHeader(&types.Header{Number: big.NewInt(int64(expect))})) require.NoError(t, err) diff --git a/cmd/rpcdaemon/commands/eth_system_test.go b/cmd/rpcdaemon/commands/eth_system_test.go index 41f823a96fd..dd0f6ec6e83 100644 --- a/cmd/rpcdaemon/commands/eth_system_test.go +++ b/cmd/rpcdaemon/commands/eth_system_test.go @@ -8,6 +8,7 @@ import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon/rpc/rpccfg" @@ -44,7 +45,7 @@ func TestGasPrice(t *testing.T) { m := createGasPriceTestKV(t, testCase.chainSize) defer m.DB.Close() stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - base := NewBaseApi(nil, stateCache, snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3), nil, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + base := NewBaseApi(nil, stateCache, snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3), nil, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) eth := NewEthAPI(base, m.DB, nil, nil, nil, 5000000, 100_000) ctx := context.Background() diff --git a/cmd/rpcdaemon/commands/gen_traces_test.go b/cmd/rpcdaemon/commands/gen_traces_test.go index 241acf35d56..905894f7b3f 100644 --- a/cmd/rpcdaemon/commands/gen_traces_test.go +++ b/cmd/rpcdaemon/commands/gen_traces_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/cli/httpcfg" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" @@ -32,7 +33,7 @@ func TestGeneratedDebugApi(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) api := NewPrivateDebugAPI(baseApi, m.DB, 0) var buf bytes.Buffer stream := jsoniter.NewStream(jsoniter.ConfigDefault, &buf, 4096) @@ -120,7 +121,7 @@ func TestGeneratedTraceApi(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) api := NewTraceAPI(baseApi, m.DB, &httpcfg.HttpCfg{}) traces, err := api.Block(context.Background(), rpc.BlockNumber(1)) if err != nil { @@ -279,7 +280,7 @@ func TestGeneratedTraceApiCollision(t *testing.T) { agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) + baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())) api := NewTraceAPI(baseApi, m.DB, &httpcfg.HttpCfg{}) traces, err := api.Transaction(context.Background(), common.HexToHash("0xb2b9fa4c999c1c8370ce1fbd1c4315a9ce7f8421fe2ebed8a9051ff2e4e7e3da")) if err != nil { diff --git a/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go b/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go index 34157f0b00b..e416f75951a 100644 --- a/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go +++ b/cmd/rpcdaemon/commands/otterscan_contract_creator_test.go @@ -4,6 +4,7 @@ import ( "testing" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/snapshotsync" @@ -14,7 +15,7 @@ func TestGetContractCreator(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB) + api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB) addr := libcommon.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") expectCreator := libcommon.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") diff --git a/cmd/rpcdaemon/commands/otterscan_search_backward_test.go b/cmd/rpcdaemon/commands/otterscan_search_backward_test.go index 477be4b8c29..ac2532aa7a5 100644 --- a/cmd/rpcdaemon/commands/otterscan_search_backward_test.go +++ b/cmd/rpcdaemon/commands/otterscan_search_backward_test.go @@ -6,6 +6,7 @@ import ( "github.com/RoaringBitmap/roaring/roaring64" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/rpc/rpccfg" @@ -152,7 +153,7 @@ func TestSearchTransactionsBefore(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB) + api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB) addr := libcommon.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") t.Run("small page size", func(t *testing.T) { diff --git a/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go b/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go index 131bcbf0a82..be5fa4c4933 100644 --- a/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go +++ b/cmd/rpcdaemon/commands/otterscan_transaction_by_sender_and_nonce_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/ledgerwatch/erigon/rpc/rpccfg" "github.com/ledgerwatch/erigon/turbo/snapshotsync" @@ -14,7 +15,7 @@ func TestGetTransactionBySenderAndNonce(t *testing.T) { m, _, _ := rpcdaemontest.CreateTestSentry(t) agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB) + api := NewOtterscanAPI(NewBaseApi(nil, nil, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB) addr := common.HexToAddress("0x537e697c7ab75a26f9ecf0ce810e3154dfcaaf44") expectCreator := common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") diff --git a/cmd/rpcdaemon/commands/send_transaction_test.go b/cmd/rpcdaemon/commands/send_transaction_test.go index ad855fe0172..b4ee5eda7bd 100644 --- a/cmd/rpcdaemon/commands/send_transaction_test.go +++ b/cmd/rpcdaemon/commands/send_transaction_test.go @@ -8,6 +8,7 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv/kvcache" @@ -74,7 +75,7 @@ func TestSendRawTransaction(t *testing.T) { ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := commands.NewEthAPI(commands.NewBaseApi(ff, stateCache, br, nil, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, nil, txPool, nil, 5000000, 100_000) + api := commands.NewEthAPI(commands.NewBaseApi(ff, stateCache, br, nil, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, nil, txPool, nil, 5000000, 100_000) buf := bytes.NewBuffer(nil) err = txn.MarshalBinary(buf) diff --git a/cmd/rpcdaemon/commands/trace_adhoc_test.go b/cmd/rpcdaemon/commands/trace_adhoc_test.go index 92f4aba64de..a8abfb56da4 100644 --- a/cmd/rpcdaemon/commands/trace_adhoc_test.go +++ b/cmd/rpcdaemon/commands/trace_adhoc_test.go @@ -6,6 +6,7 @@ import ( "testing" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/stretchr/testify/require" @@ -25,7 +26,7 @@ func TestEmptyQuery(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage("[]"), &rpc.BlockNumberOrHash{BlockNumber: &latest}) @@ -45,7 +46,7 @@ func TestCoinbaseBalance(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database var latest = rpc.LatestBlockNumber results, err := api.CallMany(context.Background(), json.RawMessage(` @@ -75,7 +76,7 @@ func TestReplayTransaction(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) var txnHash libcommon.Hash if err := m.DB.View(context.Background(), func(tx kv.Tx) error { b, err := rawdb.ReadBlockByNumber(tx, 6) @@ -106,7 +107,7 @@ func TestReplayBlockTransactions(t *testing.T) { br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) stateCache := kvcache.New(kvcache.DefaultCoherentConfig) - api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, &httpcfg.HttpCfg{}) + api := NewTraceAPI(NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, &httpcfg.HttpCfg{}) // Call GetTransactionReceipt for transaction which is not in the database n := rpc.BlockNumber(6) diff --git a/cmd/rpcdaemon/commands/txpool_api_test.go b/cmd/rpcdaemon/commands/txpool_api_test.go index 9a1b0dd7d11..fb578628234 100644 --- a/cmd/rpcdaemon/commands/txpool_api_test.go +++ b/cmd/rpcdaemon/commands/txpool_api_test.go @@ -7,6 +7,7 @@ import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" txPoolProto "github.com/ledgerwatch/erigon-lib/gointerfaces/txpool" "github.com/ledgerwatch/erigon-lib/kv/kvcache" @@ -40,7 +41,7 @@ func TestTxPoolContent(t *testing.T) { ff := rpchelper.New(ctx, nil, txPool, txpool.NewMiningClient(conn), func() {}) agg := m.HistoryV3Components() br := snapshotsync.NewBlockReaderWithSnapshots(m.BlockSnapshots, m.TransactionsV3) - api := NewTxPoolAPI(NewBaseApi(ff, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine), m.DB, txPool) + api := NewTxPoolAPI(NewBaseApi(ff, kvcache.New(kvcache.DefaultCoherentConfig), br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine, datadir.New(t.TempDir())), m.DB, txPool) expectValue := uint64(1234) txn, err := types.SignTx(types.NewTransaction(0, libcommon.Address{1}, uint256.NewInt(expectValue), params.TxGas, uint256.NewInt(10*params.GWei), nil), *types.LatestSignerForChainID(m.ChainConfig.ChainID), m.Key) diff --git a/eth/stagedsync/stage_interhashes.go b/eth/stagedsync/stage_interhashes.go index 794c4730e09..777fd7ee9c1 100644 --- a/eth/stagedsync/stage_interhashes.go +++ b/eth/stagedsync/stage_interhashes.go @@ -163,11 +163,8 @@ func RegenerateIntermediateHashes(logPrefix string, db kv.RwTx, cfg TrieCfg, exp defer stTrieCollector.Close() stTrieCollectorFunc := storageTrieCollector(stTrieCollector) - loader := trie.NewFlatDBTrieLoader(logPrefix) - if err := loader.Reset(trie.NewRetainList(0), accTrieCollectorFunc, stTrieCollectorFunc, false); err != nil { - return trie.EmptyRoot, err - } - hash, err := loader.CalcTrieRoot(db, []byte{}, ctx.Done()) + loader := trie.NewFlatDBTrieLoader(logPrefix, trie.NewRetainList(0), accTrieCollectorFunc, stTrieCollectorFunc, false) + hash, err := loader.CalcTrieRoot(db, ctx.Done()) if err != nil { return trie.EmptyRoot, err } @@ -581,11 +578,8 @@ func incrementIntermediateHashes(logPrefix string, s *StageState, db kv.RwTx, to defer stTrieCollector.Close() stTrieCollectorFunc := storageTrieCollector(stTrieCollector) - loader := trie.NewFlatDBTrieLoader(logPrefix) - if err := loader.Reset(rl, accTrieCollectorFunc, stTrieCollectorFunc, false); err != nil { - return trie.EmptyRoot, err - } - hash, err := loader.CalcTrieRoot(db, []byte{}, quit) + loader := trie.NewFlatDBTrieLoader(logPrefix, rl, accTrieCollectorFunc, stTrieCollectorFunc, false) + hash, err := loader.CalcTrieRoot(db, quit) if err != nil { return trie.EmptyRoot, err } @@ -638,19 +632,18 @@ func UnwindIntermediateHashesStage(u *UnwindState, s *StageState, tx kv.RwTx, cf return nil } -func unwindIntermediateHashesStageImpl(logPrefix string, u *UnwindState, s *StageState, db kv.RwTx, cfg TrieCfg, expectedRootHash libcommon.Hash, quit <-chan struct{}) error { +func UnwindIntermediateHashesForTrieLoader(logPrefix string, rl *trie.RetainList, u *UnwindState, s *StageState, db kv.RwTx, cfg TrieCfg, accTrieCollectorFunc trie.HashCollector2, stTrieCollectorFunc trie.StorageHashCollector2, quit <-chan struct{}) (*trie.FlatDBTrieLoader, error) { p := NewHashPromoter(db, cfg.tmpDir, quit, logPrefix) - rl := trie.NewRetainList(0) if cfg.historyV3 { cfg.agg.SetTx(db) collect := func(k, v []byte) { rl.AddKeyWithMarker(k, len(v) == 0) } if err := p.UnwindOnHistoryV3(logPrefix, cfg.agg, s.BlockNumber, u.UnwindPoint, false, collect); err != nil { - return err + return nil, err } if err := p.UnwindOnHistoryV3(logPrefix, cfg.agg, s.BlockNumber, u.UnwindPoint, true, collect); err != nil { - return err + return nil, err } } else { collect := func(k, v []byte, _ etl.CurrentTableReader, _ etl.LoadNextFunc) error { @@ -658,13 +651,17 @@ func unwindIntermediateHashesStageImpl(logPrefix string, u *UnwindState, s *Stag return nil } if err := p.Unwind(logPrefix, s, u, false /* storage */, collect); err != nil { - return err + return nil, err } if err := p.Unwind(logPrefix, s, u, true /* storage */, collect); err != nil { - return err + return nil, err } } + return trie.NewFlatDBTrieLoader(logPrefix, rl, accTrieCollectorFunc, stTrieCollectorFunc, false), nil +} + +func unwindIntermediateHashesStageImpl(logPrefix string, u *UnwindState, s *StageState, db kv.RwTx, cfg TrieCfg, expectedRootHash libcommon.Hash, quit <-chan struct{}) error { accTrieCollector := etl.NewCollector(logPrefix, cfg.tmpDir, etl.NewSortableBuffer(etl.BufferOptimalSize)) defer accTrieCollector.Close() accTrieCollectorFunc := accountTrieCollector(accTrieCollector) @@ -673,11 +670,14 @@ func unwindIntermediateHashesStageImpl(logPrefix string, u *UnwindState, s *Stag defer stTrieCollector.Close() stTrieCollectorFunc := storageTrieCollector(stTrieCollector) - loader := trie.NewFlatDBTrieLoader(logPrefix) - if err := loader.Reset(rl, accTrieCollectorFunc, stTrieCollectorFunc, false); err != nil { + rl := trie.NewRetainList(0) + + loader, err := UnwindIntermediateHashesForTrieLoader(logPrefix, rl, u, s, db, cfg, accTrieCollectorFunc, stTrieCollectorFunc, quit) + if err != nil { return err } - hash, err := loader.CalcTrieRoot(db, []byte{}, quit) + + hash, err := loader.CalcTrieRoot(db, quit) if err != nil { return err } diff --git a/turbo/adapter/ethapi/get_proof.go b/turbo/adapter/ethapi/get_proof.go index 212bced404b..49832b7f910 100644 --- a/turbo/adapter/ethapi/get_proof.go +++ b/turbo/adapter/ethapi/get_proof.go @@ -230,7 +230,3 @@ func (r *Receiver) Receive( // We ran out of modifications, simply pass through return r.defaultReceiver.Receive(itemType, accountKey, storageKey, accountValue, storageValue, hash, hasTree, cutoff) } - -func (r *Receiver) Result() trie.SubTries { - return r.defaultReceiver.Result() -} diff --git a/turbo/trie/trie_root.go b/turbo/trie/trie_root.go index 91cd8c0a0c9..fdeede3744b 100644 --- a/turbo/trie/trie_root.go +++ b/turbo/trie/trie_root.go @@ -73,7 +73,7 @@ Then delete this account (SELFDESTRUCT). // FlatDBTrieLoader reads state and intermediate trie hashes in order equal to "Preorder trie traversal" // (Preorder - visit Root, visit Left, visit Right) // -// It produces stream of values and send this stream to `defaultReceiver` +// It produces stream of values and send this stream to `receiver` // It skips storage with incorrect incarnations // // Each intermediate hash key firstly pass to RetainDecider, only if it returns "false" - such AccTrie can be used. @@ -89,13 +89,9 @@ type FlatDBTrieLoader struct { // Account item buffer accountValue accounts.Account - receiver StreamReceiver - defaultReceiver *RootHashAggregator - hc HashCollector2 - shc StorageHashCollector2 - - // Optionally construct an Account Proof for an account key specified in 'rd' - accProofResult *accounts.AccProofResult + receiver *RootHashAggregator + hc HashCollector2 + shc StorageHashCollector2 } // RootHashAggregator - calculates Merkle trie root hash from incoming data stream @@ -134,60 +130,39 @@ type RootHashAggregator struct { cutoff bool } -type StreamReceiver interface { - Receive( - itemType StreamItem, - accountKey []byte, - storageKey []byte, - accountValue *accounts.Account, - storageValue []byte, - hash []byte, - hasTree bool, - cutoff int, - ) error - - Result() SubTries - Root() libcommon.Hash -} - func NewRootHashAggregator() *RootHashAggregator { return &RootHashAggregator{ hb: NewHashBuilder(false), } } -func NewFlatDBTrieLoader(logPrefix string) *FlatDBTrieLoader { - return &FlatDBTrieLoader{ - logPrefix: logPrefix, - defaultReceiver: NewRootHashAggregator(), - } -} - -// Reset prepares the loader for reuse -func (l *FlatDBTrieLoader) Reset(rd RetainDeciderWithMarker, hc HashCollector2, shc StorageHashCollector2, trace bool) error { - l.defaultReceiver.Reset(hc, shc, trace) - l.hc = hc - l.shc = shc - l.receiver = l.defaultReceiver - l.trace = trace - l.ihSeek, l.accSeek, l.storageSeek, l.kHex, l.kHexS = make([]byte, 0, 128), make([]byte, 0, 128), make([]byte, 0, 128), make([]byte, 0, 128), make([]byte, 0, 128) - l.rd = rd - if l.trace { +func NewFlatDBTrieLoader(logPrefix string, rd RetainDeciderWithMarker, hc HashCollector2, shc StorageHashCollector2, trace bool) *FlatDBTrieLoader { + if trace { fmt.Printf("----------\n") fmt.Printf("CalcTrieRoot\n") } - l.accProofResult = nil - return nil + return &FlatDBTrieLoader{ + logPrefix: logPrefix, + receiver: &RootHashAggregator{ + hb: NewHashBuilder(false), + hc: hc, + shc: shc, + trace: trace, + }, + ihSeek: make([]byte, 0, 128), + accSeek: make([]byte, 0, 128), + storageSeek: make([]byte, 0, 128), + kHex: make([]byte, 0, 128), + kHexS: make([]byte, 0, 128), + rd: rd, + hc: hc, + shc: shc, + } } func (l *FlatDBTrieLoader) SetProofReturn(accProofResult *accounts.AccProofResult) { - l.accProofResult = accProofResult - l.defaultReceiver.proofMatch = l.rd - l.defaultReceiver.hb.SetProofReturn(accProofResult) -} - -func (l *FlatDBTrieLoader) SetStreamReceiver(receiver StreamReceiver) { - l.receiver = receiver + l.receiver.proofMatch = l.rd + l.receiver.hb.SetProofReturn(accProofResult) } // CalcTrieRoot algo: @@ -212,7 +187,7 @@ func (l *FlatDBTrieLoader) SetStreamReceiver(receiver StreamReceiver) { // SkipAccounts: // use(AccTrie) // } -func (l *FlatDBTrieLoader) CalcTrieRoot(tx kv.Tx, prefix []byte, quit <-chan struct{}) (libcommon.Hash, error) { +func (l *FlatDBTrieLoader) CalcTrieRoot(tx kv.Tx, quit <-chan struct{}) (libcommon.Hash, error) { accC, err := tx.Cursor(kv.HashedAccounts) if err != nil { @@ -245,7 +220,7 @@ func (l *FlatDBTrieLoader) CalcTrieRoot(tx kv.Tx, prefix []byte, quit <-chan str defer ss.Close() logEvery := time.NewTicker(30 * time.Second) defer logEvery.Stop() - for ihK, ihV, hasTree, err := accTrie.AtPrefix(prefix); ; ihK, ihV, hasTree, err = accTrie.Next() { // no loop termination is at he end of loop + for ihK, ihV, hasTree, err := accTrie.AtPrefix(nil); ; ihK, ihV, hasTree, err = accTrie.Next() { // no loop termination is at he end of loop if err != nil { return EmptyRoot, err } @@ -257,7 +232,7 @@ func (l *FlatDBTrieLoader) CalcTrieRoot(tx kv.Tx, prefix []byte, quit <-chan str if err1 != nil { return EmptyRoot, err1 } - if keyIsBefore(ihK, kHex) || !bytes.HasPrefix(kHex, prefix) { // read all accounts until next AccTrie + if keyIsBefore(ihK, kHex) || !bytes.HasPrefix(kHex, nil) { // read all accounts until next AccTrie break } if err = l.accountValue.DecodeForStorage(v); err != nil { @@ -324,7 +299,7 @@ func (l *FlatDBTrieLoader) CalcTrieRoot(tx kv.Tx, prefix []byte, quit <-chan str } } - if err := l.receiver.Receive(CutoffStreamItem, nil, nil, nil, nil, nil, false, len(prefix)); err != nil { + if err := l.receiver.Receive(CutoffStreamItem, nil, nil, nil, nil, nil, false, 0); err != nil { return EmptyRoot, err } return l.receiver.Root(), nil @@ -344,28 +319,6 @@ func (r *RootHashAggregator) RetainNothing(_ []byte) bool { return false } -func (r *RootHashAggregator) Reset(hc HashCollector2, shc StorageHashCollector2, trace bool) { - r.hc = hc - r.shc = shc - r.curr.Reset() - r.succ.Reset() - r.value = nil - r.groups = r.groups[:0] - r.hasTree = r.hasTree[:0] - r.hasHash = r.hasHash[:0] - r.a.Reset() - r.hb.Reset() - r.wasIH = false - r.currStorage.Reset() - r.succStorage.Reset() - r.valueStorage = nil - r.wasIHStorage = false - r.root = libcommon.Hash{} - r.trace = trace - r.hb.trace = trace - r.proofMatch = nil -} - func (r *RootHashAggregator) Receive(itemType StreamItem, accountKey []byte, storageKey []byte, @@ -544,10 +497,6 @@ func (r *RootHashAggregator) Receive(itemType StreamItem, // } // } -func (r *RootHashAggregator) Result() SubTries { - panic("don't call me") -} - func (r *RootHashAggregator) Root() libcommon.Hash { return r.root } @@ -1514,12 +1463,9 @@ func CastTrieNodeValue(hashes, rootHash []byte) []libcommon.Hash { // CalcRoot is a combination of `ResolveStateTrie` and `UpdateStateTrie` // DESCRIBED: docs/programmers_guide/guide.md#organising-ethereum-state-into-a-merkle-tree func CalcRoot(logPrefix string, tx kv.Tx) (libcommon.Hash, error) { - loader := NewFlatDBTrieLoader(logPrefix) - if err := loader.Reset(NewRetainList(0), nil, nil, false); err != nil { - return EmptyRoot, err - } + loader := NewFlatDBTrieLoader(logPrefix, NewRetainList(0), nil, nil, false) - h, err := loader.CalcTrieRoot(tx, nil, nil) + h, err := loader.CalcTrieRoot(tx, nil) if err != nil { return EmptyRoot, err }