diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 556a0e7cbe1..f35ec298106 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -171,7 +171,7 @@ func runCmd(ctx *cli.Context) error { } else { genesisConfig = new(types.Genesis) } - agg, err := state2.NewAggregator(context.Background(), datadir.New(os.TempDir()), config3.DefaultStepSize, db, log.New()) + agg, err := state2.NewAggregator2(context.Background(), datadir.New(os.TempDir()), config3.DefaultStepSize, db, log.New()) if err != nil { return err } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 6b2270207bc..a231ca88576 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -141,7 +141,7 @@ func aggregateResultsFromStateTests( MustOpen() defer _db.Close() - agg, err := libstate.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, _db, log.New()) + agg, err := libstate.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, _db, log.New()) if err != nil { return nil, err } diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 6cd8da51232..dd709b75d8e 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -1338,7 +1338,7 @@ func allSnapshots(ctx context.Context, db kv.RoDB, logger log.Logger) (*freezebl blockReader := freezeblocks.NewBlockReader(_allSnapshotsSingleton, _allBorSnapshotsSingleton, _heimdallStoreSingleton, _bridgeStoreSingleton) txNums := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, blockReader)) - _aggSingleton, err = libstate.NewAggregator(ctx, dirs, config3.DefaultStepSize, db, logger) + _aggSingleton, err = libstate.NewAggregator2(ctx, dirs, config3.DefaultStepSize, db, logger) if err != nil { panic(err) } diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index a55ac8b7af3..66fad4cb9cb 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -424,7 +424,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger blockReader = freezeblocks.NewBlockReader(allSnapshots, allBorSnapshots, heimdallStore, bridgeStore) txNumsReader := rawdbv3.TxNums.WithCustomReadTxNumFunc(freezeblocks.ReadTxNumFuncFromBlockReader(ctx, blockReader)) - agg, err := libstate.NewAggregator(ctx, cfg.Dirs, config3.DefaultStepSize, rawDB, logger) + agg, err := libstate.NewAggregator2(ctx, cfg.Dirs, config3.DefaultStepSize, rawDB, logger) if err != nil { return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, fmt.Errorf("create aggregator: %w", err) } diff --git a/cmd/state/commands/opcode_tracer.go b/cmd/state/commands/opcode_tracer.go index a5133f2a46c..d0e7319f40c 100644 --- a/cmd/state/commands/opcode_tracer.go +++ b/cmd/state/commands/opcode_tracer.go @@ -436,7 +436,7 @@ func OpcodeTracer(genesis *types.Genesis, blockNum uint64, chaindata string, num rawChainDb := mdbx.MustOpen(dirs.Chaindata) defer rawChainDb.Close() - agg, err := state2.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, rawChainDb, log.New()) + agg, err := state2.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, rawChainDb, log.New()) if err != nil { return err } diff --git a/core/genesis_write.go b/core/genesis_write.go index 4d6b52767ee..ff34fc81614 100644 --- a/core/genesis_write.go +++ b/core/genesis_write.go @@ -505,7 +505,7 @@ func GenesisToBlock(g *types.Genesis, dirs datadir.Dirs, logger log.Logger) (*ty genesisTmpDB := mdbx.New(kv.TemporaryDB, logger).InMem(dirs.DataDir).MapSize(2 * datasize.GB).GrowthStep(1 * datasize.MB).MustOpen() defer genesisTmpDB.Close() - agg, err := state2.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, genesisTmpDB, logger) + agg, err := state2.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, genesisTmpDB, logger) if err != nil { return err } diff --git a/core/state/intra_block_state_test.go b/core/state/intra_block_state_test.go index f7d0767015c..0c463a473c7 100644 --- a/core/state/intra_block_state_test.go +++ b/core/state/intra_block_state_test.go @@ -241,7 +241,7 @@ func (test *snapshotTest) run() bool { db := memdb.NewStateDB("") defer db.Close() - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(""), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(""), 16, db, log.New()) if err != nil { test.err = err return false diff --git a/core/state/state_test.go b/core/state/state_test.go index 7e3ad911053..c1a3e561f08 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -122,7 +122,7 @@ func (s *StateSuite) SetUpTest(c *checker.C) { db := memdb.NewStateDB("") defer db.Close() - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(""), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(""), 16, db, log.New()) if err != nil { panic(err) } @@ -394,7 +394,7 @@ func NewTestTemporalDb(tb testing.TB) (kv.TemporalRwDB, kv.TemporalRwTx, *state. db := memdb.NewStateDB(tb.TempDir()) tb.Cleanup(db.Close) - agg, err := state.NewAggregator(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) + agg, err := state.NewAggregator2(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) if err != nil { tb.Fatal(err) } diff --git a/core/test/domains_restart_test.go b/core/test/domains_restart_test.go index 952906fbb6b..d5ae4a7f3dd 100644 --- a/core/test/domains_restart_test.go +++ b/core/test/domains_restart_test.go @@ -66,7 +66,7 @@ func testDbAndAggregatorv3(t *testing.T, fpath string, aggStep uint64) (kv.RwDB, db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).MustOpen() t.Cleanup(db.Close) - agg, err := state.NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := state.NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(t, err) t.Cleanup(agg.Close) err = agg.OpenFolder() diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 95677431ac5..3a2bd414611 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -104,7 +104,7 @@ func testTemporalDB(t *testing.T) *temporal.DB { t.Cleanup(db.Close) - agg, err := state3.NewAggregator(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) + agg, err := state3.NewAggregator2(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) require.NoError(t, err) t.Cleanup(agg.Close) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 77748306170..cdf3d26dd89 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -131,7 +131,7 @@ func Execute(code, input []byte, cfg *Config, tempdir string) ([]byte, *state.In if !externalState { db := memdb.NewStateDB(tempdir) defer db.Close() - agg, err := state3.NewAggregator(context.Background(), datadir.New(tempdir), config3.DefaultStepSize, db, log.New()) + agg, err := state3.NewAggregator2(context.Background(), datadir.New(tempdir), config3.DefaultStepSize, db, log.New()) if err != nil { return nil, nil, err } @@ -193,7 +193,7 @@ func Create(input []byte, cfg *Config, blockNr uint64) ([]byte, libcommon.Addres db := memdb.NewStateDB(tmp) defer db.Close() - agg, err := state3.NewAggregator(context.Background(), datadir.New(tmp), config3.DefaultStepSize, db, log.New()) + agg, err := state3.NewAggregator2(context.Background(), datadir.New(tmp), config3.DefaultStepSize, db, log.New()) if err != nil { return nil, [20]byte{}, 0, err } diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index f3521a1dcf7..9df1c03717b 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -56,7 +56,7 @@ func NewTestTemporalDb(tb testing.TB) (kv.RwDB, kv.RwTx, *stateLib.Aggregator) { db := memdb.NewStateDB(tb.TempDir()) tb.Cleanup(db.Close) - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) if err != nil { tb.Fatal(err) } @@ -177,7 +177,7 @@ func testTemporalDB(t testing.TB) *temporal.DB { t.Cleanup(db.Close) - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(t.TempDir()), 16, db, log.New()) require.NoError(t, err) t.Cleanup(agg.Close) diff --git a/erigon-lib/kv/membatchwithdb/memory_mutation_test.go b/erigon-lib/kv/membatchwithdb/memory_mutation_test.go index cd4ce3cd532..7183bd46fa5 100644 --- a/erigon-lib/kv/membatchwithdb/memory_mutation_test.go +++ b/erigon-lib/kv/membatchwithdb/memory_mutation_test.go @@ -212,7 +212,7 @@ func NewTestTemporalDb(tb testing.TB) (kv.RwDB, kv.RwTx, *stateLib.Aggregator) { db := memdb.NewStateDB(tb.TempDir()) tb.Cleanup(db.Close) - agg, err := stateLib.NewAggregator(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) + agg, err := stateLib.NewAggregator2(context.Background(), datadir.New(tb.TempDir()), 16, db, log.New()) if err != nil { tb.Fatal(err) } diff --git a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go index aba7fec9396..6891112caa8 100644 --- a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go +++ b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go @@ -42,7 +42,7 @@ func NewTestDB(tb testing.TB, dirs datadir.Dirs) (kv.TemporalRwDB, *state.Aggreg rawDB = memdb.New(dirs.DataDir, kv.ChainDB) } - agg, err := state.NewAggregator(context.Background(), dirs, config3.DefaultStepSize, rawDB, log.New()) + agg, err := state.NewAggregator2(context.Background(), dirs, config3.DefaultStepSize, rawDB, log.New()) if err != nil { panic(err) } diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 26f147c3958..a589a6e1687 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -126,141 +126,14 @@ func domainIntegrityCheck(name kv.Domain, dirs datadir.Dirs, fromStep, toStep ui } } -var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false) - -func init() { - if dbgCommBtIndex { - cfg := Schema[kv.CommitmentDomain] - cfg.IndexList = AccessorBTree | AccessorExistence - Schema[kv.CommitmentDomain] = cfg - } -} - -var Schema = map[kv.Domain]domainCfg{ - kv.AccountsDomain: { - name: kv.AccountsDomain, valuesTable: kv.TblAccountVals, - - IndexList: AccessorBTree | AccessorExistence, - crossDomainIntegrity: domainIntegrityCheck, - compression: seg.CompressNone, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblAccountHistoryVals, - compression: seg.CompressNone, - - historyLargeValues: false, - filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant - - iiCfg: iiCfg{ - keysTable: kv.TblAccountHistoryKeys, valuesTable: kv.TblAccountIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant - }, - }, - }, - kv.StorageDomain: { - name: kv.StorageDomain, valuesTable: kv.TblStorageVals, - - IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressKeys, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblStorageHistoryVals, - compression: seg.CompressNone, - - historyLargeValues: false, - filenameBase: kv.StorageDomain.String(), - - iiCfg: iiCfg{ - keysTable: kv.TblStorageHistoryKeys, valuesTable: kv.TblStorageIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.StorageDomain.String(), - }, - }, - }, - kv.CodeDomain: { - name: kv.CodeDomain, valuesTable: kv.TblCodeVals, - - IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet - compressCfg: DomainCompressCfg, - largeValues: true, - - hist: histCfg{ - valuesTable: kv.TblCodeHistoryVals, - compression: seg.CompressKeys | seg.CompressVals, - - historyLargeValues: true, - filenameBase: kv.CodeDomain.String(), - - iiCfg: iiCfg{ - withExistence: false, compressorCfg: seg.DefaultCfg, - keysTable: kv.TblCodeHistoryKeys, valuesTable: kv.TblCodeIdx, - filenameBase: kv.CodeDomain.String(), - }, - }, - }, - kv.CommitmentDomain: { - name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, - - IndexList: AccessorHashMap, - compression: seg.CompressKeys, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblCommitmentHistoryVals, - compression: seg.CompressNone, - - snapshotsDisabled: true, - historyLargeValues: false, - filenameBase: kv.CommitmentDomain.String(), - - iiCfg: iiCfg{ - keysTable: kv.TblCommitmentHistoryKeys, valuesTable: kv.TblCommitmentIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.CommitmentDomain.String(), - }, - }, - }, - kv.ReceiptDomain: { - name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals, - - IndexList: AccessorBTree | AccessorExistence, - compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, - compressCfg: DomainCompressCfg, - - hist: histCfg{ - valuesTable: kv.TblReceiptHistoryVals, - compression: seg.CompressNone, - - historyLargeValues: false, - filenameBase: kv.ReceiptDomain.String(), - - iiCfg: iiCfg{ - keysTable: kv.TblReceiptHistoryKeys, valuesTable: kv.TblReceiptIdx, - withExistence: false, compressorCfg: seg.DefaultCfg, - filenameBase: kv.ReceiptDomain.String(), - }, - }, - }, -} - func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint64, db kv.RoDB, logger log.Logger) (*Aggregator, error) { - tmpdir := dirs.Tmp - salt, err := getStateIndicesSalt(dirs.Snap) - if err != nil { - return nil, err - } - ctx, ctxCancel := context.WithCancel(ctx) - a := &Aggregator{ + return &Aggregator{ ctx: ctx, ctxCancel: ctxCancel, onFreeze: func(frozenFileNames []string) {}, dirs: dirs, - tmpdir: tmpdir, + tmpdir: dirs.Tmp, aggregationStep: aggregationStep, db: db, leakDetector: dbg.NewLeakDetector("agg", dbg.SlowTx()), @@ -272,39 +145,7 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6 commitmentValuesTransform: AggregatorSqueezeCommitmentValues, produce: true, - } - - if err := a.registerDomain(kv.AccountsDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.StorageDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.CodeDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.CommitmentDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerDomain(kv.ReceiptDomain, salt, dirs, aggregationStep, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.LogAddrIdxPos, salt, dirs, aggregationStep, kv.FileLogAddressIdx, kv.TblLogAddressKeys, kv.TblLogAddressIdx, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.LogTopicIdxPos, salt, dirs, aggregationStep, kv.FileLogTopicsIdx, kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.TracesFromIdxPos, salt, dirs, aggregationStep, kv.FileTracesFromIdx, kv.TblTracesFromKeys, kv.TblTracesFromIdx, logger); err != nil { - return nil, err - } - if err := a.registerII(kv.TracesToIdxPos, salt, dirs, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil { - return nil, err - } - a.KeepRecentTxnsOfHistoriesWithDisabledSnapshots(100_000) // ~1k blocks of history - a.recalcVisibleFiles(a.DirtyFilesEndTxNumMinimax()) - - return a, nil + }, nil } // getStateIndicesSalt - try read salt for all indices from DB. Or fall-back to new salt creation. diff --git a/erigon-lib/state/aggregator2.go b/erigon-lib/state/aggregator2.go new file mode 100644 index 00000000000..f1f51d09121 --- /dev/null +++ b/erigon-lib/state/aggregator2.go @@ -0,0 +1,177 @@ +package state + +import ( + "context" + + "github.com/erigontech/erigon-lib/common/datadir" + "github.com/erigontech/erigon-lib/common/dbg" + "github.com/erigontech/erigon-lib/kv" + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/seg" +) + +// this is supposed to register domains/iis + +func NewAggregator2(ctx context.Context, dirs datadir.Dirs, aggregationStep uint64, db kv.RoDB, logger log.Logger) (*Aggregator, error) { + salt, err := getStateIndicesSalt(dirs.Snap) + if err != nil { + return nil, err + } + + a, err := NewAggregator(ctx, dirs, aggregationStep, db, logger) + if err != nil { + return nil, err + } + if err := a.registerDomain(kv.AccountsDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.StorageDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.CodeDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.CommitmentDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerDomain(kv.ReceiptDomain, salt, dirs, aggregationStep, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.LogAddrIdxPos, salt, dirs, aggregationStep, kv.FileLogAddressIdx, kv.TblLogAddressKeys, kv.TblLogAddressIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.LogTopicIdxPos, salt, dirs, aggregationStep, kv.FileLogTopicsIdx, kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.TracesFromIdxPos, salt, dirs, aggregationStep, kv.FileTracesFromIdx, kv.TblTracesFromKeys, kv.TblTracesFromIdx, logger); err != nil { + return nil, err + } + if err := a.registerII(kv.TracesToIdxPos, salt, dirs, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil { + return nil, err + } + a.KeepRecentTxnsOfHistoriesWithDisabledSnapshots(100_000) // ~1k blocks of history + a.recalcVisibleFiles(a.DirtyFilesEndTxNumMinimax()) + + return a, nil +} + +var dbgCommBtIndex = dbg.EnvBool("AGG_COMMITMENT_BT", false) + +func init() { + if dbgCommBtIndex { + cfg := Schema[kv.CommitmentDomain] + cfg.IndexList = AccessorBTree | AccessorExistence + Schema[kv.CommitmentDomain] = cfg + } +} + +var Schema = map[kv.Domain]domainCfg{ + kv.AccountsDomain: { + name: kv.AccountsDomain, valuesTable: kv.TblAccountVals, + + IndexList: AccessorBTree | AccessorExistence, + crossDomainIntegrity: domainIntegrityCheck, + compression: seg.CompressNone, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblAccountHistoryVals, + compression: seg.CompressNone, + + historyLargeValues: false, + filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant + + iiCfg: iiCfg{ + keysTable: kv.TblAccountHistoryKeys, valuesTable: kv.TblAccountIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.AccountsDomain.String(), //TODO: looks redundant + }, + }, + }, + kv.StorageDomain: { + name: kv.StorageDomain, valuesTable: kv.TblStorageVals, + + IndexList: AccessorBTree | AccessorExistence, + compression: seg.CompressKeys, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblStorageHistoryVals, + compression: seg.CompressNone, + + historyLargeValues: false, + filenameBase: kv.StorageDomain.String(), + + iiCfg: iiCfg{ + keysTable: kv.TblStorageHistoryKeys, valuesTable: kv.TblStorageIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.StorageDomain.String(), + }, + }, + }, + kv.CodeDomain: { + name: kv.CodeDomain, valuesTable: kv.TblCodeVals, + + IndexList: AccessorBTree | AccessorExistence, + compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet + compressCfg: DomainCompressCfg, + largeValues: true, + + hist: histCfg{ + valuesTable: kv.TblCodeHistoryVals, + compression: seg.CompressKeys | seg.CompressVals, + + historyLargeValues: true, + filenameBase: kv.CodeDomain.String(), + + iiCfg: iiCfg{ + withExistence: false, compressorCfg: seg.DefaultCfg, + keysTable: kv.TblCodeHistoryKeys, valuesTable: kv.TblCodeIdx, + filenameBase: kv.CodeDomain.String(), + }, + }, + }, + kv.CommitmentDomain: { + name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals, + + IndexList: AccessorHashMap, + compression: seg.CompressKeys, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblCommitmentHistoryVals, + compression: seg.CompressNone, + + snapshotsDisabled: true, + historyLargeValues: false, + filenameBase: kv.CommitmentDomain.String(), + + iiCfg: iiCfg{ + keysTable: kv.TblCommitmentHistoryKeys, valuesTable: kv.TblCommitmentIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.CommitmentDomain.String(), + }, + }, + }, + kv.ReceiptDomain: { + name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals, + + IndexList: AccessorBTree | AccessorExistence, + compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals, + compressCfg: DomainCompressCfg, + + hist: histCfg{ + valuesTable: kv.TblReceiptHistoryVals, + compression: seg.CompressNone, + + historyLargeValues: false, + filenameBase: kv.ReceiptDomain.String(), + + iiCfg: iiCfg{ + keysTable: kv.TblReceiptHistoryKeys, valuesTable: kv.TblReceiptIdx, + withExistence: false, compressorCfg: seg.DefaultCfg, + filenameBase: kv.ReceiptDomain.String(), + }, + }, + }, +} diff --git a/erigon-lib/state/aggregator_bench_test.go b/erigon-lib/state/aggregator_bench_test.go index 63fa225af81..3ae00f393cb 100644 --- a/erigon-lib/state/aggregator_bench_test.go +++ b/erigon-lib/state/aggregator_bench_test.go @@ -45,7 +45,7 @@ func testDbAndAggregatorBench(b *testing.B, aggStep uint64) (kv.RwDB, *Aggregato dirs := datadir.New(b.TempDir()) db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).MustOpen() b.Cleanup(db.Close) - agg, err := NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(b, err) b.Cleanup(agg.Close) return db, agg diff --git a/erigon-lib/state/aggregator_fuzz_test.go b/erigon-lib/state/aggregator_fuzz_test.go index 559869f66ad..659dbed084d 100644 --- a/erigon-lib/state/aggregator_fuzz_test.go +++ b/erigon-lib/state/aggregator_fuzz_test.go @@ -276,7 +276,7 @@ func testFuzzDbAndAggregatorv3(f *testing.F, aggStep uint64) (kv.RwDB, *Aggregat db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).GrowthStep(32 * datasize.MB).MapSize(2 * datasize.GB).MustOpen() f.Cleanup(db.Close) - agg, err := NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(err) f.Cleanup(agg.Close) err = agg.OpenFolder() diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index 816eb25c2e9..759df463408 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -362,7 +362,7 @@ func aggregatorV3_RestartOnDatadir(t *testing.T, rc runCfg) { agg.Close() // Start another aggregator on same datadir - anotherAgg, err := NewAggregator(context.Background(), agg.dirs, aggStep, db, logger) + anotherAgg, err := NewAggregator2(context.Background(), agg.dirs, aggStep, db, logger) require.NoError(t, err) defer anotherAgg.Close() @@ -822,7 +822,7 @@ func TestAggregatorV3_RestartOnFiles(t *testing.T) { newDb := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).MustOpen() t.Cleanup(newDb.Close) - newAgg, err := NewAggregator(context.Background(), agg.dirs, aggStep, newDb, logger) + newAgg, err := NewAggregator2(context.Background(), agg.dirs, aggStep, newDb, logger) require.NoError(t, err) require.NoError(t, newAgg.OpenFolder()) @@ -1091,7 +1091,7 @@ func testDbAndAggregatorv3(tb testing.TB, aggStep uint64) (kv.RwDB, *Aggregator) db := mdbx.New(kv.ChainDB, logger).InMem(dirs.Chaindata).GrowthStep(32 * datasize.MB).MapSize(2 * datasize.GB).MustOpen() tb.Cleanup(db.Close) - agg, err := NewAggregator(context.Background(), dirs, aggStep, db, logger) + agg, err := NewAggregator2(context.Background(), dirs, aggStep, db, logger) require.NoError(err) tb.Cleanup(agg.Close) err = agg.OpenFolder() diff --git a/eth/backend.go b/eth/backend.go index 08da323e42c..6917a1cb0c9 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -1450,7 +1450,7 @@ func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConf } } blockReader := freezeblocks.NewBlockReader(allSnapshots, allBorSnapshots, heimdallStore, bridgeStore) - agg, err := libstate.NewAggregator(ctx, dirs, config3.DefaultStepSize, db, logger) + agg, err := libstate.NewAggregator2(ctx, dirs, config3.DefaultStepSize, db, logger) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 4835364c440..3aaf44d316d 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -1505,7 +1505,7 @@ func dbCfg(label kv.Label, path string) mdbx.MdbxOpts { Accede(true) // integration tool: open db without creation and without blocking erigon } func openAgg(ctx context.Context, dirs datadir.Dirs, chainDB kv.RwDB, logger log.Logger) *libstate.Aggregator { - agg, err := libstate.NewAggregator(ctx, dirs, config3.DefaultStepSize, chainDB, logger) + agg, err := libstate.NewAggregator2(ctx, dirs, config3.DefaultStepSize, chainDB, logger) if err != nil { panic(err) } diff --git a/turbo/app/squeeze_cmd.go b/turbo/app/squeeze_cmd.go index da096ff2c02..d71d68ed47c 100644 --- a/turbo/app/squeeze_cmd.go +++ b/turbo/app/squeeze_cmd.go @@ -137,7 +137,7 @@ func squeezeStorage(ctx context.Context, dirs datadir.Dirs, logger log.Logger) e ac := agg.BeginFilesRo() defer ac.Close() - aggOld, err := state.NewAggregator(ctx, dirsOld, config3.DefaultStepSize, db, logger) + aggOld, err := state.NewAggregator2(ctx, dirsOld, config3.DefaultStepSize, db, logger) if err != nil { panic(err) } @@ -178,7 +178,7 @@ func squeezeStorage(ctx context.Context, dirs datadir.Dirs, logger log.Logger) e func squeezeCode(ctx context.Context, dirs datadir.Dirs, logger log.Logger) error { db := dbCfg(kv.ChainDB, dirs.Chaindata).MustOpen() defer db.Close() - agg, err := state.NewAggregator(ctx, dirs, config3.DefaultStepSize, db, logger) + agg, err := state.NewAggregator2(ctx, dirs, config3.DefaultStepSize, db, logger) if err != nil { return err }