Merge branch 'main' into feat/assertor-release-workflow

.github/workflows/qa-rpc-integration-tests.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Checkout RPC Tests Repository & Install Requirements
         run: |
           rm -rf ${{ runner.workspace }}/rpc-tests
-          git -c advice.detachedHead=false clone --depth 1 --branch v1.0.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
+          git -c advice.detachedHead=false clone --depth 1 --branch v1.7.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
           cd ${{ runner.workspace }}/rpc-tests
           pip3 install -r requirements.txt
 

.github/workflows/scripts/run_rpc_tests.sh

@@ -6,13 +6,23 @@ set +e # Disable exit on error
 disabled_tests=(
     # Erigon2 and Erigon3 never supported this api methods
     trace_rawTransaction
-    # false positives: Erigon return expected response. but rpc-test somehow doesn't see 1 field.
-    erigon_getHeaderByHash,erigon_getHeaderByNumber,eth_feeHistory
+    # remove these lines after https://github.com/erigontech/rpc-tests/pull/284 and https://github.com/erigontech/erigon/pull/12655 
+    eth_feeHistory/test_12.json
+    eth_feeHistory/test_13.json
+    eth_feeHistory/test_15.json
+    eth_feeHistory/test_16.json
+    eth_feeHistory/test_17.json
+    eth_feeHistory/test_18.json
+    eth_feeHistory/test_19.json
+    eth_feeHistory/test_20.json
     # total difficulty field was removed, then added back
-    eth_getBlockByHash,eth_getBlockByNumber
+    # remove this line after https://github.com/erigontech/rpc-tests/pull/282
+    eth_getBlockByHash/test_10.json
+    eth_getBlockByNumber/test_12.json
     # Erigon bugs
     debug_accountRange,debug_storageRangeAt
     # need update rpc-test - because Erigon is correct (@AskAlexSharov will do after https://github.com/erigontech/erigon/pull/12634)
+    # remove this line after https://github.com/erigontech/rpc-tests/pull/273
     debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber
     # Erigon bug https://github.com/erigontech/erigon/issues/12603
     erigon_getLatestLogs,erigon_getLogsByHash/test_04.json
@@ -25,8 +35,8 @@ disabled_tests=(
     debug_traceBlockByNumber/test_12.tar
     # remove this line after https://github.com/erigontech/rpc-tests/pull/281
     parity_getBlockReceipts
+    parity_listStorageKeys/test_12.json
     # to investigate
-    debug_traceBlockByHash
     debug_traceCallMany/test_02.tar
     debug_traceCallMany/test_04.tar
     debug_traceCallMany/test_05.tar
@@ -38,11 +48,13 @@ disabled_tests=(
     engine_exchangeTransitionConfigurationV1/test_01.json
     engine_getClientVersionV1/test_1.json
     erigon_getBalanceChangesInBlock
-    eth_createAccessList/test_16.json
+    trace_replayBlockTransactions/test_29.tar
+    # do these perhaps require Erigon up?
     admin_nodeInfo/test_01.json
     admin_peers/test_01.json
     erigon_nodeInfo/test_1.json
     eth_coinbase/test_01.json
+    eth_createAccessList/test_16.json
     eth_getTransactionByHash/test_02.json
     eth_getWork/test_01.json
     eth_mining/test_01.json
@@ -51,48 +63,12 @@ disabled_tests=(
     eth_submitWork/test_1.json
     net_peerCount/test_1.json
     net_version/test_1.json
-    txpool_content/test_01.json
     txpool_status/test_1.json
-    web3_clientVersion/test_1.json
-    eth_estimateGas/test_14.json
-    trace_replayBlockTransactions/test_29.tar
-    # recently started to fail
-    debug_traceTransaction/test_20.json
-    debug_traceTransaction/test_21.json
-    debug_traceTransaction/test_22.json
-    debug_traceTransaction/test_25.json
-    debug_traceTransaction/test_30.tar
-    debug_traceTransaction/test_33.json
-    debug_traceTransaction/test_35.tar
-    debug_traceTransaction/test_36.json
-    debug_traceTransaction/test_37.tar
-    debug_traceTransaction/test_38.tar
-    debug_traceTransaction/test_43.json
-    debug_traceTransaction/test_44.json
-    debug_traceTransaction/test_62.json
-    debug_traceTransaction/test_64.json
-    debug_traceTransaction/test_74.tar
-    debug_traceTransaction/test_75.tar
-    debug_traceTransaction/test_77.json
-    debug_traceTransaction/test_78.tar
-    debug_traceTransaction/test_79.tar
-    debug_traceTransaction/test_80.tar
-    debug_traceTransaction/test_81.tar
-    debug_traceTransaction/test_82.tar
-    debug_traceTransaction/test_83.tar
-    debug_traceTransaction/test_84.tar
-    debug_traceTransaction/test_85.tar
-    debug_traceTransaction/test_87.json
-    debug_traceTransaction/test_90.tar
-    debug_traceTransaction/test_91.tar
-    debug_traceTransaction/test_92.tar
-    debug_traceTransaction/test_93.json
-    debug_traceTransaction/test_96.json
-    trace_filter/test_16.json)
+    web3_clientVersion/test_1.json)
 
 # Transform the array into a comma-separated string
 disabled_test_list=$(IFS=,; echo "${disabled_tests[*]}")
 
 python3 ./run_tests.py -p 8545 --continue -f --json-diff -x "$disabled_test_list"
 
-exit $?
+exit $?

cl/antiquary/antiquary.go

@@ -37,6 +37,7 @@ import (
 	"github.com/erigontech/erigon/cl/persistence/blob_storage"
 	state_accessors "github.com/erigontech/erigon/cl/persistence/state"
 	"github.com/erigontech/erigon/cl/phase1/core/state"
+	"github.com/erigontech/erigon/turbo/snapshotsync"
 	"github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks"
 )
 
@@ -50,6 +51,7 @@ type Antiquary struct {
 	downloader                     proto_downloader.DownloaderClient
 	logger                         log.Logger
 	sn                             *freezeblocks.CaplinSnapshots
+	stateSn                        *snapshotsync.CaplinStateSnapshots
 	snReader                       freezeblocks.BeaconSnapshotReader
 	snBuildSema                    *semaphore.Weighted // semaphore for building only one type (blocks, caplin, v3) at a time
 	ctx                            context.Context
@@ -65,7 +67,7 @@ type Antiquary struct {
 	balances32   []byte
 }
 
-func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, logger log.Logger, states, blocks, blobs, snapgen bool, snBuildSema *semaphore.Weighted) *Antiquary {
+func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, stateSn *snapshotsync.CaplinStateSnapshots, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, logger log.Logger, states, blocks, blobs, snapgen bool, snBuildSema *semaphore.Weighted) *Antiquary {
 	backfilled := &atomic.Bool{}
 	blobBackfilled := &atomic.Bool{}
 	backfilled.Store(false)
@@ -89,6 +91,7 @@ func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, gen
 		blocks:          blocks,
 		blobs:           blobs,
 		snapgen:         snapgen,
+		stateSn:         stateSn,
 	}
 }
 

cl/antiquary/state_antiquary.go

@@ -27,7 +27,9 @@ import (
 
 	"github.com/erigontech/erigon-lib/common"
 	libcommon "github.com/erigontech/erigon-lib/common"
+	"github.com/erigontech/erigon-lib/downloader/snaptype"
 	"github.com/erigontech/erigon-lib/etl"
+	proto_downloader "github.com/erigontech/erigon-lib/gointerfaces/downloaderproto"
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon/cl/clparams"
@@ -42,6 +44,7 @@ import (
 	"github.com/erigontech/erigon/cl/phase1/core/state/raw"
 	"github.com/erigontech/erigon/cl/transition"
 	"github.com/erigontech/erigon/cl/transition/impl/eth2"
+	"github.com/erigontech/erigon/turbo/snapshotsync"
 )
 
 // pool for buffers
@@ -111,6 +114,9 @@ func (s *Antiquary) readHistoricalProcessingProgress(ctx context.Context) (progr
 	if err != nil {
 		return
 	}
+	if s.stateSn != nil {
+		progress = max(progress, s.stateSn.BlocksAvailable())
+	}
 
 	finalized, err = beacon_indicies.ReadHighestFinalized(tx)
 	if err != nil {
@@ -119,8 +125,68 @@ func (s *Antiquary) readHistoricalProcessingProgress(ctx context.Context) (progr
 	return
 }
 
+func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, stateSn *snapshotsync.CaplinStateSnapshots, validatorsTable *state_accessors.StaticValidatorTable) (bool, error) {
+	if stateSn == nil || validatorsTable.Slot() != 0 {
+		return false, nil
+	}
+	if err := stateSn.OpenFolder(); err != nil {
+		return false, err
+	}
+	blocksAvaiable := stateSn.BlocksAvailable()
+	stateSnRoTx := stateSn.View()
+	defer stateSnRoTx.Close()
+
+	start := time.Now()
+	for slot := uint64(0); slot <= stateSn.BlocksAvailable(); slot++ {
+		seg, ok := stateSnRoTx.VisibleSegment(slot, kv.StateEvents)
+		if !ok {
+			return false, fmt.Errorf("segment not found for slot %d", slot)
+		}
+		buf, err := seg.Get(slot)
+		if err != nil {
+			return false, err
+		}
+		if len(buf) == 0 {
+			continue
+		}
+		event := state_accessors.NewStateEventsFromBytes(buf)
+		state_accessors.ReplayEvents(
+			func(validatorIndex uint64, validator solid.Validator) error {
+				return validatorsTable.AddValidator(validator, validatorIndex, slot)
+			},
+			func(validatorIndex uint64, exitEpoch uint64) error {
+				return validatorsTable.AddExitEpoch(validatorIndex, slot, exitEpoch)
+			},
+			func(validatorIndex uint64, withdrawableEpoch uint64) error {
+				return validatorsTable.AddWithdrawableEpoch(validatorIndex, slot, withdrawableEpoch)
+			},
+			func(validatorIndex uint64, withdrawalCredentials libcommon.Hash) error {
+				return validatorsTable.AddWithdrawalCredentials(validatorIndex, slot, withdrawalCredentials)
+			},
+			func(validatorIndex uint64, activationEpoch uint64) error {
+				return validatorsTable.AddActivationEpoch(validatorIndex, slot, activationEpoch)
+			},
+			func(validatorIndex uint64, activationEligibilityEpoch uint64) error {
+				return validatorsTable.AddActivationEligibility(validatorIndex, slot, activationEligibilityEpoch)
+			},
+			func(validatorIndex uint64, slashed bool) error {
+				return validatorsTable.AddSlashed(validatorIndex, slot, slashed)
+			},
+			event,
+		)
+		validatorsTable.SetSlot(slot)
+	}
+	logger.Info("[Antiquary] Filled static validators table", "slots", blocksAvaiable, "elapsed", time.Since(start))
+	return true, nil
+}
+
 func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
-	var tx kv.Tx
+
+	// Check if you need to fill the static validators table
+	refilledStaticValidators, err := FillStaticValidatorsTableIfNeeded(ctx, s.logger, s.stateSn, s.validatorsTable)
+	if err != nil {
+		return err
+	}
 
 	tx, err := s.mainDB.BeginRo(ctx)
 	if err != nil {
@@ -131,6 +197,13 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 	// maps which validators changes
 	var changedValidators sync.Map
 
+	if refilledStaticValidators {
+		s.validatorsTable.ForEach(func(validatorIndex uint64, validator *state_accessors.StaticValidator) bool {
+			changedValidators.Store(validatorIndex, struct{}{})
+			return true
+		})
+	}
+
 	stateAntiquaryCollector := newBeaconStatesCollector(s.cfg, s.dirs.Tmp, s.logger)
 	defer stateAntiquaryCollector.close()
 
@@ -413,6 +486,59 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		return err
 	}
 	log.Info("Historical states antiquated", "slot", s.currentState.Slot(), "root", libcommon.Hash(stateRoot), "latency", endTime)
+	if s.snapgen {
+		if err := s.stateSn.OpenFolder(); err != nil {
+			return err
+		}
+
+		// Keep gnosis out for a bit
+		if s.currentState.BeaconConfig().ConfigName == "gnosis" {
+			return nil
+		}
+		blocksPerStatefulFile := uint64(snaptype.CaplinMergeLimit * 5)
+		from := s.stateSn.BlocksAvailable() + 1
+		if from+blocksPerStatefulFile+safetyMargin > s.currentState.Slot() {
+			return nil
+		}
+		to := s.currentState.Slot()
+		if to < (safetyMargin + blocksPerStatefulFile) {
+			return nil
+		}
+		to = to - (safetyMargin + blocksPerStatefulFile)
+		if from >= to {
+			return nil
+		}
+		if err := s.stateSn.DumpCaplinState(
+			ctx,
+			s.stateSn.BlocksAvailable()+1,
+			to,
+			blocksPerStatefulFile,
+			s.sn.Salt,
+			s.dirs,
+			1,
+			log.LvlInfo,
+			s.logger,
+		); err != nil {
+			return err
+		}
+		paths := s.stateSn.SegFileNames(from, to)
+		downloadItems := make([]*proto_downloader.AddItem, len(paths))
+		for i, path := range paths {
+			downloadItems[i] = &proto_downloader.AddItem{
+				Path: path,
+			}
+		}
+		if s.downloader != nil {
+			// Notify bittorent to seed the new snapshots
+			if _, err := s.downloader.Add(s.ctx, &proto_downloader.AddRequest{Items: downloadItems}); err != nil {
+				s.logger.Warn("[Antiquary] Failed to add items to bittorent", "err", err)
+			}
+		}
+		if err := s.stateSn.OpenFolder(); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -439,12 +565,15 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
 	if err != nil {
 		return err
 	}
+	if s.stateSn != nil {
+		targetSlot = max(targetSlot, s.stateSn.BlocksAvailable())
+	}
 	// We want to backoff by some slots until we get a correct state from DB.
 	// we start from 10 * clparams.SlotsPerDump.
 	backoffStrides := uint64(10)
 	backoffStep := backoffStrides
 
-	historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.genesisState)
+	historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.genesisState, s.stateSn)
 
 	for {
 		attempt, err := computeSlotToBeRequested(tx, s.cfg, s.genesisState.Slot(), targetSlot, backoffStep)
@@ -465,6 +594,7 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
 		if err != nil {
 			return fmt.Errorf("failed to read historical state at slot %d: %w", attempt, err)
 		}
+
 		if s.currentState == nil {
 			log.Warn("historical state not found, backoff more and try again", "slot", attempt)
 			backoffStep += backoffStrides

cl/antiquary/state_antiquary_test.go

@@ -41,7 +41,7 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, true, false, nil)
+	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, nil, reader, log.New(), true, true, true, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 }
 

cl/beacon/handler/attestation_rewards.go

@@ -178,26 +178,30 @@ func (a *ApiHandler) PostEthV1BeaconRewardsAttestations(w http.ResponseWriter, r
 	if lastSlot > stateProgress {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("requested range is not yet processed or the node is not archivial"))
 	}
+	snRoTx := a.caplinStateSnapshots.View()
+	defer snRoTx.Close()
 
-	epochData, err := state_accessors.ReadEpochData(tx, a.beaconChainCfg.RoundSlotToEpoch(lastSlot))
+	stateGetter := state_accessors.GetValFnTxAndSnapshot(tx, snRoTx)
+
+	epochData, err := state_accessors.ReadEpochData(stateGetter, a.beaconChainCfg.RoundSlotToEpoch(lastSlot))
 	if err != nil {
 		return nil, err
 	}
 
-	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, lastSlot)
+	validatorSet, err := a.stateReader.ReadValidatorsForHistoricalState(tx, stateGetter, lastSlot)
 	if err != nil {
 		return nil, err
 	}
 	if validatorSet == nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusNotFound, errors.New("no validator set found for this epoch"))
 	}
 
-	_, previousIdx, err := a.stateReader.ReadParticipations(tx, lastSlot)
+	_, previousIdx, err := a.stateReader.ReadParticipations(tx, stateGetter, lastSlot)
 	if err != nil {
 		return nil, err
 	}
 
-	_, _, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(tx, epoch*a.beaconChainCfg.SlotsPerEpoch)
+	_, _, finalizedCheckpoint, ok, err := state_accessors.ReadCheckpoints(stateGetter, epoch*a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
 		return nil, err
 	}
@@ -212,7 +216,7 @@ func (a *ApiHandler) PostEthV1BeaconRewardsAttestations(w http.ResponseWriter, r
 		return resp.WithFinalized(true).WithOptimistic(a.forkchoiceStore.IsRootOptimistic(root)), nil
 	}
 	inactivityScores := solid.NewUint64ListSSZ(int(a.beaconChainCfg.ValidatorRegistryLimit))
-	if err := a.stateReader.ReconstructUint64ListDump(tx, lastSlot, kv.InactivityScores, validatorSet.Length(), inactivityScores); err != nil {
+	if err := a.stateReader.ReconstructUint64ListDump(stateGetter, lastSlot, kv.InactivityScores, validatorSet.Length(), inactivityScores); err != nil {
 		return nil, err
 	}
 	resp, err := a.computeAttestationsRewardsForAltair(