From d82fa3d6edbdc9f9fdca9af8183fe4936978404f Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 21 Nov 2023 08:38:05 +0100 Subject: [PATCH 01/20] Restrict best LC update collection to canonical blocks Currently, the best LC update for a sync committee period may refer to blocks that have later been orphaned, if they rank better than canonical blocks according to `is_better_update`. This was done because the most important task of the light client sync protocol is to track the correct `next_sync_committee`. However, practical implementation is quite tricky because existing infrastructure such as fork choice modules can only be reused in limited form when collecting light client data. Furthermore, it becomes impossible to deterministically obtain the absolute best LC update available for any given sync committee period, because orphaned blocks may become unavailable. For these reasons, `LightClientUpdate` should only be served if they refer to data from the canonical chain as selected by fork choice. This also assists efforts for a reliable backward sync in the future. --- specs/altair/light-client/full-node.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/light-client/full-node.md b/specs/altair/light-client/full-node.md index 27651af01f..9a69b253f5 100644 --- a/specs/altair/light-client/full-node.md +++ b/specs/altair/light-client/full-node.md @@ -143,7 +143,7 @@ Full nodes SHOULD provide the best derivable `LightClientUpdate` (according to ` - `LightClientUpdate` are assigned to sync committee periods based on their `attested_header.beacon.slot` - `LightClientUpdate` are only considered if `compute_sync_committee_period_at_slot(update.attested_header.beacon.slot) == compute_sync_committee_period_at_slot(update.signature_slot)` -- Only `LightClientUpdate` with `next_sync_committee` as selected by fork choice are provided, regardless of ranking by `is_better_update`. To uniquely identify a non-finalized sync committee fork, all of `period`, `current_sync_committee` and `next_sync_committee` need to be incorporated, as sync committees may reappear over time. +- Only `LightClientUpdate` with `sync_aggregate` from blocks on the canonical chain as selected by fork choice are considered, regardless of ranking by `is_better_update`. `LightClientUpdate` referring to orphaned blocks SHOULD NOT be provided. ### `create_light_client_finality_update` From be2984156bb086d9e73445245ad046a0e8054228 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 6 Feb 2024 13:00:17 +0100 Subject: [PATCH 02/20] Add canonical data collection test infrastructure --- .../light_client/test_data_collection.py | 934 ++++++++++++++++++ tests/formats/light_client/README.md | 1 + tests/formats/light_client/data_collection.md | 76 ++ tests/generators/light_client/main.py | 1 + 4 files changed, 1012 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py create mode 100644 tests/formats/light_client/data_collection.md diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py new file mode 100644 index 0000000000..264c654810 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py @@ -0,0 +1,934 @@ +from typing import (Any, Dict, List, Set) +from dataclasses import dataclass + +from eth_utils import encode_hex +from eth2spec.test.context import ( + spec_state_test_with_matching_config, + with_presets, + with_light_client, +) +from eth2spec.test.helpers.constants import ( + ALTAIR, + MINIMAL, +) +from eth2spec.test.helpers.fork_transition import ( + transition_across_forks, +) +from eth2spec.test.helpers.forks import ( + is_post_altair, +) +from eth2spec.test.helpers.light_client import ( + compute_start_slot_at_sync_committee_period, + get_sync_aggregate, + upgrade_lc_header_to_new_spec, + upgrade_lc_update_to_new_spec, +) + + +def next_epoch_boundary_slot(spec, slot): + ## Compute the first possible epoch boundary state slot of a `Checkpoint` + ## referring to a block at given slot. + epoch = spec.compute_epoch_at_slot(slot + spec.SLOTS_PER_EPOCH - 1) + return spec.compute_start_slot_at_epoch(epoch) + + +@dataclass(frozen=True) +class BlockId(object): + slot: Any + root: Any + + +def block_to_block_id(block): + return BlockId( + slot=block.message.slot, + root=block.message.hash_tree_root(), + ) + + +def state_to_block_id(state): + parent_header = state.latest_block_header.copy() + parent_header.state_root = state.hash_tree_root() + return BlockId(slot=parent_header.slot, root=parent_header.hash_tree_root()) + + +def bootstrap_bid(bootstrap): + return BlockId( + slot=bootstrap.header.beacon.slot, + root=bootstrap.header.beacon.hash_tree_root(), + ) + + +def update_attested_bid(update): + return BlockId( + slot=update.attested_header.beacon.slot, + root=update.attested_header.beacon.hash_tree_root(), + ) + + +@dataclass +class ForkedBeaconState(object): + spec: Any + data: Any + + +@dataclass +class ForkedSignedBeaconBlock(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientHeader(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientBootstrap(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientUpdate(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientFinalityUpdate(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientOptimisticUpdate(object): + spec: Any + data: Any + + +@dataclass +class CachedLightClientData(object): + # Sync committee branches at block's post-state + current_sync_committee_branch: Any # CurrentSyncCommitteeBranch + next_sync_committee_branch: Any # NextSyncCommitteeBranch + + # Finality information at block's post-state + finalized_slot: Any # Slot + finality_branch: Any # FinalityBranch + + # Best / latest light client data + current_period_best_update: ForkedLightClientUpdate + latest_signature_slot: Any # Slot + + +@dataclass +class LightClientDataCache(object): + # Cached data for creating future `LightClientUpdate` instances. + # Key is the block ID of which the post state was used to get the data. + # Data stored for the finalized head block and all non-finalized blocks. + data: Dict[BlockId, CachedLightClientData] + + # Light client data for the latest slot that was signed by at least + # `MIN_SYNC_COMMITTEE_PARTICIPANTS`. May be older than head + latest: ForkedLightClientFinalityUpdate + + # The earliest slot for which light client data is imported + tail_slot: Any # Slot + + +@dataclass +class LightClientDataDb(object): + headers: Dict[Any, ForkedLightClientHeader] # Root -> ForkedLightClientHeader + current_branches: Dict[Any, Any] # Slot -> CurrentSyncCommitteeBranch + sync_committees: Dict[Any, Any] # SyncCommitteePeriod -> SyncCommittee + best_updates: Dict[Any, ForkedLightClientUpdate] # SyncCommitteePeriod -> ForkedLightClientUpdate + + +@dataclass +class LightClientDataStore(object): + # Cached data to accelerate creating light client data + cache: LightClientDataCache + + # Persistent light client data + db: LightClientDataDb + + +@dataclass +class LightClientDataCollectionTest(object): + steps: List[Dict[str, Any]] + files: Set[str] + + # Fork schedule + spec: Any + phases: Any + + # History access + blocks: Dict[Any, ForkedSignedBeaconBlock] # Block root -> ForkedSignedBeaconBlock + finalized_block_roots: Dict[Any, Any] # Slot -> Root + states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState + finalized_checkpoint_states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState + latest_finalized_epoch: Any # Epoch + latest_finalized_bid: BlockId + historical_tail_slot: Any # Slot + + # Light client data + lc_data_store: LightClientDataStore + + +def get_ancestor_of_block_id(test, bid, slot): # -> Optional[BlockId] + try: + block = test.blocks[bid.root] + while True: + if block.data.message.slot <= slot: + return block_to_block_id(block.data) + + block = test.blocks[block.data.message.parent_root] + except KeyError: + return None + + +def block_id_at_finalized_slot(test, slot): # -> Optional[BlockId] + while slot >= test.historical_tail_slot: + try: + return BlockId(slot=slot, root=test.finalized_block_roots[slot]) + except KeyError: + slot = slot - 1 + return None + + +def get_current_sync_committee_for_finalized_period(test, period): # -> Optional[SyncCommittee] + low_slot = max( + test.historical_tail_slot, + test.spec.compute_start_slot_at_epoch(test.spec.config.ALTAIR_FORK_EPOCH) + ) + if period < test.spec.compute_sync_committee_period_at_slot(low_slot): + return None + period_start_slot = compute_start_slot_at_sync_committee_period(test.spec, period) + sync_committee_slot = max(period_start_slot, low_slot) + bid = block_id_at_finalized_slot(test, sync_committee_slot) + if bid is None: + return None + block = test.blocks[bid.root] + state = test.finalized_checkpoint_states[block.data.message.state_root] + if sync_committee_slot > state.data.slot: + state.spec, state.data, _ = transition_across_forks(state.spec, state.data, sync_committee_slot, phases=test.phases) + assert is_post_altair(state.spec) + return state.data.current_sync_committee + + +def light_client_header_for_block(test, block): # -> ForkedLightClientHeader + if not is_post_altair(block.spec): + spec = test.phases[ALTAIR] + else: + spec = block.spec + return ForkedLightClientHeader(spec=spec, data=spec.block_to_light_client_header(block.data)) + + +def light_client_header_for_block_id(test, bid): # -> ForkedLightClientHeader + block = test.blocks[bid.root] + if not is_post_altair(block.spec): + spec = test.phases[ALTAIR] + else: + spec = block.spec + return ForkedLightClientHeader(spec=spec, data=spec.block_to_light_client_header(block.data)) + + +def sync_aggregate_for_block_id(test, bid): # -> Optional[SyncAggregate] + block = test.blocks[bid.root] + if not is_post_altair(block.spec): + return None + return block.data.message.body.sync_aggregate + + +def get_light_client_data(lc_data_store, bid): # -> CachedLightClientData + ## Fetch cached light client data about a given block. + ## Data must be cached (`cache_light_client_data`) before calling this function. + try: + return lc_data_store.cache.data[bid] + except KeyError: + raise ValueError("Trying to get light client data that was not cached") + + +def cache_light_client_data(lc_data_store, spec, state, bid, current_period_best_update, latest_signature_slot): + ## Cache data for a given block and its post-state to speed up creating future + ## `LightClientUpdate` and `LightClientBootstrap` instances that refer to this + ## block and state. + cached_data = CachedLightClientData( + current_sync_committee_branch=spec.compute_merkle_proof(state, spec.CURRENT_SYNC_COMMITTEE_GINDEX), + next_sync_committee_branch=spec.compute_merkle_proof(state, spec.NEXT_SYNC_COMMITTEE_GINDEX), + finalized_slot=spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch), + finality_branch=spec.compute_merkle_proof(state, spec.FINALIZED_ROOT_GINDEX), + current_period_best_update=current_period_best_update, + latest_signature_slot=latest_signature_slot, + ) + if bid in lc_data_store.cache.data: + raise ValueError("Redundant `cache_light_client_data` call") + lc_data_store.cache.data[bid] = cached_data + + +def delete_light_client_data(lc_data_store, bid): + ## Delete cached light client data for a given block. This needs to be called + ## when a block becomes unreachable due to finalization of a different fork. + del lc_data_store.cache.data[bid] + + +def create_light_client_finality_update_from_light_client_data(test, + attested_bid, + signature_slot, + sync_aggregate): # -> ForkedLightClientFinalityUpdate + attested_header = light_client_header_for_block_id(test, attested_bid) + attested_data = get_light_client_data(test.lc_data_store, attested_bid) + finalized_bid = block_id_at_finalized_slot(test, attested_data.finalized_slot) + if finalized_bid is not None: + if finalized_bid.slot != attested_data.finalized_slot: + # Empty slots at end of epoch, update cache for latest block slot + attested_data.finalized_slot = finalized_bid.slot + if finalized_bid.slot == attested_header.spec.GENESIS_SLOT: + finalized_header = ForkedLightClientHeader( + spec=attested_header.spec, + data=attested_header.spec.LightClientHeader(), + ) + else: + finalized_header = light_client_header_for_block_id(test, finalized_bid) + finalized_header = ForkedLightClientHeader( + spec=attested_header.spec, + data=upgrade_lc_header_to_new_spec( + finalized_header.spec, + attested_header.spec, + finalized_header.data, + ) + ) + finality_branch = attested_data.finality_branch + return ForkedLightClientFinalityUpdate( + spec=attested_header.spec, + data=attested_header.spec.LightClientFinalityUpdate( + attested_header=attested_header.data, + finalized_header=finalized_header.data, + finality_branch=finality_branch, + sync_aggregate=sync_aggregate, + signature_slot=signature_slot, + ), + ) + + +def create_light_client_update_from_light_client_data(test, + attested_bid, + signature_slot, + sync_aggregate, + next_sync_committee): # -> ForkedLightClientUpdate + finality_update = create_light_client_finality_update_from_light_client_data( + test, attested_bid, signature_slot, sync_aggregate) + attested_data = get_light_client_data(test.lc_data_store, attested_bid) + return ForkedLightClientUpdate( + spec=finality_update.spec, + data=finality_update.spec.LightClientUpdate( + attested_header=finality_update.data.attested_header, + next_sync_committee=next_sync_committee, + next_sync_committee_branch=attested_data.next_sync_committee_branch, + finalized_header=finality_update.data.finalized_header, + finality_branch=finality_update.data.finality_branch, + sync_aggregate=finality_update.data.sync_aggregate, + signature_slot=finality_update.data.signature_slot, + ) + ) + + +def create_light_client_update(test, spec, state, block, parent_bid): + ## Create `LightClientUpdate` instances for a given block and its post-state, + ## and keep track of best / latest ones. Data about the parent block's + ## post-state must be cached (`cache_light_client_data`) before calling this. + + # Verify attested block (parent) is recent enough and that state is available + attested_bid = parent_bid + attested_slot = attested_bid.slot + if attested_slot < test.lc_data_store.cache.tail_slot: + cache_light_client_data( + test.lc_data_store, + spec, + state, + block_to_block_id(block), + current_period_best_update=ForkedLightClientUpdate(spec=None, data=None), + latest_signature_slot=spec.GENESIS_SLOT, + ) + return + + # If sync committee period changed, reset `best` + attested_period = spec.compute_sync_committee_period_at_slot(attested_slot) + signature_slot = block.message.slot + signature_period = spec.compute_sync_committee_period_at_slot(signature_slot) + attested_data = get_light_client_data(test.lc_data_store, attested_bid) + if attested_period != signature_period: + best = ForkedLightClientUpdate(spec=None, data=None) + else: + best = attested_data.current_period_best_update + + # If sync committee does not have sufficient participants, do not bump latest + sync_aggregate = block.message.body.sync_aggregate + num_active_participants = sum(sync_aggregate.sync_committee_bits) + if num_active_participants < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS: + latest_signature_slot = attested_data.latest_signature_slot + else: + latest_signature_slot = signature_slot + + # To update `best`, sync committee must have sufficient participants, and + # `signature_slot` must be in `attested_slot`'s sync committee period + if ( + num_active_participants < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS + or attested_period != signature_period + ): + cache_light_client_data( + test.lc_data_store, + spec, + state, + block_to_block_id(block), + current_period_best_update=best, + latest_signature_slot=latest_signature_slot, + ) + return + + # Check if light client data improved + update = create_light_client_update_from_light_client_data( + test, attested_bid, signature_slot, sync_aggregate, state.next_sync_committee) + is_better = ( + best.spec is None + or spec.is_better_update(update.data, upgrade_lc_update_to_new_spec(best.spec, update.spec, best.data)) + ) + + # Update best light client data for current sync committee period + if is_better: + best = update + cache_light_client_data( + test.lc_data_store, + spec, + state, + block_to_block_id(block), + current_period_best_update=best, + latest_signature_slot=latest_signature_slot, + ) + + +def create_light_client_bootstrap(test, spec, bid): + block = test.blocks[bid.root] + period = spec.compute_sync_committee_period_at_slot(bid.slot) + if period not in test.lc_data_store.db.sync_committees: + test.lc_data_store.db.sync_committees[period] = \ + get_current_sync_committee_for_finalized_period(test, period) + test.lc_data_store.db.headers[bid.root] = ForkedLightClientHeader( + spec=block.spec, data=block.spec.block_to_light_client_header(block.data)) + test.lc_data_store.db.current_branches[bid.slot] = \ + get_light_client_data(test.lc_data_store, bid).current_sync_committee_branch + + +def process_new_block_for_light_client(test, spec, state, block, parent_bid): + ## Update light client data with information from a new block. + if block.message.slot < test.lc_data_store.cache.tail_slot: + return + + if is_post_altair(spec): + create_light_client_update(test, spec, state, block, parent_bid) + else: + raise ValueError("`tail_slot` cannot be before Altair") + + +def process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid): + ## Update light client data to account for a new head block. + ## Note that `old_finalized_bid` is not yet updated when this is called. + if head_bid.slot < test.lc_data_store.cache.tail_slot: + return + + # Commit best light client data for non-finalized periods + head_period = spec.compute_sync_committee_period_at_slot(head_bid.slot) + low_slot = max(test.lc_data_store.cache.tail_slot, old_finalized_bid.slot) + low_period = spec.compute_sync_committee_period_at_slot(low_slot) + bid = head_bid + for period in reversed(range(low_period, head_period + 1)): + period_end_slot = compute_start_slot_at_sync_committee_period(spec, period + 1) - 1 + bid = get_ancestor_of_block_id(test, bid, period_end_slot) + if bid is None or bid.slot < low_slot: + break + best = get_light_client_data(test.lc_data_store, bid).current_period_best_update + if ( + best.spec is None + or sum(best.data.sync_aggregate.sync_committee_bits) < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS + ): + test.lc_data_store.db.best_updates.pop(period, None) + else: + test.lc_data_store.db.best_updates[period] = best + + # Update latest light client data + head_data = get_light_client_data(test.lc_data_store, head_bid) + signature_slot = head_data.latest_signature_slot + if signature_slot <= low_slot: + test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) + return + signature_bid = get_ancestor_of_block_id(test, head_bid, signature_slot) + if signature_bid is None or signature_bid.slot <= low_slot: + test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) + return + attested_bid = get_ancestor_of_block_id(test, signature_bid, signature_bid.slot - 1) + if attested_bid is None or attested_bid.slot < low_slot: + test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) + return + sync_aggregate = sync_aggregate_for_block_id(test, signature_bid) + assert sync_aggregate is not None + test.lc_data_store.cache.latest = create_light_client_finality_update_from_light_client_data( + test, attested_bid, signature_slot, sync_aggregate) + + +def process_finalization_for_light_client(test, spec, finalized_bid, old_finalized_bid): + ## Prune cached data that is no longer useful for creating future + ## `LightClientUpdate` and `LightClientBootstrap` instances. + ## This needs to be called whenever `finalized_checkpoint` changes. + finalized_slot = finalized_bid.slot + if finalized_slot < test.lc_data_store.cache.tail_slot: + return + + # Cache `LightClientBootstrap` for newly finalized epoch boundary blocks + first_new_slot = old_finalized_bid.slot + 1 + low_slot = max(first_new_slot, test.lc_data_store.cache.tail_slot) + boundary_slot = finalized_slot + while boundary_slot >= low_slot: + bid = block_id_at_finalized_slot(test, boundary_slot) + if bid is None: + break + if bid.slot >= low_slot: + create_light_client_bootstrap(test, spec, bid) + boundary_slot = next_epoch_boundary_slot(spec, bid.slot) + if boundary_slot < spec.SLOTS_PER_EPOCH: + break + boundary_slot = boundary_slot - spec.SLOTS_PER_EPOCH + + # Prune light client data that is no longer referrable by future updates + bids_to_delete = [] + for bid in test.lc_data_store.cache.data: + if bid.slot >= finalized_bid.slot: + continue + bids_to_delete.append(bid) + for bid in bids_to_delete: + delete_light_client_data(test.lc_data_store, bid) + + +def get_light_client_bootstrap(test, block_root): # -> ForkedLightClientBootstrap + try: + header = test.lc_data_store.db.headers[block_root] + except KeyError: + return ForkedLightClientBootstrap(spec=None, data=None) + + slot = header.data.beacon.slot + period = header.spec.compute_sync_committee_period_at_slot(slot) + return ForkedLightClientBootstrap( + spec=header.spec, + data=header.spec.LightClientBootstrap( + header=header.data, + current_sync_committee=test.lc_data_store.db.sync_committees[period], + current_sync_committee_branch=test.lc_data_store.db.current_branches[slot], + ) + ) + + +def get_light_client_update_for_period(test, period): # -> ForkedLightClientUpdate + try: + return test.lc_data_store.db.best_updates[period] + except KeyError: + return ForkedLightClientUpdate(spec=None, data=None) + + +def get_light_client_finality_update(test): # -> ForkedLightClientFinalityUpdate + return test.lc_data_store.cache.latest + + +def get_light_client_optimistic_update(test): # -> ForkedLightClientOptimisticUpdate + finality_update = get_light_client_finality_update(test) + if finality_update.spec is None: + return ForkedLightClientOptimisticUpdate(spec=None, data=None) + return ForkedLightClientOptimisticUpdate( + spec=finality_update.spec, + data=finality_update.spec.LightClientOptimisticUpdate( + attested_header=finality_update.data.attested_header, + sync_aggregate=finality_update.data.sync_aggregate, + signature_slot=finality_update.data.signature_slot, + ), + ) + + +def setup_test(spec, state, phases=None): + assert spec.compute_slots_since_epoch_start(state.slot) == 0 + + test = LightClientDataCollectionTest( + steps=[], + files=set(), + spec=spec, + phases=phases, + blocks={}, + finalized_block_roots={}, + states={}, + finalized_checkpoint_states={}, + latest_finalized_epoch=state.finalized_checkpoint.epoch, + latest_finalized_bid=BlockId( + slot=spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch), + root=state.finalized_checkpoint.root, + ), + historical_tail_slot=state.slot, + lc_data_store=LightClientDataStore( + cache=LightClientDataCache( + data={}, + latest=ForkedLightClientFinalityUpdate(spec=None, data=None), + tail_slot=max(state.slot, spec.compute_start_slot_at_epoch(spec.config.ALTAIR_FORK_EPOCH)), + ), + db=LightClientDataDb( + headers={}, + current_branches={}, + sync_committees={}, + best_updates={}, + ), + ), + ) + bid = state_to_block_id(state) + yield "initial_state", state + test.blocks[bid.root] = ForkedSignedBeaconBlock(spec=spec, data=spec.SignedBeaconBlock( + message=spec.BeaconBlock(state_root=state.hash_tree_root()), + )) + test.finalized_block_roots[bid.slot] = bid.root + test.states[state.hash_tree_root()] = ForkedBeaconState(spec=spec, data=state) + test.finalized_checkpoint_states[state.hash_tree_root()] = ForkedBeaconState(spec=spec, data=state) + cache_light_client_data( + test.lc_data_store, spec, state, bid, + current_period_best_update=ForkedLightClientUpdate(spec=None, data=None), + latest_signature_slot=spec.GENESIS_SLOT, + ) + create_light_client_bootstrap(test, spec, bid) + + return test + + +def finish_test(test): + yield "steps", test.steps + + +def encode_object(test, prefix, obj, slot, genesis_validators_root): + yield from [] # Consistently enable `yield from` syntax in calling tests + + file_name = f"{prefix}_{slot}_{encode_hex(obj.data.hash_tree_root())}" + if file_name not in test.files: + test.files.add(file_name) + yield file_name, obj.data + return { + "fork_digest": encode_hex(obj.spec.compute_fork_digest( + obj.spec.compute_fork_version(obj.spec.compute_epoch_at_slot(slot)), + genesis_validators_root, + )), + "data": file_name, + } + + +def add_new_block(test, spec, state, slot=None, num_sync_participants=0): + if slot is None: + slot = state.slot + 1 + assert slot > state.slot + parent_bid = state_to_block_id(state) + + # Advance to target slot - 1 to ensure sync aggregate can be efficiently computed + if state.slot < slot - 1: + spec, state, _ = transition_across_forks(spec, state, slot - 1, phases=test.phases) + + # Compute sync aggregate, using: + # - sync committee based on target slot + # - fork digest based on target slot - 1 + # - signed data based on parent_bid.slot + # All three slots may be from different forks + sync_aggregate, signature_slot = get_sync_aggregate( + spec, state, num_participants=num_sync_participants, phases=test.phases) + assert signature_slot == slot + + # Apply final block with computed sync aggregate + spec, state, block = transition_across_forks( + spec, state, slot, phases=test.phases, with_block=True, sync_aggregate=sync_aggregate) + bid = block_to_block_id(block) + test.blocks[bid.root] = ForkedSignedBeaconBlock(spec=spec, data=block) + test.states[block.message.state_root] = ForkedBeaconState(spec=spec, data=state) + process_new_block_for_light_client(test, spec, state, block, parent_bid) + block_obj = yield from encode_object( + test, "block", ForkedSignedBeaconBlock(spec=spec, data=block), block.message.slot, + state.genesis_validators_root, + ) + test.steps.append({ + "new_block": block_obj + }) + return spec, state, bid + + +def select_new_head(test, spec, head_bid): + old_finalized_bid = test.latest_finalized_bid + process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid) + + # Process finalization + block = test.blocks[head_bid.root] + state = test.states[block.data.message.state_root] + if state.data.finalized_checkpoint.epoch != spec.GENESIS_EPOCH: + block = test.blocks[state.data.finalized_checkpoint.root] + bid = block_to_block_id(block.data) + new_finalized_bid = bid + if new_finalized_bid.slot > old_finalized_bid.slot: + old_finalized_epoch = None + new_finalized_epoch = state.data.finalized_checkpoint.epoch + while bid.slot > test.latest_finalized_bid.slot: + test.finalized_block_roots[bid.slot] = bid.root + finalized_epoch = spec.compute_epoch_at_slot(bid.slot + spec.SLOTS_PER_EPOCH - 1) + if finalized_epoch != old_finalized_epoch: + state = test.states[block.data.message.state_root] + test.finalized_checkpoint_states[block.data.message.state_root] = state + old_finalized_epoch = finalized_epoch + block = test.blocks[block.data.message.parent_root] + bid = block_to_block_id(block.data) + test.latest_finalized_epoch = new_finalized_epoch + test.latest_finalized_bid = new_finalized_bid + process_finalization_for_light_client(test, spec, new_finalized_bid, old_finalized_bid) + + blocks_to_delete = [] + for block_root, block in test.blocks.items(): + if block.data.message.slot < new_finalized_bid.slot: + blocks_to_delete.append(block_root) + for block_root in blocks_to_delete: + del test.blocks[block_root] + states_to_delete = [] + for state_root, state in test.states.items(): + if state.data.slot < new_finalized_bid.slot: + states_to_delete.append(state_root) + for state_root in states_to_delete: + del test.states[state_root] + + yield from [] # Consistently enable `yield from` syntax in calling tests + + bootstraps = [] + for state in test.finalized_checkpoint_states.values(): + bid = state_to_block_id(state.data) + entry = { + "block_root": encode_hex(bid.root), + } + bootstrap = get_light_client_bootstrap(test, bid.root) + if bootstrap.spec is not None: + bootstrap_obj = yield from encode_object( + test, "bootstrap", bootstrap, bootstrap.data.header.beacon.slot, + state.data.genesis_validators_root, + ) + entry["bootstrap"] = bootstrap_obj + bootstraps.append(entry) + + best_updates = [] + low_period = spec.compute_sync_committee_period_at_slot(test.lc_data_store.cache.tail_slot) + head_period = spec.compute_sync_committee_period_at_slot(head_bid.slot) + for period in range(low_period, head_period + 1): + entry = { + "period": int(period), + } + update = get_light_client_update_for_period(test, period) + if update.spec is not None: + update_obj = yield from encode_object( + test, "update", update, update.data.attested_header.beacon.slot, + state.data.genesis_validators_root, + ) + entry["update"] = update_obj + best_updates.append(entry) + + checks = { + "latest_finalized_checkpoint": { + "epoch": int(test.latest_finalized_epoch), + "root": encode_hex(test.latest_finalized_bid.root), + }, + "bootstraps": bootstraps, + "best_updates": best_updates, + } + finality_update = get_light_client_finality_update(test) + if finality_update.spec is not None: + finality_update_obj = yield from encode_object( + test, "finality_update", finality_update, finality_update.data.attested_header.beacon.slot, + state.data.genesis_validators_root, + ) + checks["latest_finality_update"] = finality_update_obj + optimistic_update = get_light_client_finality_update(test) + if optimistic_update.spec is not None: + optimistic_update_obj = yield from encode_object( + test, "optimistic_update", optimistic_update, optimistic_update.data.attested_header.beacon.slot, + state.data.genesis_validators_root, + ) + checks["latest_finality_update"] = optimistic_update_obj + + test.steps.append({ + "new_head": { + "head_block_root": encode_hex(head_bid.root), + "checks": checks, + } + }) + + +@with_light_client +@spec_state_test_with_matching_config +@with_presets([MINIMAL], reason="too slow") +def test_light_client_data_collection(spec, state): + # Start test + test = yield from setup_test(spec, state) + + # Genesis block is post Altair and is finalized, so can be used as bootstrap + genesis_bid = BlockId(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) + assert bootstrap_bid(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid + + # No blocks have been imported, so no other light client data is available + period = spec.compute_sync_committee_period_at_slot(state.slot) + assert get_light_client_update_for_period(test, period).spec is None + assert get_light_client_finality_update(test).spec is None + assert get_light_client_optimistic_update(test).spec is None + + # Start branch A with a block that has an empty sync aggregate + spec_a, state_a, bid_1 = yield from add_new_block(test, spec, state, slot=1) + yield from select_new_head(test, spec_a, bid_1) + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert get_light_client_update_for_period(test, period).spec is None + assert get_light_client_finality_update(test).spec is None + assert get_light_client_optimistic_update(test).spec is None + + # Start branch B with a block that has 1 participant + spec_b, state_b, bid_2 = yield from add_new_block(test, spec, state, slot=2, num_sync_participants=1) + yield from select_new_head(test, spec_b, bid_2) + period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == genesis_bid + assert update_attested_bid(get_light_client_finality_update(test).data) == genesis_bid + assert update_attested_bid(get_light_client_optimistic_update(test).data) == genesis_bid + + # Build on branch A, once more with an empty sync aggregate + spec_a, state_a, bid_3 = yield from add_new_block(test, spec_a, state_a, slot=3) + yield from select_new_head(test, spec_a, bid_3) + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert get_light_client_update_for_period(test, period).spec is None + assert get_light_client_finality_update(test).spec is None + assert get_light_client_optimistic_update(test).spec is None + + # Build on branch B, this time with an empty sync aggregate + spec_b, state_b, bid_4 = yield from add_new_block(test, spec_b, state_b, slot=4) + yield from select_new_head(test, spec_b, bid_4) + period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == genesis_bid + assert update_attested_bid(get_light_client_finality_update(test).data) == genesis_bid + assert update_attested_bid(get_light_client_optimistic_update(test).data) == genesis_bid + + # Build on branch B, once more with 1 participant + spec_b, state_b, bid_5 = yield from add_new_block(test, spec_b, state_b, slot=5, num_sync_participants=1) + yield from select_new_head(test, spec_b, bid_5) + period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == genesis_bid + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_4 + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_4 + + # Build on branch B, this time with 3 participants + spec_b, state_b, bid_6 = yield from add_new_block(test, spec_b, state_b, slot=6, num_sync_participants=3) + yield from select_new_head(test, spec_b, bid_6) + period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_5 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_5 + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_5 + + # Build on branch A, with 2 participants + spec_a, state_a, bid_7 = yield from add_new_block(test, spec_a, state_a, slot=7, num_sync_participants=2) + yield from select_new_head(test, spec_a, bid_7) + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_3 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_3 + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_3 + + # Branch A: epoch 1, slot 5 + slot = spec_a.compute_start_slot_at_epoch(1) + 5 + spec_a, state_a, bid_1_5 = yield from add_new_block(test, spec_a, state_a, slot=slot, num_sync_participants=4) + yield from select_new_head(test, spec_a, bid_1_5) + assert get_light_client_bootstrap(test, bid_7.root).spec is None + assert get_light_client_bootstrap(test, bid_1_5.root).spec is None + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_7 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_7 + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_7 + + # Branch B: epoch 2, slot 4 + slot = spec_b.compute_start_slot_at_epoch(2) + 4 + spec_b, state_b, bid_2_4 = yield from add_new_block(test, spec_b, state_b, slot=slot, num_sync_participants=5) + yield from select_new_head(test, spec_b, bid_2_4) + assert get_light_client_bootstrap(test, bid_7.root).spec is None + assert get_light_client_bootstrap(test, bid_1_5.root).spec is None + assert get_light_client_bootstrap(test, bid_2_4.root).spec is None + period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_6 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_6 + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_6 + + # Branch A: epoch 3, slot 0 + slot = spec_a.compute_start_slot_at_epoch(3) + 0 + spec_a, state_a, bid_3_0 = yield from add_new_block(test, spec_a, state_a, slot=slot, num_sync_participants=6) + yield from select_new_head(test, spec_a, bid_3_0) + assert get_light_client_bootstrap(test, bid_7.root).spec is None + assert get_light_client_bootstrap(test, bid_1_5.root).spec is None + assert get_light_client_bootstrap(test, bid_2_4.root).spec is None + assert get_light_client_bootstrap(test, bid_3_0.root).spec is None + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_1_5 + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_1_5 + + # Branch A: fill epoch + for i in range(1, spec_a.SLOTS_PER_EPOCH): + spec_a, state_a, bid_a = yield from add_new_block(test, spec_a, state_a) + yield from select_new_head(test, spec_a, bid_a) + assert get_light_client_bootstrap(test, bid_7.root).spec is None + assert get_light_client_bootstrap(test, bid_1_5.root).spec is None + assert get_light_client_bootstrap(test, bid_2_4.root).spec is None + assert get_light_client_bootstrap(test, bid_3_0.root).spec is None + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_1_5 + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_1_5 + assert state_a.slot == spec_a.compute_start_slot_at_epoch(4) - 1 + bid_3_n = bid_a + + # Branch A: epoch 4, slot 0 + slot = spec_a.compute_start_slot_at_epoch(4) + 0 + spec_a, state_a, bid_4_0 = yield from add_new_block(test, spec_a, state_a, slot=slot, num_sync_participants=6) + yield from select_new_head(test, spec_a, bid_4_0) + assert get_light_client_bootstrap(test, bid_7.root).spec is None + assert get_light_client_bootstrap(test, bid_1_5.root).spec is None + assert get_light_client_bootstrap(test, bid_2_4.root).spec is None + assert get_light_client_bootstrap(test, bid_3_0.root).spec is None + assert get_light_client_bootstrap(test, bid_4_0.root).spec is None + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_3_n + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_3_n + + # Branch A: fill epoch + for i in range(1, spec_a.SLOTS_PER_EPOCH): + spec_a, state_a, bid_a = yield from add_new_block(test, spec_a, state_a) + yield from select_new_head(test, spec_a, bid_a) + assert get_light_client_bootstrap(test, bid_7.root).spec is None + assert get_light_client_bootstrap(test, bid_1_5.root).spec is None + assert get_light_client_bootstrap(test, bid_2_4.root).spec is None + assert get_light_client_bootstrap(test, bid_3_0.root).spec is None + assert get_light_client_bootstrap(test, bid_4_0.root).spec is None + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_3_n + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_3_n + assert state_a.slot == spec_a.compute_start_slot_at_epoch(5) - 1 + bid_4_n = bid_a + + # Branch A: epoch 6, slot 2 + slot = spec_a.compute_start_slot_at_epoch(6) + 2 + spec_a, state_a, bid_6_2 = yield from add_new_block(test, spec_a, state_a, slot=slot, num_sync_participants=6) + yield from select_new_head(test, spec_a, bid_6_2) + assert bootstrap_bid(get_light_client_bootstrap(test, bid_7.root).data) == bid_7 + assert bootstrap_bid(get_light_client_bootstrap(test, bid_1_5.root).data) == bid_1_5 + assert get_light_client_bootstrap(test, bid_2_4.root).spec is None + assert bootstrap_bid(get_light_client_bootstrap(test, bid_3_0.root).data) == bid_3_0 + assert get_light_client_bootstrap(test, bid_4_0.root).spec is None + period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert update_attested_bid(get_light_client_finality_update(test).data) == bid_4_n + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_4_n + + # Finish test + yield from finish_test(test) diff --git a/tests/formats/light_client/README.md b/tests/formats/light_client/README.md index 505b416019..050b406f0b 100644 --- a/tests/formats/light_client/README.md +++ b/tests/formats/light_client/README.md @@ -3,6 +3,7 @@ This series of tests provides reference test vectors for the light client sync protocol spec. Handlers: +- `data_collection`: see [Light client data collection test format](./data_collection.md) - `single_merkle_proof`: see [Single leaf merkle proof test format](./single_merkle_proof.md) - `sync`: see [Light client sync test format](./sync.md) - `update_ranking`: see [`LightClientUpdate` ranking test format](./update_ranking.md) diff --git a/tests/formats/light_client/data_collection.md b/tests/formats/light_client/data_collection.md new file mode 100644 index 0000000000..d8f13e5ed0 --- /dev/null +++ b/tests/formats/light_client/data_collection.md @@ -0,0 +1,76 @@ +# Light client data collection tests + +This series of tests provies reference test vectors for validating that a full node collects canonical data for serving to light clients implementing the light client sync protocol to sync to the latest block header. + +## Test case format + +### `initial_state.ssz_snappy` + +An SSZ-snappy encoded object of type `BeaconState` to initialize the blockchain from. The state's `slot` is epoch aligned. + +### `steps.yaml` + +The steps to execute in sequence. + +#### `new_block` execution step + +The new block described by the test step should be imported, but does not become head yet. + +```yaml +{ + fork_digest: string -- Encoded `ForkDigest`-context of `block` + data: string -- name of the `*.ssz_snappy` file to load + as a `SignedBeaconBlock` object +} +``` + +#### `new_head` execution step + +The given block (previously imported) should become head, leading to potential updates to: + +- The best `LightClientUpdate` for non-finalized sync committee periods. +- The latest `LightClientFinalityUpdate` and `LightClientOptimisticUpdate`. +- The latest finalized `Checkpoint` (across all branches). +- The available `LightClientBootstrap` instances for newly finalized `Checkpoint`s. + +```yaml +{ + head_block_root: Bytes32 -- string, hex encoded, with 0x prefix + checks: { + latest_finalized_checkpoint: { -- tracked across all branches + epoch: int -- integer, decimal + root: Bytes32 -- string, hex encoded, with 0x prefix + } + bootstraps: [ -- one entry per `LightClientBootstrap` + block_root: Bytes32 -- string, hex encoded, with 0x prefix + bootstrap: { -- only exists if a `LightClientBootstrap` is available + fork_digest: string -- Encoded `ForkDigest`-context of `data` + data: string -- name of the `*.ssz_snappy` file to load + as a `LightClientBootstrap` object + } + ] + best_updates: [ -- one entry per sync committee period + period: int, -- integer, decimal + update: { -- only exists if a best `LightClientUpdate` is available + fork_digest: string -- Encoded `ForkDigest`-context of `data` + data: string -- name of the `*.ssz_snappy` file to load + as a `LightClientUpdate` object + } + ] + latest_finality_update: { -- only exists if a `LightClientFinalityUpdate` is available + fork_digest: string -- Encoded `ForkDigest`-context of `data` + data: string -- name of the `*.ssz_snappy` file to load + as a `LightClientFinalityUpdate` object + } + latest_optimistic_update: { -- only exists if a `LightClientOptimisticUpdate` is available + fork_digest: string -- Encoded `ForkDigest`-context of `data` + data: string -- name of the `*.ssz_snappy` file to load + as a `LightClientOptimisticUpdate` object + } + } +} +``` + +## Condition + +A test-runner should initialize a simplified blockchain from `initial_state`. An external signal is used to control fork choice. The test-runner should then proceed to execute all the test steps in sequence, collecting light client data during execution. After each `new_head` step, it should verify that the collected light client data matches the provided `checks`. diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index cfe34aee4b..341321a2ae 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -4,6 +4,7 @@ if __name__ == "__main__": altair_mods = {key: 'eth2spec.test.altair.light_client.test_' + key for key in [ + 'data_collection', 'single_merkle_proof', 'sync', 'update_ranking', From 2154298e080ff30d8adecc34be7ee204f64174f9 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 6 Feb 2024 13:01:58 +0100 Subject: [PATCH 03/20] Typo --- tests/formats/light_client/data_collection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/formats/light_client/data_collection.md b/tests/formats/light_client/data_collection.md index d8f13e5ed0..f9c1fa7a0e 100644 --- a/tests/formats/light_client/data_collection.md +++ b/tests/formats/light_client/data_collection.md @@ -1,6 +1,6 @@ # Light client data collection tests -This series of tests provies reference test vectors for validating that a full node collects canonical data for serving to light clients implementing the light client sync protocol to sync to the latest block header. +This series of tests provides reference test vectors for validating that a full node collects canonical data for serving to light clients implementing the light client sync protocol to sync to the latest block header. ## Test case format From 248f32b59a81d44e33612cfd5800f00a5973b119 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 6 Feb 2024 13:49:21 +0100 Subject: [PATCH 04/20] Lint --- .../light_client/test_data_collection.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py index 264c654810..2cc39131c1 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py @@ -26,8 +26,8 @@ def next_epoch_boundary_slot(spec, slot): - ## Compute the first possible epoch boundary state slot of a `Checkpoint` - ## referring to a block at given slot. + # Compute the first possible epoch boundary state slot of a `Checkpoint` + # referring to a block at given slot. epoch = spec.compute_epoch_at_slot(slot + spec.SLOTS_PER_EPOCH - 1) return spec.compute_start_slot_at_epoch(epoch) @@ -212,7 +212,8 @@ def get_current_sync_committee_for_finalized_period(test, period): # -> Optiona block = test.blocks[bid.root] state = test.finalized_checkpoint_states[block.data.message.state_root] if sync_committee_slot > state.data.slot: - state.spec, state.data, _ = transition_across_forks(state.spec, state.data, sync_committee_slot, phases=test.phases) + state.spec, state.data, _ = transition_across_forks( + state.spec, state.data, sync_committee_slot, phases=test.phases) assert is_post_altair(state.spec) return state.data.current_sync_committee @@ -242,8 +243,8 @@ def sync_aggregate_for_block_id(test, bid): # -> Optional[SyncAggregate] def get_light_client_data(lc_data_store, bid): # -> CachedLightClientData - ## Fetch cached light client data about a given block. - ## Data must be cached (`cache_light_client_data`) before calling this function. + # Fetch cached light client data about a given block. + # Data must be cached (`cache_light_client_data`) before calling this function. try: return lc_data_store.cache.data[bid] except KeyError: @@ -251,9 +252,9 @@ def get_light_client_data(lc_data_store, bid): # -> CachedLightClientData def cache_light_client_data(lc_data_store, spec, state, bid, current_period_best_update, latest_signature_slot): - ## Cache data for a given block and its post-state to speed up creating future - ## `LightClientUpdate` and `LightClientBootstrap` instances that refer to this - ## block and state. + # Cache data for a given block and its post-state to speed up creating future + # `LightClientUpdate` and `LightClientBootstrap` instances that refer to this + # block and state. cached_data = CachedLightClientData( current_sync_committee_branch=spec.compute_merkle_proof(state, spec.CURRENT_SYNC_COMMITTEE_GINDEX), next_sync_committee_branch=spec.compute_merkle_proof(state, spec.NEXT_SYNC_COMMITTEE_GINDEX), @@ -268,8 +269,8 @@ def cache_light_client_data(lc_data_store, spec, state, bid, current_period_best def delete_light_client_data(lc_data_store, bid): - ## Delete cached light client data for a given block. This needs to be called - ## when a block becomes unreachable due to finalization of a different fork. + # Delete cached light client data for a given block. This needs to be called + # when a block becomes unreachable due to finalization of a different fork. del lc_data_store.cache.data[bid] @@ -335,9 +336,9 @@ def create_light_client_update_from_light_client_data(test, def create_light_client_update(test, spec, state, block, parent_bid): - ## Create `LightClientUpdate` instances for a given block and its post-state, - ## and keep track of best / latest ones. Data about the parent block's - ## post-state must be cached (`cache_light_client_data`) before calling this. + # Create `LightClientUpdate` instances for a given block and its post-state, + # and keep track of best / latest ones. Data about the parent block's + # post-state must be cached (`cache_light_client_data`) before calling this. # Verify attested block (parent) is recent enough and that state is available attested_bid = parent_bid @@ -421,7 +422,7 @@ def create_light_client_bootstrap(test, spec, bid): def process_new_block_for_light_client(test, spec, state, block, parent_bid): - ## Update light client data with information from a new block. + # Update light client data with information from a new block. if block.message.slot < test.lc_data_store.cache.tail_slot: return @@ -432,8 +433,8 @@ def process_new_block_for_light_client(test, spec, state, block, parent_bid): def process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid): - ## Update light client data to account for a new head block. - ## Note that `old_finalized_bid` is not yet updated when this is called. + # Update light client data to account for a new head block. + # Note that `old_finalized_bid` is not yet updated when this is called. if head_bid.slot < test.lc_data_store.cache.tail_slot: return @@ -477,9 +478,9 @@ def process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid def process_finalization_for_light_client(test, spec, finalized_bid, old_finalized_bid): - ## Prune cached data that is no longer useful for creating future - ## `LightClientUpdate` and `LightClientBootstrap` instances. - ## This needs to be called whenever `finalized_checkpoint` changes. + # Prune cached data that is no longer useful for creating future + # `LightClientUpdate` and `LightClientBootstrap` instances. + # This needs to be called whenever `finalized_checkpoint` changes. finalized_slot = finalized_bid.slot if finalized_slot < test.lc_data_store.cache.tail_slot: return From c0d037f1b4648738683538aa30ca8ef77bb1a600 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 23 Feb 2024 13:56:56 +0100 Subject: [PATCH 05/20] Fix missing `optimistc_update` in new tests --- .../eth2spec/test/altair/light_client/test_data_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py index 2cc39131c1..8cd32e40a1 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py @@ -747,13 +747,13 @@ def select_new_head(test, spec, head_bid): state.data.genesis_validators_root, ) checks["latest_finality_update"] = finality_update_obj - optimistic_update = get_light_client_finality_update(test) + optimistic_update = get_light_client_optimistic_update(test) if optimistic_update.spec is not None: optimistic_update_obj = yield from encode_object( test, "optimistic_update", optimistic_update, optimistic_update.data.attested_header.beacon.slot, state.data.genesis_validators_root, ) - checks["latest_finality_update"] = optimistic_update_obj + checks["latest_optimistic_update"] = optimistic_update_obj test.steps.append({ "new_head": { From b8f0ddcf78da9da31e99162648b17f82b709a29c Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sun, 3 Mar 2024 20:49:37 +0100 Subject: [PATCH 06/20] Add more tests for multi-period reorgs --- .../light_client/test_data_collection.py | 157 +++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py index 8cd32e40a1..55ee5a74be 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py @@ -4,11 +4,16 @@ from eth_utils import encode_hex from eth2spec.test.context import ( spec_state_test_with_matching_config, + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, with_presets, + with_state, with_light_client, ) from eth2spec.test.helpers.constants import ( - ALTAIR, + ALTAIR, BELLATRIX, CAPELLA, DENEB, MINIMAL, ) from eth2spec.test.helpers.fork_transition import ( @@ -933,3 +938,153 @@ def test_light_client_data_collection(spec, state): # Finish test yield from finish_test(test) + + +def run_test_multi_fork(spec, phases, state, fork_1, fork_2): + # Start test + test = yield from setup_test(spec, state, phases=phases) + + # Genesis block is post Altair and is finalized, so can be used as bootstrap + genesis_bid = BlockId(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) + assert bootstrap_bid(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid + + # Shared history up to final epoch of period before `fork_1` + fork_1_epoch = getattr(phases[fork_1].config, fork_1.upper() + '_FORK_EPOCH') + fork_1_period = spec.compute_sync_committee_period(fork_1_epoch) + slot = compute_start_slot_at_sync_committee_period(spec, fork_1_period) - spec.SLOTS_PER_EPOCH + spec, state, bid = yield from add_new_block(test, spec, state, slot=slot, num_sync_participants=1) + yield from select_new_head(test, spec, bid) + assert get_light_client_bootstrap(test, bid.root).spec is None + slot_period = spec.compute_sync_committee_period_at_slot(slot) + if slot_period == 0: + assert update_attested_bid(get_light_client_update_for_period(test, 0).data) == genesis_bid + else: + for period in range(0, slot_period): + assert get_light_client_update_for_period(test, period).spec is None # attested period != signature period + state_period = spec.compute_sync_committee_period_at_slot(state.slot) + + # Branch A: Advance past `fork_2`, having blocks at slots 0 and 4 of each epoch + spec_a = spec + state_a = state + slot_a = state_a.slot + bids_a = [bid] + num_sync_participants_a = 1 + fork_2_epoch = getattr(phases[fork_2].config, fork_2.upper() + '_FORK_EPOCH') + while spec_a.get_current_epoch(state_a) <= fork_2_epoch: + attested_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + slot_a += 4 + signature_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + if signature_period != attested_period: + num_sync_participants_a = 0 + num_sync_participants_a += 1 + spec_a, state_a, bid_a = yield from add_new_block( + test, spec_a, state_a, slot=slot_a, num_sync_participants=num_sync_participants_a) + yield from select_new_head(test, spec_a, bid_a) + for bid in bids_a: + assert get_light_client_bootstrap(test, bid.root).spec is None + if attested_period == signature_period: + assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] + else: + assert signature_period == attested_period + 1 + assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] + assert get_light_client_update_for_period(test, signature_period).spec is None + assert update_attested_bid(get_light_client_finality_update(test).data) == bids_a[-1] + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bids_a[-1] + bids_a.append(bid_a) + + # Branch B: Advance past `fork_2`, having blocks at slots 1 and 5 of each epoch but no sync participation + spec_b = spec + state_b = state + slot_b = state_b.slot + bids_b = [bid] + while spec_b.get_current_epoch(state_b) <= fork_2_epoch: + slot_b += 4 + signature_period = spec_b.compute_sync_committee_period_at_slot(slot_b) + spec_b, state_b, bid_b = yield from add_new_block( + test, spec_b, state_b, slot=slot_b) + # Simulate that this does not become head yet, e.g., this branch was withheld + for bid in bids_b: + assert get_light_client_bootstrap(test, bid.root).spec is None + bids_b.append(bid_b) + + # Branch B: Another block that becomes head + attested_period = spec_b.compute_sync_committee_period_at_slot(slot_b) + slot_b += 1 + signature_period = spec_b.compute_sync_committee_period_at_slot(slot_b) + num_sync_participants_b = 1 + spec_b, state_b, bid_b = yield from add_new_block( + test, spec_b, state_b, slot=slot_b, num_sync_participants=num_sync_participants_b) + yield from select_new_head(test, spec_b, bid_b) + for bid in bids_b: + assert get_light_client_bootstrap(test, bid.root).spec is None + if attested_period == signature_period: + assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_b[-1] + else: + assert signature_period == attested_period + 1 + assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_b[-2] + assert get_light_client_update_for_period(test, signature_period).spec is None + assert update_attested_bid(get_light_client_finality_update(test).data) == bids_b[-1] + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bids_b[-1] + bids_b.append(bid_b) + + # All data for periods between the common ancestor of the two branches should have reorged. + # As there was no sync participation on branch B, that means it is deleted. + state_b_period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) + for period in range(state_period + 1, state_b_period): + assert get_light_client_update_for_period(test, period).spec is None + + # Branch A: Another block, reorging branch B once more + attested_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + slot_a = slot_b + 1 + signature_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + if signature_period != attested_period: + num_sync_participants_a = 0 + num_sync_participants_a += 1 + spec_a, state_a, bid_a = yield from add_new_block( + test, spec_a, state_a, slot=slot_a, num_sync_participants=num_sync_participants_a) + yield from select_new_head(test, spec_a, bid_a) + for bid in bids_a: + assert get_light_client_bootstrap(test, bid.root).spec is None + if attested_period == signature_period: + assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] + else: + assert signature_period == attested_period + 1 + assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] + assert get_light_client_update_for_period(test, signature_period).spec is None + assert update_attested_bid(get_light_client_finality_update(test).data) == bids_a[-1] + assert update_attested_bid(get_light_client_optimistic_update(test).data) == bids_a[-1] + bids_a.append(bid_a) + + # Data has been restored + state_a_period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + for period in range(state_period + 1, state_a_period): + assert get_light_client_update_for_period(test, period).spec is not None + + # Finish test + yield from finish_test(test) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 + 'DENEB_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_capella_deneb_reorg_aligned(spec, phases, state): + yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) + 'DENEB_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_capella_deneb_reorg_unaligned(spec, phases, state): + yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) From 946849637f89c8c182c71f5f8a16ac0fe6d216dc Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Fri, 22 Nov 2024 07:20:53 -0600 Subject: [PATCH 07/20] Fix nits in data_collection format --- tests/formats/light_client/data_collection.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/formats/light_client/data_collection.md b/tests/formats/light_client/data_collection.md index f9c1fa7a0e..b0d17a68e9 100644 --- a/tests/formats/light_client/data_collection.md +++ b/tests/formats/light_client/data_collection.md @@ -18,7 +18,7 @@ The new block described by the test step should be imported, but does not become ```yaml { - fork_digest: string -- Encoded `ForkDigest`-context of `block` + fork_digest: string -- encoded `ForkDigest`-context of `block` data: string -- name of the `*.ssz_snappy` file to load as a `SignedBeaconBlock` object } @@ -44,26 +44,26 @@ The given block (previously imported) should become head, leading to potential u bootstraps: [ -- one entry per `LightClientBootstrap` block_root: Bytes32 -- string, hex encoded, with 0x prefix bootstrap: { -- only exists if a `LightClientBootstrap` is available - fork_digest: string -- Encoded `ForkDigest`-context of `data` + fork_digest: string -- encoded `ForkDigest`-context of `data` data: string -- name of the `*.ssz_snappy` file to load as a `LightClientBootstrap` object } ] best_updates: [ -- one entry per sync committee period - period: int, -- integer, decimal + period: int -- integer, decimal update: { -- only exists if a best `LightClientUpdate` is available - fork_digest: string -- Encoded `ForkDigest`-context of `data` + fork_digest: string -- encoded `ForkDigest`-context of `data` data: string -- name of the `*.ssz_snappy` file to load as a `LightClientUpdate` object } ] latest_finality_update: { -- only exists if a `LightClientFinalityUpdate` is available - fork_digest: string -- Encoded `ForkDigest`-context of `data` + fork_digest: string -- encoded `ForkDigest`-context of `data` data: string -- name of the `*.ssz_snappy` file to load as a `LightClientFinalityUpdate` object } latest_optimistic_update: { -- only exists if a `LightClientOptimisticUpdate` is available - fork_digest: string -- Encoded `ForkDigest`-context of `data` + fork_digest: string -- encoded `ForkDigest`-context of `data` data: string -- name of the `*.ssz_snappy` file to load as a `LightClientOptimisticUpdate` object } From 5639ca69d6ae13ffbaeafd29561e5fce448394fe Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 22 Nov 2024 09:45:56 -0600 Subject: [PATCH 08/20] Rename two classes for consistency --- .../light_client/test_data_collection.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py index 27e8e5437c..57a7183077 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py @@ -42,13 +42,13 @@ def next_epoch_boundary_slot(spec, slot): @dataclass(frozen=True) -class BlockId(object): +class BlockID(object): slot: Any root: Any def block_to_block_id(block): - return BlockId( + return BlockID( slot=block.message.slot, root=block.message.hash_tree_root(), ) @@ -57,18 +57,18 @@ def block_to_block_id(block): def state_to_block_id(state): parent_header = state.latest_block_header.copy() parent_header.state_root = state.hash_tree_root() - return BlockId(slot=parent_header.slot, root=parent_header.hash_tree_root()) + return BlockID(slot=parent_header.slot, root=parent_header.hash_tree_root()) def bootstrap_bid(bootstrap): - return BlockId( + return BlockID( slot=bootstrap.header.beacon.slot, root=bootstrap.header.beacon.hash_tree_root(), ) def update_attested_bid(update): - return BlockId( + return BlockID( slot=update.attested_header.beacon.slot, root=update.attested_header.beacon.hash_tree_root(), ) @@ -136,7 +136,7 @@ class LightClientDataCache(object): # Cached data for creating future `LightClientUpdate` instances. # Key is the block ID of which the post state was used to get the data. # Data stored for the finalized head block and all non-finalized blocks. - data: Dict[BlockId, CachedLightClientData] + data: Dict[BlockID, CachedLightClientData] # Light client data for the latest slot that was signed by at least # `MIN_SYNC_COMMITTEE_PARTICIPANTS`. May be older than head @@ -147,7 +147,7 @@ class LightClientDataCache(object): @dataclass -class LightClientDataDb(object): +class LightClientDataDB(object): headers: Dict[Any, ForkedLightClientHeader] # Root -> ForkedLightClientHeader current_branches: Dict[Any, Any] # Slot -> CurrentSyncCommitteeBranch sync_committees: Dict[Any, Any] # SyncCommitteePeriod -> SyncCommittee @@ -162,7 +162,7 @@ class LightClientDataStore(object): cache: LightClientDataCache # Persistent light client data - db: LightClientDataDb + db: LightClientDataDB @dataclass @@ -179,14 +179,14 @@ class LightClientDataCollectionTest(object): states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState finalized_checkpoint_states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState latest_finalized_epoch: Any # Epoch - latest_finalized_bid: BlockId + latest_finalized_bid: BlockID historical_tail_slot: Any # Slot # Light client data lc_data_store: LightClientDataStore -def get_ancestor_of_block_id(test, bid, slot): # -> Optional[BlockId] +def get_ancestor_of_block_id(test, bid, slot): # -> Optional[BlockID] try: block = test.blocks[bid.root] while True: @@ -198,10 +198,10 @@ def get_ancestor_of_block_id(test, bid, slot): # -> Optional[BlockId] return None -def block_id_at_finalized_slot(test, slot): # -> Optional[BlockId] +def block_id_at_finalized_slot(test, slot): # -> Optional[BlockID] while slot >= test.historical_tail_slot: try: - return BlockId(slot=slot, root=test.finalized_block_roots[slot]) + return BlockID(slot=slot, root=test.finalized_block_roots[slot]) except KeyError: slot = slot - 1 return None @@ -586,7 +586,7 @@ def setup_test(spec, state, phases=None): states={}, finalized_checkpoint_states={}, latest_finalized_epoch=state.finalized_checkpoint.epoch, - latest_finalized_bid=BlockId( + latest_finalized_bid=BlockID( slot=spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch), root=state.finalized_checkpoint.root, ), @@ -598,7 +598,7 @@ def setup_test(spec, state, phases=None): latest=ForkedLightClientFinalityUpdate(spec=None, data=None), tail_slot=max(state.slot, spec.compute_start_slot_at_epoch(spec.config.ALTAIR_FORK_EPOCH)), ), - db=LightClientDataDb( + db=LightClientDataDB( headers={}, current_branches={}, sync_committees={}, @@ -792,7 +792,7 @@ def test_light_client_data_collection(spec, state): test = yield from setup_test(spec, state) # Genesis block is post Altair and is finalized, so can be used as bootstrap - genesis_bid = BlockId(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) + genesis_bid = BlockID(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) assert bootstrap_bid(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid # No blocks have been imported, so no other light client data is available @@ -961,7 +961,7 @@ def run_test_multi_fork(spec, phases, state, fork_1, fork_2): test = yield from setup_test(spec, state, phases=phases) # Genesis block is post Altair and is finalized, so can be used as bootstrap - genesis_bid = BlockId(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) + genesis_bid = BlockID(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) assert bootstrap_bid(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid # Shared history up to final epoch of period before `fork_1` From aff4e348354cab9be3ffadb90a4ac78eeb41cf82 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 22 Nov 2024 10:43:05 -0600 Subject: [PATCH 09/20] Move bellatrix/capella tests into respective dirs --- .../test/bellatrix/light_client/__init__.py | 0 .../light_client/test_data_collection.py | 41 +++++++++++++++++++ .../light_client/test_data_collection.py | 40 ++++++++++++++++++ tests/generators/light_client/main.py | 8 +++- 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py create mode 100644 tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py new file mode 100644 index 0000000000..dced8d0b3e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py @@ -0,0 +1,41 @@ +from eth2spec.test.context import ( + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, + with_presets, + with_state, +) +from eth2spec.test.helpers.constants import ( + BELLATRIX, CAPELLA, DENEB, + MINIMAL, +) +from eth2spec.test.altair.light_client.test_data_collection import ( + run_test_multi_fork +) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 + 'DENEB_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_capella_deneb_reorg_aligned(spec, phases, state): + yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) + 'DENEB_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_capella_deneb_reorg_unaligned(spec, phases, state): + yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) diff --git a/tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py new file mode 100644 index 0000000000..7911f1c320 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py @@ -0,0 +1,40 @@ +from eth2spec.test.context import ( + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, + with_presets, + with_state, +) +from eth2spec.test.helpers.constants import ( + CAPELLA, DENEB, ELECTRA, + MINIMAL, +) +from eth2spec.test.altair.light_client.test_data_collection import ( + run_test_multi_fork +) + +@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) +@spec_test +@with_config_overrides({ + 'DENEB_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 + 'ELECTRA_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_electra_reorg_aligned(spec, phases, state): + yield from run_test_multi_fork(spec, phases, state, DENEB, ELECTRA) + + +@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) +@spec_test +@with_config_overrides({ + 'DENEB_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) + 'ELECTRA_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_electra_reorg_unaligned(spec, phases, state): + yield from run_test_multi_fork(spec, phases, state, DENEB, ELECTRA) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index 04d1d423be..2501773ac5 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -9,12 +9,18 @@ 'sync', 'update_ranking', ]} - bellatrix_mods = altair_mods + + _new_bellatrix_mods = {key: 'eth2spec.test.bellatrix.light_client.test_' + key for key in [ + 'data_collection', + ]} + bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) _new_capella_mods = {key: 'eth2spec.test.capella.light_client.test_' + key for key in [ + 'data_collection', 'single_merkle_proof', ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) + deneb_mods = capella_mods electra_mods = deneb_mods From b6259a9fd7f6bca6ae89dc09f04f2f0d61638469 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Fri, 22 Nov 2024 10:59:05 -0600 Subject: [PATCH 10/20] Revert "Move bellatrix/capella tests into respective dirs" This reverts commit aff4e348354cab9be3ffadb90a4ac78eeb41cf82. --- .../test/bellatrix/light_client/__init__.py | 0 .../light_client/test_data_collection.py | 41 ------------------- .../light_client/test_data_collection.py | 40 ------------------ tests/generators/light_client/main.py | 8 +--- 4 files changed, 1 insertion(+), 88 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py delete mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py delete mode 100644 tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py deleted file mode 100644 index dced8d0b3e..0000000000 --- a/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py +++ /dev/null @@ -1,41 +0,0 @@ -from eth2spec.test.context import ( - spec_test, - with_config_overrides, - with_matching_spec_config, - with_phases, - with_presets, - with_state, -) -from eth2spec.test.helpers.constants import ( - BELLATRIX, CAPELLA, DENEB, - MINIMAL, -) -from eth2spec.test.altair.light_client.test_data_collection import ( - run_test_multi_fork -) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 - 'DENEB_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_capella_deneb_reorg_aligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) - 'DENEB_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_capella_deneb_reorg_unaligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) diff --git a/tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py deleted file mode 100644 index 7911f1c320..0000000000 --- a/tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py +++ /dev/null @@ -1,40 +0,0 @@ -from eth2spec.test.context import ( - spec_test, - with_config_overrides, - with_matching_spec_config, - with_phases, - with_presets, - with_state, -) -from eth2spec.test.helpers.constants import ( - CAPELLA, DENEB, ELECTRA, - MINIMAL, -) -from eth2spec.test.altair.light_client.test_data_collection import ( - run_test_multi_fork -) - -@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) -@spec_test -@with_config_overrides({ - 'DENEB_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 - 'ELECTRA_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_electra_reorg_aligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, DENEB, ELECTRA) - - -@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) -@spec_test -@with_config_overrides({ - 'DENEB_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) - 'ELECTRA_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_electra_reorg_unaligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, DENEB, ELECTRA) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index 2501773ac5..04d1d423be 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -9,18 +9,12 @@ 'sync', 'update_ranking', ]} - - _new_bellatrix_mods = {key: 'eth2spec.test.bellatrix.light_client.test_' + key for key in [ - 'data_collection', - ]} - bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) + bellatrix_mods = altair_mods _new_capella_mods = {key: 'eth2spec.test.capella.light_client.test_' + key for key in [ - 'data_collection', 'single_merkle_proof', ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) - deneb_mods = capella_mods electra_mods = deneb_mods From e00e866b84cb5b1b3a5fd25ef9af6d43088cb479 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Nov 2024 13:15:08 +0100 Subject: [PATCH 11/20] Synchronise capitalization change request across files --- tests/formats/light_client/sync.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/formats/light_client/sync.md b/tests/formats/light_client/sync.md index 1706b4c162..c6e62a7c8b 100644 --- a/tests/formats/light_client/sync.md +++ b/tests/formats/light_client/sync.md @@ -9,8 +9,8 @@ This series of tests provides reference test vectors for validating that a light ```yaml genesis_validators_root: Bytes32 -- string, hex encoded, with 0x prefix trusted_block_root: Bytes32 -- string, hex encoded, with 0x prefix -bootstrap_fork_digest: string -- Encoded `ForkDigest`-context of `bootstrap` -store_fork_digest: string -- Encoded `ForkDigest`-context of `store` object being tested +bootstrap_fork_digest: string -- encoded `ForkDigest`-context of `bootstrap` +store_fork_digest: string -- encoded `ForkDigest`-context of `store` object being tested ``` ### `bootstrap.ssz_snappy` @@ -60,7 +60,7 @@ The function `process_light_client_update(store, update, current_slot, genesis_v ```yaml { - update_fork_digest: string -- Encoded `ForkDigest`-context of `update` + update_fork_digest: string -- encoded `ForkDigest`-context of `update` update: string -- name of the `*.ssz_snappy` file to load as a `LightClientUpdate` object current_slot: int -- integer, decimal @@ -78,7 +78,7 @@ The `store` should be upgraded to reflect the new `store_fork_digest`: ```yaml { - store_fork_digest: string -- Encoded `ForkDigest`-context of `store` + store_fork_digest: string -- encoded `ForkDigest`-context of `store` checks: {: value} -- the assertions. } ``` From 84bef3c6881edaa4892362461433a2de3f848e52 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Nov 2024 13:50:37 +0100 Subject: [PATCH 12/20] Split LC sync test into multiple files --- .../test/altair/light_client/test_sync.py | 460 +----------------- .../test/capella/light_client/test_sync.py | 36 ++ .../test/deneb/light_client/__init__.py | 0 .../test/deneb/light_client/test_sync.py | 50 ++ .../test/electra/light_client/__init__.py | 0 .../test/electra/light_client/test_sync.py | 64 +++ .../test/helpers/light_client_sync.py | 342 +++++++++++++ 7 files changed, 505 insertions(+), 447 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py create mode 100644 tests/core/pyspec/eth2spec/test/deneb/light_client/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py create mode 100644 tests/core/pyspec/eth2spec/test/electra/light_client/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index 45c7d77887..8000ceb799 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -1,14 +1,6 @@ -from typing import (Any, Dict, List) - -from eth_utils import encode_hex from eth2spec.test.context import ( spec_state_test_with_matching_config, - spec_test, - with_config_overrides, - with_matching_spec_config, - with_phases, with_presets, - with_state, with_light_client, ) from eth2spec.test.helpers.attestations import ( @@ -16,23 +8,17 @@ state_transition_with_full_block, ) from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA, MINIMAL, ) -from eth2spec.test.helpers.fork_transition import ( - do_fork, - transition_across_forks, -) -from eth2spec.test.helpers.forks import ( - get_spec_for_fork_version, - is_post_capella, is_post_deneb, is_post_electra, -) from eth2spec.test.helpers.light_client import ( - compute_start_slot_at_next_sync_committee_period, get_sync_aggregate, - upgrade_lc_bootstrap_to_new_spec, - upgrade_lc_update_to_new_spec, - upgrade_lc_store_to_new_spec, + compute_start_slot_at_next_sync_committee_period, +) +from eth2spec.test.helpers.light_client_sync import ( + emit_force_update, + emit_update, + finish_lc_sync_test, + setup_lc_sync_test, ) from eth2spec.test.helpers.state import ( next_slots, @@ -40,162 +26,12 @@ ) -class LightClientSyncTest(object): - steps: List[Dict[str, Any]] - genesis_validators_root: Any - s_spec: Any - store: Any - - -def get_store_fork_version(s_spec): - if is_post_electra(s_spec): - return s_spec.config.ELECTRA_FORK_VERSION - if is_post_deneb(s_spec): - return s_spec.config.DENEB_FORK_VERSION - if is_post_capella(s_spec): - return s_spec.config.CAPELLA_FORK_VERSION - return s_spec.config.ALTAIR_FORK_VERSION - - -def setup_test(spec, state, s_spec=None, phases=None): - test = LightClientSyncTest() - test.steps = [] - - if s_spec is None: - s_spec = spec - if phases is None: - phases = { - spec.fork: spec, - s_spec.fork: s_spec, - } - test.s_spec = s_spec - - yield "genesis_validators_root", "meta", "0x" + state.genesis_validators_root.hex() - test.genesis_validators_root = state.genesis_validators_root - - next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2 - 1) - trusted_block = state_transition_with_full_block(spec, state, True, True) - trusted_block_root = trusted_block.message.hash_tree_root() - yield "trusted_block_root", "meta", "0x" + trusted_block_root.hex() - - data_fork_version = spec.compute_fork_version(spec.compute_epoch_at_slot(trusted_block.message.slot)) - data_fork_digest = spec.compute_fork_digest(data_fork_version, test.genesis_validators_root) - d_spec = get_spec_for_fork_version(spec, data_fork_version, phases) - data = d_spec.create_light_client_bootstrap(state, trusted_block) - yield "bootstrap_fork_digest", "meta", encode_hex(data_fork_digest) - yield "bootstrap", data - - upgraded = upgrade_lc_bootstrap_to_new_spec(d_spec, test.s_spec, data, phases) - test.store = test.s_spec.initialize_light_client_store(trusted_block_root, upgraded) - store_fork_version = get_store_fork_version(test.s_spec) - store_fork_digest = test.s_spec.compute_fork_digest(store_fork_version, test.genesis_validators_root) - yield "store_fork_digest", "meta", encode_hex(store_fork_digest) - - return test - - -def finish_test(test): - yield "steps", test.steps - - -def get_update_file_name(d_spec, update): - if d_spec.is_sync_committee_update(update): - suffix1 = "s" - else: - suffix1 = "x" - if d_spec.is_finality_update(update): - suffix2 = "f" - else: - suffix2 = "x" - return f"update_{encode_hex(update.attested_header.beacon.hash_tree_root())}_{suffix1}{suffix2}" - - -def get_checks(s_spec, store): - if is_post_capella(s_spec): - return { - "finalized_header": { - 'slot': int(store.finalized_header.beacon.slot), - 'beacon_root': encode_hex(store.finalized_header.beacon.hash_tree_root()), - 'execution_root': encode_hex(s_spec.get_lc_execution_root(store.finalized_header)), - }, - "optimistic_header": { - 'slot': int(store.optimistic_header.beacon.slot), - 'beacon_root': encode_hex(store.optimistic_header.beacon.hash_tree_root()), - 'execution_root': encode_hex(s_spec.get_lc_execution_root(store.optimistic_header)), - }, - } - - return { - "finalized_header": { - 'slot': int(store.finalized_header.beacon.slot), - 'beacon_root': encode_hex(store.finalized_header.beacon.hash_tree_root()), - }, - "optimistic_header": { - 'slot': int(store.optimistic_header.beacon.slot), - 'beacon_root': encode_hex(store.optimistic_header.beacon.hash_tree_root()), - }, - } - - -def emit_force_update(test, spec, state): - current_slot = state.slot - test.s_spec.process_light_client_store_force_update(test.store, current_slot) - - yield from [] # Consistently enable `yield from` syntax in calling tests - test.steps.append({ - "force_update": { - "current_slot": int(current_slot), - "checks": get_checks(test.s_spec, test.store), - } - }) - - -def emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, with_next=True, phases=None): - data_fork_version = spec.compute_fork_version(spec.compute_epoch_at_slot(attested_block.message.slot)) - data_fork_digest = spec.compute_fork_digest(data_fork_version, test.genesis_validators_root) - d_spec = get_spec_for_fork_version(spec, data_fork_version, phases) - data = d_spec.create_light_client_update(state, block, attested_state, attested_block, finalized_block) - if not with_next: - data.next_sync_committee = spec.SyncCommittee() - data.next_sync_committee_branch = spec.NextSyncCommitteeBranch() - current_slot = state.slot - - upgraded = upgrade_lc_update_to_new_spec(d_spec, test.s_spec, data, phases) - test.s_spec.process_light_client_update(test.store, upgraded, current_slot, test.genesis_validators_root) - - yield get_update_file_name(d_spec, data), data - test.steps.append({ - "process_update": { - "update_fork_digest": encode_hex(data_fork_digest), - "update": get_update_file_name(d_spec, data), - "current_slot": int(current_slot), - "checks": get_checks(test.s_spec, test.store), - } - }) - return upgraded - - -def emit_upgrade_store(test, new_s_spec, phases=None): - test.store = upgrade_lc_store_to_new_spec(test.s_spec, new_s_spec, test.store, phases) - test.s_spec = new_s_spec - store_fork_version = get_store_fork_version(test.s_spec) - store_fork_digest = test.s_spec.compute_fork_digest(store_fork_version, test.genesis_validators_root) - - yield from [] # Consistently enable `yield from` syntax in calling tests - test.steps.append({ - "upgrade_store": { - "store_fork_digest": encode_hex(store_fork_digest), - "checks": get_checks(test.s_spec, test.store), - } - }) - - @with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_light_client_sync(spec, state): # Start test - test = yield from setup_test(spec, state) + test = yield from setup_lc_sync_test(spec, state) # Initial `LightClientUpdate`, populating `store.next_sync_committee` # ``` @@ -409,7 +245,7 @@ def test_light_client_sync(spec, state): assert test.store.optimistic_header.beacon.slot == attested_state.slot # Finish test - yield from finish_test(test) + yield from finish_lc_sync_test(test) @with_light_client @@ -428,7 +264,7 @@ def test_supply_sync_committee_from_past_update(spec, state): past_state = state.copy() # Start test - test = yield from setup_test(spec, state) + test = yield from setup_lc_sync_test(spec, state) assert not spec.is_next_sync_committee_known(test.store) # Apply `LightClientUpdate` from the past, populating `store.next_sync_committee` @@ -439,7 +275,7 @@ def test_supply_sync_committee_from_past_update(spec, state): assert test.store.optimistic_header.beacon.slot == state.slot # Finish test - yield from finish_test(test) + yield from finish_lc_sync_test(test) @with_light_client @@ -447,7 +283,7 @@ def test_supply_sync_committee_from_past_update(spec, state): @with_presets([MINIMAL], reason="too slow") def test_advance_finality_without_sync_committee(spec, state): # Start test - test = yield from setup_test(spec, state) + test = yield from setup_lc_sync_test(spec, state) # Initial `LightClientUpdate`, populating `store.next_sync_committee` next_slots(spec, state, spec.SLOTS_PER_EPOCH - 1) @@ -515,274 +351,4 @@ def test_advance_finality_without_sync_committee(spec, state): assert test.store.optimistic_header.beacon.slot == attested_state.slot # Finish test - yield from finish_test(test) - - -def run_test_single_fork(spec, phases, state, fork): - # Start test - test = yield from setup_test(spec, state, phases=phases) - - # Initial `LightClientUpdate` - finalized_block = spec.SignedBeaconBlock() - finalized_block.message.state_root = state.hash_tree_root() - finalized_state = state.copy() - attested_block = state_transition_with_full_block(spec, state, True, True) - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update is None - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Jump to two slots before fork - fork_epoch = getattr(phases[fork].config, fork.upper() + '_FORK_EPOCH') - transition_to(spec, state, spec.compute_start_slot_at_epoch(fork_epoch) - 4) - attested_block = state_transition_with_full_block(spec, state, True, True) - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - update = yield from emit_update( - test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update == update - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Perform `LightClientStore` upgrade - yield from emit_upgrade_store(test, phases[fork], phases=phases) - update = test.store.best_valid_update - - # Final slot before fork, check that importing the pre-fork format still works - attested_block = block.copy() - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update == update - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Upgrade to post-fork spec, attested block is still before the fork - attested_block = block.copy() - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - state, block = do_fork(state, spec, phases[fork], fork_epoch, sync_aggregate=sync_aggregate) - spec = phases[fork] - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update == update - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Another block after the fork, this time attested block is after the fork - attested_block = block.copy() - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update == update - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Jump to next epoch - transition_to(spec, state, spec.compute_start_slot_at_epoch(fork_epoch + 1) - 2) - attested_block = state_transition_with_full_block(spec, state, True, True) - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update == update - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Finalize the fork - finalized_block = block.copy() - finalized_state = state.copy() - _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH - 1, True, True) - attested_block = state_transition_with_full_block(spec, state, True, True) - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update is None - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Finish test - yield from finish_test(test) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 3, # `setup_test` advances to epoch 2 -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=CAPELLA) -@with_presets([MINIMAL], reason="too slow") -def test_capella_fork(spec, phases, state): - yield from run_test_single_fork(spec, phases, state, CAPELLA) - - -@with_phases(phases=[CAPELLA], other_phases=[DENEB]) -@spec_test -@with_config_overrides({ - 'DENEB_FORK_EPOCH': 3, # `setup_test` advances to epoch 2 -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_fork(spec, phases, state): - yield from run_test_single_fork(spec, phases, state, DENEB) - - -@with_phases(phases=[DENEB], other_phases=[ELECTRA]) -@spec_test -@with_config_overrides({ - 'ELECTRA_FORK_EPOCH': 3, # `setup_test` advances to epoch 2 -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_electra_fork(spec, phases, state): - yield from run_test_single_fork(spec, phases, state, ELECTRA) - - -def run_test_multi_fork(spec, phases, state, fork_1, fork_2): - # Start test - test = yield from setup_test(spec, state, phases[fork_2], phases) - - # Set up so that finalized is from `spec`, ... - finalized_block = spec.SignedBeaconBlock() - finalized_block.message.state_root = state.hash_tree_root() - finalized_state = state.copy() - - # ..., attested is from `fork_1`, ... - fork_1_epoch = getattr(phases[fork_1].config, fork_1.upper() + '_FORK_EPOCH') - spec, state, attested_block = transition_across_forks( - spec, - state, - spec.compute_start_slot_at_epoch(fork_1_epoch), - phases, - with_block=True, - ) - attested_state = state.copy() - - # ..., and signature is from `fork_2` - fork_2_epoch = getattr(phases[fork_2].config, fork_2.upper() + '_FORK_EPOCH') - spec, state, _ = transition_across_forks( - spec, state, spec.compute_start_slot_at_epoch(fork_2_epoch) - 1, phases) - sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) - spec, state, block = transition_across_forks( - spec, - state, - spec.compute_start_slot_at_epoch(fork_2_epoch), - phases, - with_block=True, - sync_aggregate=sync_aggregate, - ) - - # Check that update applies - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update is None - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Finish test - yield from finish_test(test) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 3, # `setup_test` advances to epoch 2 - 'DENEB_FORK_EPOCH': 4, -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_capella_deneb_fork(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB, ELECTRA]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 3, # `setup_test` advances to epoch 2 - 'DENEB_FORK_EPOCH': 4, - 'ELECTRA_FORK_EPOCH': 5, -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_capella_electra_fork(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, CAPELLA, ELECTRA) - - -@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) -@spec_test -@with_config_overrides({ - 'DENEB_FORK_EPOCH': 3, # `setup_test` advances to epoch 2 - 'ELECTRA_FORK_EPOCH': 4, -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_electra_fork(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, DENEB, ELECTRA) - - -def run_test_upgraded_store_with_legacy_data(spec, phases, state, fork): - # Start test (Legacy bootstrap with an upgraded store) - test = yield from setup_test(spec, state, phases[fork], phases) - - # Initial `LightClientUpdate` (check that the upgraded store can process it) - finalized_block = spec.SignedBeaconBlock() - finalized_block.message.state_root = state.hash_tree_root() - finalized_state = state.copy() - attested_block = state_transition_with_full_block(spec, state, True, True) - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update is None - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Finish test - yield from finish_test(test) - - -@with_phases(phases=[ALTAIR, BELLATRIX], other_phases=[CAPELLA]) -@spec_test -@with_state -@with_matching_spec_config(emitted_fork=CAPELLA) -@with_presets([MINIMAL], reason="too slow") -def test_capella_store_with_legacy_data(spec, phases, state): - yield from run_test_upgraded_store_with_legacy_data(spec, phases, state, CAPELLA) - - -@with_phases(phases=[ALTAIR, BELLATRIX, CAPELLA], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_store_with_legacy_data(spec, phases, state): - yield from run_test_upgraded_store_with_legacy_data(spec, phases, state, DENEB) - - -@with_phases(phases=[ALTAIR, BELLATRIX, CAPELLA, DENEB], other_phases=[CAPELLA, DENEB, ELECTRA]) -@spec_test -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_electra_store_with_legacy_data(spec, phases, state): - yield from run_test_upgraded_store_with_legacy_data(spec, phases, state, ELECTRA) + yield from finish_lc_sync_test(test) diff --git a/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py new file mode 100644 index 0000000000..3958900be5 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py @@ -0,0 +1,36 @@ +from eth2spec.test.context import ( + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, + with_presets, + with_state, +) +from eth2spec.test.helpers.constants import ( + ALTAIR, BELLATRIX, CAPELLA, + MINIMAL, +) +from eth2spec.test.helpers.light_client_sync import ( + run_lc_sync_test_single_fork, + run_lc_sync_test_upgraded_store_with_legacy_data, +) + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=CAPELLA) +@with_presets([MINIMAL], reason="too slow") +def test_capella_fork(spec, phases, state): + yield from run_lc_sync_test_single_fork(spec, phases, state, CAPELLA) + + +@with_phases(phases=[ALTAIR, BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_state +@with_matching_spec_config(emitted_fork=CAPELLA) +@with_presets([MINIMAL], reason="too slow") +def test_capella_store_with_legacy_data(spec, phases, state): + yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, CAPELLA) diff --git a/tests/core/pyspec/eth2spec/test/deneb/light_client/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/light_client/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py new file mode 100644 index 0000000000..d19e1e0238 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py @@ -0,0 +1,50 @@ +from eth2spec.test.context import ( + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, + with_presets, + with_state, +) +from eth2spec.test.helpers.constants import ( + ALTAIR, BELLATRIX, CAPELLA, DENEB, + MINIMAL, +) +from eth2spec.test.helpers.light_client_sync import ( + run_lc_sync_test_multi_fork, + run_lc_sync_test_single_fork, + run_lc_sync_test_upgraded_store_with_legacy_data, +) + +@with_phases(phases=[CAPELLA], other_phases=[DENEB]) +@spec_test +@with_config_overrides({ + 'DENEB_FORK_EPOCH': 3, # Test setup advances to epoch 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_fork(spec, phases, state): + yield from run_lc_sync_test_single_fork(spec, phases, state, DENEB) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'DENEB_FORK_EPOCH': 4, +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_capella_deneb_fork(spec, phases, state): + yield from run_lc_sync_test_multi_fork(spec, phases, state, CAPELLA, DENEB) + + +@with_phases(phases=[ALTAIR, BELLATRIX, CAPELLA], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_store_with_legacy_data(spec, phases, state): + yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, DENEB) diff --git a/tests/core/pyspec/eth2spec/test/electra/light_client/__init__.py b/tests/core/pyspec/eth2spec/test/electra/light_client/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py new file mode 100644 index 0000000000..2b20552d6b --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py @@ -0,0 +1,64 @@ +from eth2spec.test.context import ( + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, + with_presets, + with_state, +) +from eth2spec.test.helpers.constants import ( + ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA, + MINIMAL, +) +from eth2spec.test.helpers.light_client_sync import ( + run_lc_sync_test_multi_fork, + run_lc_sync_test_single_fork, + run_lc_sync_test_upgraded_store_with_legacy_data, +) + +@with_phases(phases=[DENEB], other_phases=[ELECTRA]) +@spec_test +@with_config_overrides({ + 'ELECTRA_FORK_EPOCH': 3, # Test setup advances to epoch 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_electra_fork(spec, phases, state): + yield from run_lc_sync_test_single_fork(spec, phases, state, ELECTRA) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB, ELECTRA]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'DENEB_FORK_EPOCH': 4, + 'ELECTRA_FORK_EPOCH': 5, +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_capella_electra_fork(spec, phases, state): + yield from run_lc_sync_test_multi_fork(spec, phases, state, CAPELLA, ELECTRA) + + +@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) +@spec_test +@with_config_overrides({ + 'DENEB_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'ELECTRA_FORK_EPOCH': 4, +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_electra_fork(spec, phases, state): + yield from run_lc_sync_test_multi_fork(spec, phases, state, DENEB, ELECTRA) + + +@with_phases(phases=[ALTAIR, BELLATRIX, CAPELLA, DENEB], other_phases=[CAPELLA, DENEB, ELECTRA]) +@spec_test +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_electra_store_with_legacy_data(spec, phases, state): + yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, ELECTRA) diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py b/tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py new file mode 100644 index 0000000000..e64b0a2eca --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py @@ -0,0 +1,342 @@ +from typing import (Any, Dict, List) + +from eth_utils import encode_hex +from eth2spec.test.helpers.attestations import ( + next_slots_with_attestations, + state_transition_with_full_block, +) +from eth2spec.test.helpers.fork_transition import ( + do_fork, + transition_across_forks, +) +from eth2spec.test.helpers.forks import ( + get_spec_for_fork_version, + is_post_capella, is_post_deneb, is_post_electra, +) +from eth2spec.test.helpers.light_client import ( + get_sync_aggregate, + upgrade_lc_bootstrap_to_new_spec, + upgrade_lc_update_to_new_spec, + upgrade_lc_store_to_new_spec, +) +from eth2spec.test.helpers.state import ( + next_slots, + transition_to, +) + + +class LightClientSyncTest(object): + steps: List[Dict[str, Any]] + genesis_validators_root: Any + s_spec: Any + store: Any + + +def _get_store_fork_version(s_spec): + if is_post_electra(s_spec): + return s_spec.config.ELECTRA_FORK_VERSION + if is_post_deneb(s_spec): + return s_spec.config.DENEB_FORK_VERSION + if is_post_capella(s_spec): + return s_spec.config.CAPELLA_FORK_VERSION + return s_spec.config.ALTAIR_FORK_VERSION + + +def setup_lc_sync_test(spec, state, s_spec=None, phases=None): + test = LightClientSyncTest() + test.steps = [] + + if s_spec is None: + s_spec = spec + if phases is None: + phases = { + spec.fork: spec, + s_spec.fork: s_spec, + } + test.s_spec = s_spec + + yield "genesis_validators_root", "meta", "0x" + state.genesis_validators_root.hex() + test.genesis_validators_root = state.genesis_validators_root + + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2 - 1) + trusted_block = state_transition_with_full_block(spec, state, True, True) + trusted_block_root = trusted_block.message.hash_tree_root() + yield "trusted_block_root", "meta", "0x" + trusted_block_root.hex() + + data_fork_version = spec.compute_fork_version(spec.compute_epoch_at_slot(trusted_block.message.slot)) + data_fork_digest = spec.compute_fork_digest(data_fork_version, test.genesis_validators_root) + d_spec = get_spec_for_fork_version(spec, data_fork_version, phases) + data = d_spec.create_light_client_bootstrap(state, trusted_block) + yield "bootstrap_fork_digest", "meta", encode_hex(data_fork_digest) + yield "bootstrap", data + + upgraded = upgrade_lc_bootstrap_to_new_spec(d_spec, test.s_spec, data, phases) + test.store = test.s_spec.initialize_light_client_store(trusted_block_root, upgraded) + store_fork_version = _get_store_fork_version(test.s_spec) + store_fork_digest = test.s_spec.compute_fork_digest(store_fork_version, test.genesis_validators_root) + yield "store_fork_digest", "meta", encode_hex(store_fork_digest) + + return test + + +def finish_lc_sync_test(test): + yield "steps", test.steps + + +def _get_update_file_name(d_spec, update): + if d_spec.is_sync_committee_update(update): + suffix1 = "s" + else: + suffix1 = "x" + if d_spec.is_finality_update(update): + suffix2 = "f" + else: + suffix2 = "x" + return f"update_{encode_hex(update.attested_header.beacon.hash_tree_root())}_{suffix1}{suffix2}" + + +def _get_checks(s_spec, store): + if is_post_capella(s_spec): + return { + "finalized_header": { + 'slot': int(store.finalized_header.beacon.slot), + 'beacon_root': encode_hex(store.finalized_header.beacon.hash_tree_root()), + 'execution_root': encode_hex(s_spec.get_lc_execution_root(store.finalized_header)), + }, + "optimistic_header": { + 'slot': int(store.optimistic_header.beacon.slot), + 'beacon_root': encode_hex(store.optimistic_header.beacon.hash_tree_root()), + 'execution_root': encode_hex(s_spec.get_lc_execution_root(store.optimistic_header)), + }, + } + + return { + "finalized_header": { + 'slot': int(store.finalized_header.beacon.slot), + 'beacon_root': encode_hex(store.finalized_header.beacon.hash_tree_root()), + }, + "optimistic_header": { + 'slot': int(store.optimistic_header.beacon.slot), + 'beacon_root': encode_hex(store.optimistic_header.beacon.hash_tree_root()), + }, + } + + +def emit_force_update(test, spec, state): + current_slot = state.slot + test.s_spec.process_light_client_store_force_update(test.store, current_slot) + + yield from [] # Consistently enable `yield from` syntax in calling tests + test.steps.append({ + "force_update": { + "current_slot": int(current_slot), + "checks": _get_checks(test.s_spec, test.store), + } + }) + + +def emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, with_next=True, phases=None): + data_fork_version = spec.compute_fork_version(spec.compute_epoch_at_slot(attested_block.message.slot)) + data_fork_digest = spec.compute_fork_digest(data_fork_version, test.genesis_validators_root) + d_spec = get_spec_for_fork_version(spec, data_fork_version, phases) + data = d_spec.create_light_client_update(state, block, attested_state, attested_block, finalized_block) + if not with_next: + data.next_sync_committee = spec.SyncCommittee() + data.next_sync_committee_branch = spec.NextSyncCommitteeBranch() + current_slot = state.slot + + upgraded = upgrade_lc_update_to_new_spec(d_spec, test.s_spec, data, phases) + test.s_spec.process_light_client_update(test.store, upgraded, current_slot, test.genesis_validators_root) + + yield _get_update_file_name(d_spec, data), data + test.steps.append({ + "process_update": { + "update_fork_digest": encode_hex(data_fork_digest), + "update": _get_update_file_name(d_spec, data), + "current_slot": int(current_slot), + "checks": _get_checks(test.s_spec, test.store), + } + }) + return upgraded + + +def _emit_upgrade_store(test, new_s_spec, phases=None): + test.store = upgrade_lc_store_to_new_spec(test.s_spec, new_s_spec, test.store, phases) + test.s_spec = new_s_spec + store_fork_version = _get_store_fork_version(test.s_spec) + store_fork_digest = test.s_spec.compute_fork_digest(store_fork_version, test.genesis_validators_root) + + yield from [] # Consistently enable `yield from` syntax in calling tests + test.steps.append({ + "upgrade_store": { + "store_fork_digest": encode_hex(store_fork_digest), + "checks": _get_checks(test.s_spec, test.store), + } + }) + + +def run_lc_sync_test_single_fork(spec, phases, state, fork): + # Start test + test = yield from setup_lc_sync_test(spec, state, phases=phases) + + # Initial `LightClientUpdate` + finalized_block = spec.SignedBeaconBlock() + finalized_block.message.state_root = state.hash_tree_root() + finalized_state = state.copy() + attested_block = state_transition_with_full_block(spec, state, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Jump to two slots before fork + fork_epoch = getattr(phases[fork].config, fork.upper() + '_FORK_EPOCH') + transition_to(spec, state, spec.compute_start_slot_at_epoch(fork_epoch) - 4) + attested_block = state_transition_with_full_block(spec, state, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + update = yield from emit_update( + test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Perform `LightClientStore` upgrade + yield from _emit_upgrade_store(test, phases[fork], phases=phases) + update = test.store.best_valid_update + + # Final slot before fork, check that importing the pre-fork format still works + attested_block = block.copy() + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Upgrade to post-fork spec, attested block is still before the fork + attested_block = block.copy() + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + state, block = do_fork(state, spec, phases[fork], fork_epoch, sync_aggregate=sync_aggregate) + spec = phases[fork] + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Another block after the fork, this time attested block is after the fork + attested_block = block.copy() + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Jump to next epoch + transition_to(spec, state, spec.compute_start_slot_at_epoch(fork_epoch + 1) - 2) + attested_block = state_transition_with_full_block(spec, state, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update == update + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Finalize the fork + finalized_block = block.copy() + finalized_state = state.copy() + _, _, state = next_slots_with_attestations(spec, state, 2 * spec.SLOTS_PER_EPOCH - 1, True, True) + attested_block = state_transition_with_full_block(spec, state, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Finish test + yield from finish_lc_sync_test(test) + + +def run_lc_sync_test_multi_fork(spec, phases, state, fork_1, fork_2): + # Start test + test = yield from setup_lc_sync_test(spec, state, phases[fork_2], phases) + + # Set up so that finalized is from `spec`, ... + finalized_block = spec.SignedBeaconBlock() + finalized_block.message.state_root = state.hash_tree_root() + finalized_state = state.copy() + + # ..., attested is from `fork_1`, ... + fork_1_epoch = getattr(phases[fork_1].config, fork_1.upper() + '_FORK_EPOCH') + spec, state, attested_block = transition_across_forks( + spec, + state, + spec.compute_start_slot_at_epoch(fork_1_epoch), + phases, + with_block=True, + ) + attested_state = state.copy() + + # ..., and signature is from `fork_2` + fork_2_epoch = getattr(phases[fork_2].config, fork_2.upper() + '_FORK_EPOCH') + spec, state, _ = transition_across_forks( + spec, state, spec.compute_start_slot_at_epoch(fork_2_epoch) - 1, phases) + sync_aggregate, _ = get_sync_aggregate(spec, state, phases=phases) + spec, state, block = transition_across_forks( + spec, + state, + spec.compute_start_slot_at_epoch(fork_2_epoch), + phases, + with_block=True, + sync_aggregate=sync_aggregate, + ) + + # Check that update applies + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Finish test + yield from finish_lc_sync_test(test) + + +def run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, fork): + # Start test (Legacy bootstrap with an upgraded store) + test = yield from setup_lc_sync_test(spec, state, phases[fork], phases) + + # Initial `LightClientUpdate` (check that the upgraded store can process it) + finalized_block = spec.SignedBeaconBlock() + finalized_block.message.state_root = state.hash_tree_root() + finalized_state = state.copy() + attested_block = state_transition_with_full_block(spec, state, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Finish test + yield from finish_lc_sync_test(test) From 75c65e63bf1636011166fb65db50fc3a1830bcb4 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Nov 2024 14:25:20 +0100 Subject: [PATCH 13/20] Split LC data collection test into multiple files --- .../light_client/test_data_collection.py | 1047 +---------------- .../light_client/test_data_collection.py | 40 + .../light_client/test_data_collection.py | 41 + .../helpers/light_client_data_collection.py | 897 ++++++++++++++ 4 files changed, 1032 insertions(+), 993 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py create mode 100644 tests/core/pyspec/eth2spec/test/electra/light_client/test_data_collection.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py index 57a7183077..af73b26345 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_data_collection.py @@ -1,799 +1,36 @@ -from typing import (Any, Dict, List, Set) -from dataclasses import dataclass - -from eth_utils import encode_hex from eth2spec.test.context import ( spec_state_test_with_matching_config, - spec_test, - with_config_overrides, - with_matching_spec_config, - with_phases, with_presets, - with_state, with_light_client, ) from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA, MINIMAL, ) -from eth2spec.test.helpers.fork_transition import ( - transition_across_forks, -) -from eth2spec.test.helpers.forks import ( - is_post_altair, -) -from eth2spec.test.helpers.light_client import ( - compute_start_slot_at_sync_committee_period, - get_sync_aggregate, - latest_current_sync_committee_gindex, - latest_finalized_root_gindex, - latest_next_sync_committee_gindex, - latest_normalize_merkle_branch, - upgrade_lc_header_to_new_spec, - upgrade_lc_update_to_new_spec, +from eth2spec.test.helpers.light_client_data_collection import ( + add_new_block, + finish_lc_data_collection_test, + get_lc_bootstrap_block_id, + get_lc_update_attested_block_id, + get_light_client_bootstrap, + get_light_client_finality_update, + get_light_client_optimistic_update, + get_light_client_update_for_period, + select_new_head, + setup_lc_data_collection_test, + BlockID, ) -def next_epoch_boundary_slot(spec, slot): - # Compute the first possible epoch boundary state slot of a `Checkpoint` - # referring to a block at given slot. - epoch = spec.compute_epoch_at_slot(slot + spec.SLOTS_PER_EPOCH - 1) - return spec.compute_start_slot_at_epoch(epoch) - - -@dataclass(frozen=True) -class BlockID(object): - slot: Any - root: Any - - -def block_to_block_id(block): - return BlockID( - slot=block.message.slot, - root=block.message.hash_tree_root(), - ) - - -def state_to_block_id(state): - parent_header = state.latest_block_header.copy() - parent_header.state_root = state.hash_tree_root() - return BlockID(slot=parent_header.slot, root=parent_header.hash_tree_root()) - - -def bootstrap_bid(bootstrap): - return BlockID( - slot=bootstrap.header.beacon.slot, - root=bootstrap.header.beacon.hash_tree_root(), - ) - - -def update_attested_bid(update): - return BlockID( - slot=update.attested_header.beacon.slot, - root=update.attested_header.beacon.hash_tree_root(), - ) - - -@dataclass -class ForkedBeaconState(object): - spec: Any - data: Any - - -@dataclass -class ForkedSignedBeaconBlock(object): - spec: Any - data: Any - - -@dataclass -class ForkedLightClientHeader(object): - spec: Any - data: Any - - -@dataclass -class ForkedLightClientBootstrap(object): - spec: Any - data: Any - - -@dataclass -class ForkedLightClientUpdate(object): - spec: Any - data: Any - - -@dataclass -class ForkedLightClientFinalityUpdate(object): - spec: Any - data: Any - - -@dataclass -class ForkedLightClientOptimisticUpdate(object): - spec: Any - data: Any - - -@dataclass -class CachedLightClientData(object): - # Sync committee branches at block's post-state - current_sync_committee_branch: Any # CurrentSyncCommitteeBranch - next_sync_committee_branch: Any # NextSyncCommitteeBranch - - # Finality information at block's post-state - finalized_slot: Any # Slot - finality_branch: Any # FinalityBranch - - # Best / latest light client data - current_period_best_update: ForkedLightClientUpdate - latest_signature_slot: Any # Slot - - -@dataclass -class LightClientDataCache(object): - # Cached data for creating future `LightClientUpdate` instances. - # Key is the block ID of which the post state was used to get the data. - # Data stored for the finalized head block and all non-finalized blocks. - data: Dict[BlockID, CachedLightClientData] - - # Light client data for the latest slot that was signed by at least - # `MIN_SYNC_COMMITTEE_PARTICIPANTS`. May be older than head - latest: ForkedLightClientFinalityUpdate - - # The earliest slot for which light client data is imported - tail_slot: Any # Slot - - -@dataclass -class LightClientDataDB(object): - headers: Dict[Any, ForkedLightClientHeader] # Root -> ForkedLightClientHeader - current_branches: Dict[Any, Any] # Slot -> CurrentSyncCommitteeBranch - sync_committees: Dict[Any, Any] # SyncCommitteePeriod -> SyncCommittee - best_updates: Dict[Any, ForkedLightClientUpdate] # SyncCommitteePeriod -> ForkedLightClientUpdate - - -@dataclass -class LightClientDataStore(object): - spec: Any - - # Cached data to accelerate creating light client data - cache: LightClientDataCache - - # Persistent light client data - db: LightClientDataDB - - -@dataclass -class LightClientDataCollectionTest(object): - steps: List[Dict[str, Any]] - files: Set[str] - - # Fork schedule - phases: Any - - # History access - blocks: Dict[Any, ForkedSignedBeaconBlock] # Block root -> ForkedSignedBeaconBlock - finalized_block_roots: Dict[Any, Any] # Slot -> Root - states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState - finalized_checkpoint_states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState - latest_finalized_epoch: Any # Epoch - latest_finalized_bid: BlockID - historical_tail_slot: Any # Slot - - # Light client data - lc_data_store: LightClientDataStore - - -def get_ancestor_of_block_id(test, bid, slot): # -> Optional[BlockID] - try: - block = test.blocks[bid.root] - while True: - if block.data.message.slot <= slot: - return block_to_block_id(block.data) - - block = test.blocks[block.data.message.parent_root] - except KeyError: - return None - - -def block_id_at_finalized_slot(test, slot): # -> Optional[BlockID] - while slot >= test.historical_tail_slot: - try: - return BlockID(slot=slot, root=test.finalized_block_roots[slot]) - except KeyError: - slot = slot - 1 - return None - - -def get_current_sync_committee_for_finalized_period(test, period): # -> Optional[SyncCommittee] - low_slot = max( - test.historical_tail_slot, - test.lc_data_store.spec.compute_start_slot_at_epoch( - test.lc_data_store.spec.config.ALTAIR_FORK_EPOCH) - ) - if period < test.lc_data_store.spec.compute_sync_committee_period_at_slot(low_slot): - return None - period_start_slot = compute_start_slot_at_sync_committee_period(test.lc_data_store.spec, period) - sync_committee_slot = max(period_start_slot, low_slot) - bid = block_id_at_finalized_slot(test, sync_committee_slot) - if bid is None: - return None - block = test.blocks[bid.root] - state = test.finalized_checkpoint_states[block.data.message.state_root] - if sync_committee_slot > state.data.slot: - state.spec, state.data, _ = transition_across_forks( - state.spec, state.data, sync_committee_slot, phases=test.phases) - assert is_post_altair(state.spec) - return state.data.current_sync_committee - - -def light_client_header_for_block(test, block): # -> ForkedLightClientHeader - if not is_post_altair(block.spec): - spec = test.phases[ALTAIR] - else: - spec = block.spec - return ForkedLightClientHeader(spec=spec, data=spec.block_to_light_client_header(block.data)) - - -def light_client_header_for_block_id(test, bid): # -> ForkedLightClientHeader - block = test.blocks[bid.root] - if not is_post_altair(block.spec): - spec = test.phases[ALTAIR] - else: - spec = block.spec - return ForkedLightClientHeader(spec=spec, data=spec.block_to_light_client_header(block.data)) - - -def sync_aggregate_for_block_id(test, bid): # -> Optional[SyncAggregate] - block = test.blocks[bid.root] - if not is_post_altair(block.spec): - return None - return block.data.message.body.sync_aggregate - - -def get_light_client_data(lc_data_store, bid): # -> CachedLightClientData - # Fetch cached light client data about a given block. - # Data must be cached (`cache_light_client_data`) before calling this function. - try: - return lc_data_store.cache.data[bid] - except KeyError: - raise ValueError("Trying to get light client data that was not cached") - - -def cache_light_client_data(lc_data_store, spec, state, bid, current_period_best_update, latest_signature_slot): - # Cache data for a given block and its post-state to speed up creating future - # `LightClientUpdate` and `LightClientBootstrap` instances that refer to this - # block and state. - cached_data = CachedLightClientData( - current_sync_committee_branch=latest_normalize_merkle_branch( - lc_data_store.spec, - spec.compute_merkle_proof(state, spec.current_sync_committee_gindex_at_slot(state.slot)), - latest_current_sync_committee_gindex(lc_data_store.spec)), - next_sync_committee_branch=latest_normalize_merkle_branch( - lc_data_store.spec, - spec.compute_merkle_proof(state, spec.next_sync_committee_gindex_at_slot(state.slot)), - latest_next_sync_committee_gindex(lc_data_store.spec)), - finalized_slot=spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch), - finality_branch=latest_normalize_merkle_branch( - lc_data_store.spec, - spec.compute_merkle_proof(state, spec.finalized_root_gindex_at_slot(state.slot)), - latest_finalized_root_gindex(lc_data_store.spec)), - current_period_best_update=current_period_best_update, - latest_signature_slot=latest_signature_slot, - ) - if bid in lc_data_store.cache.data: - raise ValueError("Redundant `cache_light_client_data` call") - lc_data_store.cache.data[bid] = cached_data - - -def delete_light_client_data(lc_data_store, bid): - # Delete cached light client data for a given block. This needs to be called - # when a block becomes unreachable due to finalization of a different fork. - del lc_data_store.cache.data[bid] - - -def create_light_client_finality_update_from_light_client_data(test, - attested_bid, - signature_slot, - sync_aggregate): # -> ForkedLightClientFinalityUpdate - attested_header = light_client_header_for_block_id(test, attested_bid) - attested_data = get_light_client_data(test.lc_data_store, attested_bid) - finalized_bid = block_id_at_finalized_slot(test, attested_data.finalized_slot) - if finalized_bid is not None: - if finalized_bid.slot != attested_data.finalized_slot: - # Empty slots at end of epoch, update cache for latest block slot - attested_data.finalized_slot = finalized_bid.slot - if finalized_bid.slot == attested_header.spec.GENESIS_SLOT: - finalized_header = ForkedLightClientHeader( - spec=attested_header.spec, - data=attested_header.spec.LightClientHeader(), - ) - else: - finalized_header = light_client_header_for_block_id(test, finalized_bid) - finalized_header = ForkedLightClientHeader( - spec=attested_header.spec, - data=upgrade_lc_header_to_new_spec( - finalized_header.spec, - attested_header.spec, - finalized_header.data, - ) - ) - finality_branch = attested_data.finality_branch - return ForkedLightClientFinalityUpdate( - spec=attested_header.spec, - data=attested_header.spec.LightClientFinalityUpdate( - attested_header=attested_header.data, - finalized_header=finalized_header.data, - finality_branch=finality_branch, - sync_aggregate=sync_aggregate, - signature_slot=signature_slot, - ), - ) - - -def create_light_client_update_from_light_client_data(test, - attested_bid, - signature_slot, - sync_aggregate, - next_sync_committee): # -> ForkedLightClientUpdate - finality_update = create_light_client_finality_update_from_light_client_data( - test, attested_bid, signature_slot, sync_aggregate) - attested_data = get_light_client_data(test.lc_data_store, attested_bid) - return ForkedLightClientUpdate( - spec=finality_update.spec, - data=finality_update.spec.LightClientUpdate( - attested_header=finality_update.data.attested_header, - next_sync_committee=next_sync_committee, - next_sync_committee_branch=attested_data.next_sync_committee_branch, - finalized_header=finality_update.data.finalized_header, - finality_branch=finality_update.data.finality_branch, - sync_aggregate=finality_update.data.sync_aggregate, - signature_slot=finality_update.data.signature_slot, - ) - ) - - -def create_light_client_update(test, spec, state, block, parent_bid): - # Create `LightClientUpdate` instances for a given block and its post-state, - # and keep track of best / latest ones. Data about the parent block's - # post-state must be cached (`cache_light_client_data`) before calling this. - - # Verify attested block (parent) is recent enough and that state is available - attested_bid = parent_bid - attested_slot = attested_bid.slot - if attested_slot < test.lc_data_store.cache.tail_slot: - cache_light_client_data( - test.lc_data_store, - spec, - state, - block_to_block_id(block), - current_period_best_update=ForkedLightClientUpdate(spec=None, data=None), - latest_signature_slot=spec.GENESIS_SLOT, - ) - return - - # If sync committee period changed, reset `best` - attested_period = spec.compute_sync_committee_period_at_slot(attested_slot) - signature_slot = block.message.slot - signature_period = spec.compute_sync_committee_period_at_slot(signature_slot) - attested_data = get_light_client_data(test.lc_data_store, attested_bid) - if attested_period != signature_period: - best = ForkedLightClientUpdate(spec=None, data=None) - else: - best = attested_data.current_period_best_update - - # If sync committee does not have sufficient participants, do not bump latest - sync_aggregate = block.message.body.sync_aggregate - num_active_participants = sum(sync_aggregate.sync_committee_bits) - if num_active_participants < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS: - latest_signature_slot = attested_data.latest_signature_slot - else: - latest_signature_slot = signature_slot - - # To update `best`, sync committee must have sufficient participants, and - # `signature_slot` must be in `attested_slot`'s sync committee period - if ( - num_active_participants < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS - or attested_period != signature_period - ): - cache_light_client_data( - test.lc_data_store, - spec, - state, - block_to_block_id(block), - current_period_best_update=best, - latest_signature_slot=latest_signature_slot, - ) - return - - # Check if light client data improved - update = create_light_client_update_from_light_client_data( - test, attested_bid, signature_slot, sync_aggregate, state.next_sync_committee) - is_better = ( - best.spec is None - or spec.is_better_update(update.data, upgrade_lc_update_to_new_spec( - best.spec, update.spec, best.data, test.phases)) - ) - - # Update best light client data for current sync committee period - if is_better: - best = update - cache_light_client_data( - test.lc_data_store, - spec, - state, - block_to_block_id(block), - current_period_best_update=best, - latest_signature_slot=latest_signature_slot, - ) - - -def create_light_client_bootstrap(test, spec, bid): - block = test.blocks[bid.root] - period = spec.compute_sync_committee_period_at_slot(bid.slot) - if period not in test.lc_data_store.db.sync_committees: - test.lc_data_store.db.sync_committees[period] = \ - get_current_sync_committee_for_finalized_period(test, period) - test.lc_data_store.db.headers[bid.root] = ForkedLightClientHeader( - spec=block.spec, data=block.spec.block_to_light_client_header(block.data)) - test.lc_data_store.db.current_branches[bid.slot] = \ - get_light_client_data(test.lc_data_store, bid).current_sync_committee_branch - - -def process_new_block_for_light_client(test, spec, state, block, parent_bid): - # Update light client data with information from a new block. - if block.message.slot < test.lc_data_store.cache.tail_slot: - return - - if is_post_altair(spec): - create_light_client_update(test, spec, state, block, parent_bid) - else: - raise ValueError("`tail_slot` cannot be before Altair") - - -def process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid): - # Update light client data to account for a new head block. - # Note that `old_finalized_bid` is not yet updated when this is called. - if head_bid.slot < test.lc_data_store.cache.tail_slot: - return - - # Commit best light client data for non-finalized periods - head_period = spec.compute_sync_committee_period_at_slot(head_bid.slot) - low_slot = max(test.lc_data_store.cache.tail_slot, old_finalized_bid.slot) - low_period = spec.compute_sync_committee_period_at_slot(low_slot) - bid = head_bid - for period in reversed(range(low_period, head_period + 1)): - period_end_slot = compute_start_slot_at_sync_committee_period(spec, period + 1) - 1 - bid = get_ancestor_of_block_id(test, bid, period_end_slot) - if bid is None or bid.slot < low_slot: - break - best = get_light_client_data(test.lc_data_store, bid).current_period_best_update - if ( - best.spec is None - or sum(best.data.sync_aggregate.sync_committee_bits) < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS - ): - test.lc_data_store.db.best_updates.pop(period, None) - else: - test.lc_data_store.db.best_updates[period] = best - - # Update latest light client data - head_data = get_light_client_data(test.lc_data_store, head_bid) - signature_slot = head_data.latest_signature_slot - if signature_slot <= low_slot: - test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) - return - signature_bid = get_ancestor_of_block_id(test, head_bid, signature_slot) - if signature_bid is None or signature_bid.slot <= low_slot: - test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) - return - attested_bid = get_ancestor_of_block_id(test, signature_bid, signature_bid.slot - 1) - if attested_bid is None or attested_bid.slot < low_slot: - test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) - return - sync_aggregate = sync_aggregate_for_block_id(test, signature_bid) - assert sync_aggregate is not None - test.lc_data_store.cache.latest = create_light_client_finality_update_from_light_client_data( - test, attested_bid, signature_slot, sync_aggregate) - - -def process_finalization_for_light_client(test, spec, finalized_bid, old_finalized_bid): - # Prune cached data that is no longer useful for creating future - # `LightClientUpdate` and `LightClientBootstrap` instances. - # This needs to be called whenever `finalized_checkpoint` changes. - finalized_slot = finalized_bid.slot - if finalized_slot < test.lc_data_store.cache.tail_slot: - return - - # Cache `LightClientBootstrap` for newly finalized epoch boundary blocks - first_new_slot = old_finalized_bid.slot + 1 - low_slot = max(first_new_slot, test.lc_data_store.cache.tail_slot) - boundary_slot = finalized_slot - while boundary_slot >= low_slot: - bid = block_id_at_finalized_slot(test, boundary_slot) - if bid is None: - break - if bid.slot >= low_slot: - create_light_client_bootstrap(test, spec, bid) - boundary_slot = next_epoch_boundary_slot(spec, bid.slot) - if boundary_slot < spec.SLOTS_PER_EPOCH: - break - boundary_slot = boundary_slot - spec.SLOTS_PER_EPOCH - - # Prune light client data that is no longer referrable by future updates - bids_to_delete = [] - for bid in test.lc_data_store.cache.data: - if bid.slot >= finalized_bid.slot: - continue - bids_to_delete.append(bid) - for bid in bids_to_delete: - delete_light_client_data(test.lc_data_store, bid) - - -def get_light_client_bootstrap(test, block_root): # -> ForkedLightClientBootstrap - try: - header = test.lc_data_store.db.headers[block_root] - except KeyError: - return ForkedLightClientBootstrap(spec=None, data=None) - - slot = header.data.beacon.slot - period = header.spec.compute_sync_committee_period_at_slot(slot) - return ForkedLightClientBootstrap( - spec=header.spec, - data=header.spec.LightClientBootstrap( - header=header.data, - current_sync_committee=test.lc_data_store.db.sync_committees[period], - current_sync_committee_branch=test.lc_data_store.db.current_branches[slot], - ) - ) - - -def get_light_client_update_for_period(test, period): # -> ForkedLightClientUpdate - try: - return test.lc_data_store.db.best_updates[period] - except KeyError: - return ForkedLightClientUpdate(spec=None, data=None) - - -def get_light_client_finality_update(test): # -> ForkedLightClientFinalityUpdate - return test.lc_data_store.cache.latest - - -def get_light_client_optimistic_update(test): # -> ForkedLightClientOptimisticUpdate - finality_update = get_light_client_finality_update(test) - if finality_update.spec is None: - return ForkedLightClientOptimisticUpdate(spec=None, data=None) - return ForkedLightClientOptimisticUpdate( - spec=finality_update.spec, - data=finality_update.spec.LightClientOptimisticUpdate( - attested_header=finality_update.data.attested_header, - sync_aggregate=finality_update.data.sync_aggregate, - signature_slot=finality_update.data.signature_slot, - ), - ) - - -def setup_test(spec, state, phases=None): - assert spec.compute_slots_since_epoch_start(state.slot) == 0 - - test = LightClientDataCollectionTest( - steps=[], - files=set(), - phases=phases, - blocks={}, - finalized_block_roots={}, - states={}, - finalized_checkpoint_states={}, - latest_finalized_epoch=state.finalized_checkpoint.epoch, - latest_finalized_bid=BlockID( - slot=spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch), - root=state.finalized_checkpoint.root, - ), - historical_tail_slot=state.slot, - lc_data_store=LightClientDataStore( - spec=spec, - cache=LightClientDataCache( - data={}, - latest=ForkedLightClientFinalityUpdate(spec=None, data=None), - tail_slot=max(state.slot, spec.compute_start_slot_at_epoch(spec.config.ALTAIR_FORK_EPOCH)), - ), - db=LightClientDataDB( - headers={}, - current_branches={}, - sync_committees={}, - best_updates={}, - ), - ), - ) - bid = state_to_block_id(state) - yield "initial_state", state - test.blocks[bid.root] = ForkedSignedBeaconBlock(spec=spec, data=spec.SignedBeaconBlock( - message=spec.BeaconBlock(state_root=state.hash_tree_root()), - )) - test.finalized_block_roots[bid.slot] = bid.root - test.states[state.hash_tree_root()] = ForkedBeaconState(spec=spec, data=state) - test.finalized_checkpoint_states[state.hash_tree_root()] = ForkedBeaconState(spec=spec, data=state) - cache_light_client_data( - test.lc_data_store, spec, state, bid, - current_period_best_update=ForkedLightClientUpdate(spec=None, data=None), - latest_signature_slot=spec.GENESIS_SLOT, - ) - create_light_client_bootstrap(test, spec, bid) - - return test - - -def finish_test(test): - yield "steps", test.steps - - -def encode_object(test, prefix, obj, slot, genesis_validators_root): - yield from [] # Consistently enable `yield from` syntax in calling tests - - file_name = f"{prefix}_{slot}_{encode_hex(obj.data.hash_tree_root())}" - if file_name not in test.files: - test.files.add(file_name) - yield file_name, obj.data - return { - "fork_digest": encode_hex(obj.spec.compute_fork_digest( - obj.spec.compute_fork_version(obj.spec.compute_epoch_at_slot(slot)), - genesis_validators_root, - )), - "data": file_name, - } - - -def add_new_block(test, spec, state, slot=None, num_sync_participants=0): - if slot is None: - slot = state.slot + 1 - assert slot > state.slot - parent_bid = state_to_block_id(state) - - # Advance to target slot - 1 to ensure sync aggregate can be efficiently computed - if state.slot < slot - 1: - spec, state, _ = transition_across_forks(spec, state, slot - 1, phases=test.phases) - - # Compute sync aggregate, using: - # - sync committee based on target slot - # - fork digest based on target slot - 1 - # - signed data based on parent_bid.slot - # All three slots may be from different forks - sync_aggregate, signature_slot = get_sync_aggregate( - spec, state, num_participants=num_sync_participants, phases=test.phases) - assert signature_slot == slot - - # Apply final block with computed sync aggregate - spec, state, block = transition_across_forks( - spec, state, slot, phases=test.phases, with_block=True, sync_aggregate=sync_aggregate) - bid = block_to_block_id(block) - test.blocks[bid.root] = ForkedSignedBeaconBlock(spec=spec, data=block) - test.states[block.message.state_root] = ForkedBeaconState(spec=spec, data=state) - process_new_block_for_light_client(test, spec, state, block, parent_bid) - block_obj = yield from encode_object( - test, "block", ForkedSignedBeaconBlock(spec=spec, data=block), block.message.slot, - state.genesis_validators_root, - ) - test.steps.append({ - "new_block": block_obj - }) - return spec, state, bid - - -def select_new_head(test, spec, head_bid): - old_finalized_bid = test.latest_finalized_bid - process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid) - - # Process finalization - block = test.blocks[head_bid.root] - state = test.states[block.data.message.state_root] - if state.data.finalized_checkpoint.epoch != spec.GENESIS_EPOCH: - block = test.blocks[state.data.finalized_checkpoint.root] - bid = block_to_block_id(block.data) - new_finalized_bid = bid - if new_finalized_bid.slot > old_finalized_bid.slot: - old_finalized_epoch = None - new_finalized_epoch = state.data.finalized_checkpoint.epoch - while bid.slot > test.latest_finalized_bid.slot: - test.finalized_block_roots[bid.slot] = bid.root - finalized_epoch = spec.compute_epoch_at_slot(bid.slot + spec.SLOTS_PER_EPOCH - 1) - if finalized_epoch != old_finalized_epoch: - state = test.states[block.data.message.state_root] - test.finalized_checkpoint_states[block.data.message.state_root] = state - old_finalized_epoch = finalized_epoch - block = test.blocks[block.data.message.parent_root] - bid = block_to_block_id(block.data) - test.latest_finalized_epoch = new_finalized_epoch - test.latest_finalized_bid = new_finalized_bid - process_finalization_for_light_client(test, spec, new_finalized_bid, old_finalized_bid) - - blocks_to_delete = [] - for block_root, block in test.blocks.items(): - if block.data.message.slot < new_finalized_bid.slot: - blocks_to_delete.append(block_root) - for block_root in blocks_to_delete: - del test.blocks[block_root] - states_to_delete = [] - for state_root, state in test.states.items(): - if state.data.slot < new_finalized_bid.slot: - states_to_delete.append(state_root) - for state_root in states_to_delete: - del test.states[state_root] - - yield from [] # Consistently enable `yield from` syntax in calling tests - - bootstraps = [] - for state in test.finalized_checkpoint_states.values(): - bid = state_to_block_id(state.data) - entry = { - "block_root": encode_hex(bid.root), - } - bootstrap = get_light_client_bootstrap(test, bid.root) - if bootstrap.spec is not None: - bootstrap_obj = yield from encode_object( - test, "bootstrap", bootstrap, bootstrap.data.header.beacon.slot, - state.data.genesis_validators_root, - ) - entry["bootstrap"] = bootstrap_obj - bootstraps.append(entry) - - best_updates = [] - low_period = spec.compute_sync_committee_period_at_slot(test.lc_data_store.cache.tail_slot) - head_period = spec.compute_sync_committee_period_at_slot(head_bid.slot) - for period in range(low_period, head_period + 1): - entry = { - "period": int(period), - } - update = get_light_client_update_for_period(test, period) - if update.spec is not None: - update_obj = yield from encode_object( - test, "update", update, update.data.attested_header.beacon.slot, - state.data.genesis_validators_root, - ) - entry["update"] = update_obj - best_updates.append(entry) - - checks = { - "latest_finalized_checkpoint": { - "epoch": int(test.latest_finalized_epoch), - "root": encode_hex(test.latest_finalized_bid.root), - }, - "bootstraps": bootstraps, - "best_updates": best_updates, - } - finality_update = get_light_client_finality_update(test) - if finality_update.spec is not None: - finality_update_obj = yield from encode_object( - test, "finality_update", finality_update, finality_update.data.attested_header.beacon.slot, - state.data.genesis_validators_root, - ) - checks["latest_finality_update"] = finality_update_obj - optimistic_update = get_light_client_optimistic_update(test) - if optimistic_update.spec is not None: - optimistic_update_obj = yield from encode_object( - test, "optimistic_update", optimistic_update, optimistic_update.data.attested_header.beacon.slot, - state.data.genesis_validators_root, - ) - checks["latest_optimistic_update"] = optimistic_update_obj - - test.steps.append({ - "new_head": { - "head_block_root": encode_hex(head_bid.root), - "checks": checks, - } - }) - - @with_light_client @spec_state_test_with_matching_config @with_presets([MINIMAL], reason="too slow") def test_light_client_data_collection(spec, state): # Start test - test = yield from setup_test(spec, state) + test = yield from setup_lc_data_collection_test(spec, state) # Genesis block is post Altair and is finalized, so can be used as bootstrap genesis_bid = BlockID(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) - assert bootstrap_bid(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid + assert get_lc_bootstrap_block_id(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid # No blocks have been imported, so no other light client data is available period = spec.compute_sync_committee_period_at_slot(state.slot) @@ -813,9 +50,9 @@ def test_light_client_data_collection(spec, state): spec_b, state_b, bid_2 = yield from add_new_block(test, spec, state, slot=2, num_sync_participants=1) yield from select_new_head(test, spec_b, bid_2) period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == genesis_bid - assert update_attested_bid(get_light_client_finality_update(test).data) == genesis_bid - assert update_attested_bid(get_light_client_optimistic_update(test).data) == genesis_bid + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == genesis_bid + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == genesis_bid + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == genesis_bid # Build on branch A, once more with an empty sync aggregate spec_a, state_a, bid_3 = yield from add_new_block(test, spec_a, state_a, slot=3) @@ -829,33 +66,33 @@ def test_light_client_data_collection(spec, state): spec_b, state_b, bid_4 = yield from add_new_block(test, spec_b, state_b, slot=4) yield from select_new_head(test, spec_b, bid_4) period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == genesis_bid - assert update_attested_bid(get_light_client_finality_update(test).data) == genesis_bid - assert update_attested_bid(get_light_client_optimistic_update(test).data) == genesis_bid + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == genesis_bid + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == genesis_bid + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == genesis_bid # Build on branch B, once more with 1 participant spec_b, state_b, bid_5 = yield from add_new_block(test, spec_b, state_b, slot=5, num_sync_participants=1) yield from select_new_head(test, spec_b, bid_5) period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == genesis_bid - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_4 - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_4 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == genesis_bid + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_4 + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_4 # Build on branch B, this time with 3 participants spec_b, state_b, bid_6 = yield from add_new_block(test, spec_b, state_b, slot=6, num_sync_participants=3) yield from select_new_head(test, spec_b, bid_6) period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_5 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_5 - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_5 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_5 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_5 + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_5 # Build on branch A, with 2 participants spec_a, state_a, bid_7 = yield from add_new_block(test, spec_a, state_a, slot=7, num_sync_participants=2) yield from select_new_head(test, spec_a, bid_7) period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_3 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_3 - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_3 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_3 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_3 + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_3 # Branch A: epoch 1, slot 5 slot = spec_a.compute_start_slot_at_epoch(1) + 5 @@ -864,9 +101,9 @@ def test_light_client_data_collection(spec, state): assert get_light_client_bootstrap(test, bid_7.root).spec is None assert get_light_client_bootstrap(test, bid_1_5.root).spec is None period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_7 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_7 - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_7 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_7 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_7 + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_7 # Branch B: epoch 2, slot 4 slot = spec_b.compute_start_slot_at_epoch(2) + 4 @@ -876,9 +113,9 @@ def test_light_client_data_collection(spec, state): assert get_light_client_bootstrap(test, bid_1_5.root).spec is None assert get_light_client_bootstrap(test, bid_2_4.root).spec is None period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_6 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_6 - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_6 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_6 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_6 + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_6 # Branch A: epoch 3, slot 0 slot = spec_a.compute_start_slot_at_epoch(3) + 0 @@ -889,9 +126,9 @@ def test_light_client_data_collection(spec, state): assert get_light_client_bootstrap(test, bid_2_4.root).spec is None assert get_light_client_bootstrap(test, bid_3_0.root).spec is None period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_1_5 - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_1_5 # Branch A: fill epoch for i in range(1, spec_a.SLOTS_PER_EPOCH): @@ -902,9 +139,9 @@ def test_light_client_data_collection(spec, state): assert get_light_client_bootstrap(test, bid_2_4.root).spec is None assert get_light_client_bootstrap(test, bid_3_0.root).spec is None period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_1_5 - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_1_5 assert state_a.slot == spec_a.compute_start_slot_at_epoch(4) - 1 bid_3_n = bid_a @@ -918,9 +155,9 @@ def test_light_client_data_collection(spec, state): assert get_light_client_bootstrap(test, bid_3_0.root).spec is None assert get_light_client_bootstrap(test, bid_4_0.root).spec is None period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_3_n - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_3_n + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_3_n + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_3_n # Branch A: fill epoch for i in range(1, spec_a.SLOTS_PER_EPOCH): @@ -932,9 +169,9 @@ def test_light_client_data_collection(spec, state): assert get_light_client_bootstrap(test, bid_3_0.root).spec is None assert get_light_client_bootstrap(test, bid_4_0.root).spec is None period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_3_n - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_3_n + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_3_n + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_3_n assert state_a.slot == spec_a.compute_start_slot_at_epoch(5) - 1 bid_4_n = bid_a @@ -942,191 +179,15 @@ def test_light_client_data_collection(spec, state): slot = spec_a.compute_start_slot_at_epoch(6) + 2 spec_a, state_a, bid_6_2 = yield from add_new_block(test, spec_a, state_a, slot=slot, num_sync_participants=6) yield from select_new_head(test, spec_a, bid_6_2) - assert bootstrap_bid(get_light_client_bootstrap(test, bid_7.root).data) == bid_7 - assert bootstrap_bid(get_light_client_bootstrap(test, bid_1_5.root).data) == bid_1_5 + assert get_lc_bootstrap_block_id(get_light_client_bootstrap(test, bid_7.root).data) == bid_7 + assert get_lc_bootstrap_block_id(get_light_client_bootstrap(test, bid_1_5.root).data) == bid_1_5 assert get_light_client_bootstrap(test, bid_2_4.root).spec is None - assert bootstrap_bid(get_light_client_bootstrap(test, bid_3_0.root).data) == bid_3_0 + assert get_lc_bootstrap_block_id(get_light_client_bootstrap(test, bid_3_0.root).data) == bid_3_0 assert get_light_client_bootstrap(test, bid_4_0.root).spec is None period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - assert update_attested_bid(get_light_client_update_for_period(test, period).data) == bid_1_5 - assert update_attested_bid(get_light_client_finality_update(test).data) == bid_4_n - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bid_4_n - - # Finish test - yield from finish_test(test) - - -def run_test_multi_fork(spec, phases, state, fork_1, fork_2): - # Start test - test = yield from setup_test(spec, state, phases=phases) - - # Genesis block is post Altair and is finalized, so can be used as bootstrap - genesis_bid = BlockID(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) - assert bootstrap_bid(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid - - # Shared history up to final epoch of period before `fork_1` - fork_1_epoch = getattr(phases[fork_1].config, fork_1.upper() + '_FORK_EPOCH') - fork_1_period = spec.compute_sync_committee_period(fork_1_epoch) - slot = compute_start_slot_at_sync_committee_period(spec, fork_1_period) - spec.SLOTS_PER_EPOCH - spec, state, bid = yield from add_new_block(test, spec, state, slot=slot, num_sync_participants=1) - yield from select_new_head(test, spec, bid) - assert get_light_client_bootstrap(test, bid.root).spec is None - slot_period = spec.compute_sync_committee_period_at_slot(slot) - if slot_period == 0: - assert update_attested_bid(get_light_client_update_for_period(test, 0).data) == genesis_bid - else: - for period in range(0, slot_period): - assert get_light_client_update_for_period(test, period).spec is None # attested period != signature period - state_period = spec.compute_sync_committee_period_at_slot(state.slot) - - # Branch A: Advance past `fork_2`, having blocks at slots 0 and 4 of each epoch - spec_a = spec - state_a = state - slot_a = state_a.slot - bids_a = [bid] - num_sync_participants_a = 1 - fork_2_epoch = getattr(phases[fork_2].config, fork_2.upper() + '_FORK_EPOCH') - while spec_a.get_current_epoch(state_a) <= fork_2_epoch: - attested_period = spec_a.compute_sync_committee_period_at_slot(slot_a) - slot_a += 4 - signature_period = spec_a.compute_sync_committee_period_at_slot(slot_a) - if signature_period != attested_period: - num_sync_participants_a = 0 - num_sync_participants_a += 1 - spec_a, state_a, bid_a = yield from add_new_block( - test, spec_a, state_a, slot=slot_a, num_sync_participants=num_sync_participants_a) - yield from select_new_head(test, spec_a, bid_a) - for bid in bids_a: - assert get_light_client_bootstrap(test, bid.root).spec is None - if attested_period == signature_period: - assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] - else: - assert signature_period == attested_period + 1 - assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] - assert get_light_client_update_for_period(test, signature_period).spec is None - assert update_attested_bid(get_light_client_finality_update(test).data) == bids_a[-1] - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bids_a[-1] - bids_a.append(bid_a) - - # Branch B: Advance past `fork_2`, having blocks at slots 1 and 5 of each epoch but no sync participation - spec_b = spec - state_b = state - slot_b = state_b.slot - bids_b = [bid] - while spec_b.get_current_epoch(state_b) <= fork_2_epoch: - slot_b += 4 - signature_period = spec_b.compute_sync_committee_period_at_slot(slot_b) - spec_b, state_b, bid_b = yield from add_new_block( - test, spec_b, state_b, slot=slot_b) - # Simulate that this does not become head yet, e.g., this branch was withheld - for bid in bids_b: - assert get_light_client_bootstrap(test, bid.root).spec is None - bids_b.append(bid_b) - - # Branch B: Another block that becomes head - attested_period = spec_b.compute_sync_committee_period_at_slot(slot_b) - slot_b += 1 - signature_period = spec_b.compute_sync_committee_period_at_slot(slot_b) - num_sync_participants_b = 1 - spec_b, state_b, bid_b = yield from add_new_block( - test, spec_b, state_b, slot=slot_b, num_sync_participants=num_sync_participants_b) - yield from select_new_head(test, spec_b, bid_b) - for bid in bids_b: - assert get_light_client_bootstrap(test, bid.root).spec is None - if attested_period == signature_period: - assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_b[-1] - else: - assert signature_period == attested_period + 1 - assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_b[-2] - assert get_light_client_update_for_period(test, signature_period).spec is None - assert update_attested_bid(get_light_client_finality_update(test).data) == bids_b[-1] - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bids_b[-1] - bids_b.append(bid_b) - - # All data for periods between the common ancestor of the two branches should have reorged. - # As there was no sync participation on branch B, that means it is deleted. - state_b_period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) - for period in range(state_period + 1, state_b_period): - assert get_light_client_update_for_period(test, period).spec is None - - # Branch A: Another block, reorging branch B once more - attested_period = spec_a.compute_sync_committee_period_at_slot(slot_a) - slot_a = slot_b + 1 - signature_period = spec_a.compute_sync_committee_period_at_slot(slot_a) - if signature_period != attested_period: - num_sync_participants_a = 0 - num_sync_participants_a += 1 - spec_a, state_a, bid_a = yield from add_new_block( - test, spec_a, state_a, slot=slot_a, num_sync_participants=num_sync_participants_a) - yield from select_new_head(test, spec_a, bid_a) - for bid in bids_a: - assert get_light_client_bootstrap(test, bid.root).spec is None - if attested_period == signature_period: - assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] - else: - assert signature_period == attested_period + 1 - assert update_attested_bid(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] - assert get_light_client_update_for_period(test, signature_period).spec is None - assert update_attested_bid(get_light_client_finality_update(test).data) == bids_a[-1] - assert update_attested_bid(get_light_client_optimistic_update(test).data) == bids_a[-1] - bids_a.append(bid_a) - - # Data has been restored - state_a_period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) - for period in range(state_period + 1, state_a_period): - assert get_light_client_update_for_period(test, period).spec is not None + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, period).data) == bid_1_5 + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bid_4_n + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bid_4_n # Finish test - yield from finish_test(test) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 - 'DENEB_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_capella_deneb_reorg_aligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) - 'DENEB_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_capella_deneb_reorg_unaligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, CAPELLA, DENEB) - - -@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) -@spec_test -@with_config_overrides({ - 'DENEB_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 - 'ELECTRA_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_electra_reorg_aligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, DENEB, ELECTRA) - - -@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) -@spec_test -@with_config_overrides({ - 'DENEB_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) - 'ELECTRA_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_electra_reorg_unaligned(spec, phases, state): - yield from run_test_multi_fork(spec, phases, state, DENEB, ELECTRA) + yield from finish_lc_data_collection_test(test) diff --git a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py new file mode 100644 index 0000000000..03b7286988 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py @@ -0,0 +1,40 @@ +from eth2spec.test.context import ( + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, + with_presets, + with_state, +) +from eth2spec.test.helpers.constants import ( + BELLATRIX, CAPELLA, DENEB, + MINIMAL, +) +from eth2spec.test.helpers.light_client_data_collection import ( + run_lc_data_collection_test_multi_fork, +) + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 + 'DENEB_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_capella_deneb_reorg_aligned(spec, phases, state): + yield from run_lc_data_collection_test_multi_fork(spec, phases, state, CAPELLA, DENEB) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) +@spec_test +@with_config_overrides({ + 'CAPELLA_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) + 'DENEB_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_capella_deneb_reorg_unaligned(spec, phases, state): + yield from run_lc_data_collection_test_multi_fork(spec, phases, state, CAPELLA, DENEB) diff --git a/tests/core/pyspec/eth2spec/test/electra/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/electra/light_client/test_data_collection.py new file mode 100644 index 0000000000..d85b0dfda1 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/electra/light_client/test_data_collection.py @@ -0,0 +1,41 @@ +from eth2spec.test.context import ( + spec_test, + with_config_overrides, + with_matching_spec_config, + with_phases, + with_presets, + with_state, +) +from eth2spec.test.helpers.constants import ( + CAPELLA, DENEB, ELECTRA, + MINIMAL, +) +from eth2spec.test.helpers.light_client_data_collection import ( + run_lc_data_collection_test_multi_fork, +) + + +@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) +@spec_test +@with_config_overrides({ + 'DENEB_FORK_EPOCH': 1 * 8, # SyncCommitteePeriod 1 + 'ELECTRA_FORK_EPOCH': 2 * 8, # SyncCommitteePeriod 2 +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_electra_reorg_aligned(spec, phases, state): + yield from run_lc_data_collection_test_multi_fork(spec, phases, state, DENEB, ELECTRA) + + +@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) +@spec_test +@with_config_overrides({ + 'DENEB_FORK_EPOCH': 1 * 8 + 4, # SyncCommitteePeriod 1 (+ 4 epochs) + 'ELECTRA_FORK_EPOCH': 3 * 8 + 4, # SyncCommitteePeriod 3 (+ 4 epochs) +}, emit=False) +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_electra_reorg_unaligned(spec, phases, state): + yield from run_lc_data_collection_test_multi_fork(spec, phases, state, DENEB, ELECTRA) diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py b/tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py new file mode 100644 index 0000000000..d56ea05310 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py @@ -0,0 +1,897 @@ +from typing import (Any, Dict, List, Set) +from dataclasses import dataclass + +from eth_utils import encode_hex +from eth2spec.test.helpers.constants import ( + ALTAIR, +) +from eth2spec.test.helpers.fork_transition import ( + transition_across_forks, +) +from eth2spec.test.helpers.forks import ( + is_post_altair, +) +from eth2spec.test.helpers.light_client import ( + compute_start_slot_at_sync_committee_period, + get_sync_aggregate, + latest_current_sync_committee_gindex, + latest_finalized_root_gindex, + latest_next_sync_committee_gindex, + latest_normalize_merkle_branch, + upgrade_lc_header_to_new_spec, + upgrade_lc_update_to_new_spec, +) + + +def _next_epoch_boundary_slot(spec, slot): + # Compute the first possible epoch boundary state slot of a `Checkpoint` + # referring to a block at given slot. + epoch = spec.compute_epoch_at_slot(slot + spec.SLOTS_PER_EPOCH - 1) + return spec.compute_start_slot_at_epoch(epoch) + + +@dataclass(frozen=True) +class BlockID(object): + slot: Any + root: Any + + +def _block_to_block_id(block): + return BlockID( + slot=block.message.slot, + root=block.message.hash_tree_root(), + ) + + +def _state_to_block_id(state): + parent_header = state.latest_block_header.copy() + parent_header.state_root = state.hash_tree_root() + return BlockID(slot=parent_header.slot, root=parent_header.hash_tree_root()) + + +def get_lc_bootstrap_block_id(bootstrap): + return BlockID( + slot=bootstrap.header.beacon.slot, + root=bootstrap.header.beacon.hash_tree_root(), + ) + + +def get_lc_update_attested_block_id(update): + return BlockID( + slot=update.attested_header.beacon.slot, + root=update.attested_header.beacon.hash_tree_root(), + ) + + +@dataclass +class ForkedBeaconState(object): + spec: Any + data: Any + + +@dataclass +class ForkedSignedBeaconBlock(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientHeader(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientBootstrap(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientUpdate(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientFinalityUpdate(object): + spec: Any + data: Any + + +@dataclass +class ForkedLightClientOptimisticUpdate(object): + spec: Any + data: Any + + +@dataclass +class CachedLightClientData(object): + # Sync committee branches at block's post-state + current_sync_committee_branch: Any # CurrentSyncCommitteeBranch + next_sync_committee_branch: Any # NextSyncCommitteeBranch + + # Finality information at block's post-state + finalized_slot: Any # Slot + finality_branch: Any # FinalityBranch + + # Best / latest light client data + current_period_best_update: ForkedLightClientUpdate + latest_signature_slot: Any # Slot + + +@dataclass +class LightClientDataCache(object): + # Cached data for creating future `LightClientUpdate` instances. + # Key is the block ID of which the post state was used to get the data. + # Data stored for the finalized head block and all non-finalized blocks. + data: Dict[BlockID, CachedLightClientData] + + # Light client data for the latest slot that was signed by at least + # `MIN_SYNC_COMMITTEE_PARTICIPANTS`. May be older than head + latest: ForkedLightClientFinalityUpdate + + # The earliest slot for which light client data is imported + tail_slot: Any # Slot + + +@dataclass +class LightClientDataDB(object): + headers: Dict[Any, ForkedLightClientHeader] # Root -> ForkedLightClientHeader + current_branches: Dict[Any, Any] # Slot -> CurrentSyncCommitteeBranch + sync_committees: Dict[Any, Any] # SyncCommitteePeriod -> SyncCommittee + best_updates: Dict[Any, ForkedLightClientUpdate] # SyncCommitteePeriod -> ForkedLightClientUpdate + + +@dataclass +class LightClientDataStore(object): + spec: Any + + # Cached data to accelerate creating light client data + cache: LightClientDataCache + + # Persistent light client data + db: LightClientDataDB + + +@dataclass +class LightClientDataCollectionTest(object): + steps: List[Dict[str, Any]] + files: Set[str] + + # Fork schedule + phases: Any + + # History access + blocks: Dict[Any, ForkedSignedBeaconBlock] # Block root -> ForkedSignedBeaconBlock + finalized_block_roots: Dict[Any, Any] # Slot -> Root + states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState + finalized_checkpoint_states: Dict[Any, ForkedBeaconState] # State root -> ForkedBeaconState + latest_finalized_epoch: Any # Epoch + latest_finalized_bid: BlockID + historical_tail_slot: Any # Slot + + # Light client data + lc_data_store: LightClientDataStore + + +def get_ancestor_of_block_id(test, bid, slot): # -> Optional[BlockID] + try: + block = test.blocks[bid.root] + while True: + if block.data.message.slot <= slot: + return _block_to_block_id(block.data) + + block = test.blocks[block.data.message.parent_root] + except KeyError: + return None + + +def _block_id_at_finalized_slot(test, slot): # -> Optional[BlockID] + while slot >= test.historical_tail_slot: + try: + return BlockID(slot=slot, root=test.finalized_block_roots[slot]) + except KeyError: + slot = slot - 1 + return None + + +def _get_current_sync_committee_for_finalized_period(test, period): # -> Optional[SyncCommittee] + low_slot = max( + test.historical_tail_slot, + test.lc_data_store.spec.compute_start_slot_at_epoch( + test.lc_data_store.spec.config.ALTAIR_FORK_EPOCH) + ) + if period < test.lc_data_store.spec.compute_sync_committee_period_at_slot(low_slot): + return None + period_start_slot = compute_start_slot_at_sync_committee_period(test.lc_data_store.spec, period) + sync_committee_slot = max(period_start_slot, low_slot) + bid = _block_id_at_finalized_slot(test, sync_committee_slot) + if bid is None: + return None + block = test.blocks[bid.root] + state = test.finalized_checkpoint_states[block.data.message.state_root] + if sync_committee_slot > state.data.slot: + state.spec, state.data, _ = transition_across_forks( + state.spec, state.data, sync_committee_slot, phases=test.phases) + assert is_post_altair(state.spec) + return state.data.current_sync_committee + + +def _light_client_header_for_block(test, block): # -> ForkedLightClientHeader + if not is_post_altair(block.spec): + spec = test.phases[ALTAIR] + else: + spec = block.spec + return ForkedLightClientHeader(spec=spec, data=spec.block_to_light_client_header(block.data)) + + +def _light_client_header_for_block_id(test, bid): # -> ForkedLightClientHeader + block = test.blocks[bid.root] + if not is_post_altair(block.spec): + spec = test.phases[ALTAIR] + else: + spec = block.spec + return ForkedLightClientHeader(spec=spec, data=spec.block_to_light_client_header(block.data)) + + +def _sync_aggregate_for_block_id(test, bid): # -> Optional[SyncAggregate] + block = test.blocks[bid.root] + if not is_post_altair(block.spec): + return None + return block.data.message.body.sync_aggregate + + +def _get_light_client_data(lc_data_store, bid): # -> CachedLightClientData + # Fetch cached light client data about a given block. + # Data must be cached (`_cache_lc_data`) before calling this function. + try: + return lc_data_store.cache.data[bid] + except KeyError: + raise ValueError("Trying to get light client data that was not cached") + + +def _cache_lc_data(lc_data_store, spec, state, bid, current_period_best_update, latest_signature_slot): + # Cache data for a given block and its post-state to speed up creating future + # `LightClientUpdate` and `LightClientBootstrap` instances that refer to this + # block and state. + cached_data = CachedLightClientData( + current_sync_committee_branch=latest_normalize_merkle_branch( + lc_data_store.spec, + spec.compute_merkle_proof(state, spec.current_sync_committee_gindex_at_slot(state.slot)), + latest_current_sync_committee_gindex(lc_data_store.spec)), + next_sync_committee_branch=latest_normalize_merkle_branch( + lc_data_store.spec, + spec.compute_merkle_proof(state, spec.next_sync_committee_gindex_at_slot(state.slot)), + latest_next_sync_committee_gindex(lc_data_store.spec)), + finalized_slot=spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch), + finality_branch=latest_normalize_merkle_branch( + lc_data_store.spec, + spec.compute_merkle_proof(state, spec.finalized_root_gindex_at_slot(state.slot)), + latest_finalized_root_gindex(lc_data_store.spec)), + current_period_best_update=current_period_best_update, + latest_signature_slot=latest_signature_slot, + ) + if bid in lc_data_store.cache.data: + raise ValueError("Redundant `_cache_lc_data` call") + lc_data_store.cache.data[bid] = cached_data + + +def _delete_light_client_data(lc_data_store, bid): + # Delete cached light client data for a given block. This needs to be called + # when a block becomes unreachable due to finalization of a different fork. + del lc_data_store.cache.data[bid] + + +def _create_lc_finality_update_from_lc_data(test, + attested_bid, + signature_slot, + sync_aggregate): # -> ForkedLightClientFinalityUpdate + attested_header = _light_client_header_for_block_id(test, attested_bid) + attested_data = _get_light_client_data(test.lc_data_store, attested_bid) + finalized_bid = _block_id_at_finalized_slot(test, attested_data.finalized_slot) + if finalized_bid is not None: + if finalized_bid.slot != attested_data.finalized_slot: + # Empty slots at end of epoch, update cache for latest block slot + attested_data.finalized_slot = finalized_bid.slot + if finalized_bid.slot == attested_header.spec.GENESIS_SLOT: + finalized_header = ForkedLightClientHeader( + spec=attested_header.spec, + data=attested_header.spec.LightClientHeader(), + ) + else: + finalized_header = _light_client_header_for_block_id(test, finalized_bid) + finalized_header = ForkedLightClientHeader( + spec=attested_header.spec, + data=upgrade_lc_header_to_new_spec( + finalized_header.spec, + attested_header.spec, + finalized_header.data, + ) + ) + finality_branch = attested_data.finality_branch + return ForkedLightClientFinalityUpdate( + spec=attested_header.spec, + data=attested_header.spec.LightClientFinalityUpdate( + attested_header=attested_header.data, + finalized_header=finalized_header.data, + finality_branch=finality_branch, + sync_aggregate=sync_aggregate, + signature_slot=signature_slot, + ), + ) + + +def _create_lc_update_from_lc_data(test, + attested_bid, + signature_slot, + sync_aggregate, + next_sync_committee): # -> ForkedLightClientUpdate + finality_update = _create_lc_finality_update_from_lc_data( + test, attested_bid, signature_slot, sync_aggregate) + attested_data = _get_light_client_data(test.lc_data_store, attested_bid) + return ForkedLightClientUpdate( + spec=finality_update.spec, + data=finality_update.spec.LightClientUpdate( + attested_header=finality_update.data.attested_header, + next_sync_committee=next_sync_committee, + next_sync_committee_branch=attested_data.next_sync_committee_branch, + finalized_header=finality_update.data.finalized_header, + finality_branch=finality_update.data.finality_branch, + sync_aggregate=finality_update.data.sync_aggregate, + signature_slot=finality_update.data.signature_slot, + ) + ) + + +def _create_lc_update(test, spec, state, block, parent_bid): + # Create `LightClientUpdate` instances for a given block and its post-state, + # and keep track of best / latest ones. Data about the parent block's + # post-state must be cached (`_cache_lc_data`) before calling this. + + # Verify attested block (parent) is recent enough and that state is available + attested_bid = parent_bid + attested_slot = attested_bid.slot + if attested_slot < test.lc_data_store.cache.tail_slot: + _cache_lc_data( + test.lc_data_store, + spec, + state, + _block_to_block_id(block), + current_period_best_update=ForkedLightClientUpdate(spec=None, data=None), + latest_signature_slot=spec.GENESIS_SLOT, + ) + return + + # If sync committee period changed, reset `best` + attested_period = spec.compute_sync_committee_period_at_slot(attested_slot) + signature_slot = block.message.slot + signature_period = spec.compute_sync_committee_period_at_slot(signature_slot) + attested_data = _get_light_client_data(test.lc_data_store, attested_bid) + if attested_period != signature_period: + best = ForkedLightClientUpdate(spec=None, data=None) + else: + best = attested_data.current_period_best_update + + # If sync committee does not have sufficient participants, do not bump latest + sync_aggregate = block.message.body.sync_aggregate + num_active_participants = sum(sync_aggregate.sync_committee_bits) + if num_active_participants < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS: + latest_signature_slot = attested_data.latest_signature_slot + else: + latest_signature_slot = signature_slot + + # To update `best`, sync committee must have sufficient participants, and + # `signature_slot` must be in `attested_slot`'s sync committee period + if ( + num_active_participants < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS + or attested_period != signature_period + ): + _cache_lc_data( + test.lc_data_store, + spec, + state, + _block_to_block_id(block), + current_period_best_update=best, + latest_signature_slot=latest_signature_slot, + ) + return + + # Check if light client data improved + update = _create_lc_update_from_lc_data( + test, attested_bid, signature_slot, sync_aggregate, state.next_sync_committee) + is_better = ( + best.spec is None + or spec.is_better_update(update.data, upgrade_lc_update_to_new_spec( + best.spec, update.spec, best.data, test.phases)) + ) + + # Update best light client data for current sync committee period + if is_better: + best = update + _cache_lc_data( + test.lc_data_store, + spec, + state, + _block_to_block_id(block), + current_period_best_update=best, + latest_signature_slot=latest_signature_slot, + ) + + +def _create_lc_bootstrap(test, spec, bid): + block = test.blocks[bid.root] + period = spec.compute_sync_committee_period_at_slot(bid.slot) + if period not in test.lc_data_store.db.sync_committees: + test.lc_data_store.db.sync_committees[period] = \ + _get_current_sync_committee_for_finalized_period(test, period) + test.lc_data_store.db.headers[bid.root] = ForkedLightClientHeader( + spec=block.spec, data=block.spec.block_to_light_client_header(block.data)) + test.lc_data_store.db.current_branches[bid.slot] = \ + _get_light_client_data(test.lc_data_store, bid).current_sync_committee_branch + + +def _process_new_block_for_light_client(test, spec, state, block, parent_bid): + # Update light client data with information from a new block. + if block.message.slot < test.lc_data_store.cache.tail_slot: + return + + if is_post_altair(spec): + _create_lc_update(test, spec, state, block, parent_bid) + else: + raise ValueError("`tail_slot` cannot be before Altair") + + +def _process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid): + # Update light client data to account for a new head block. + # Note that `old_finalized_bid` is not yet updated when this is called. + if head_bid.slot < test.lc_data_store.cache.tail_slot: + return + + # Commit best light client data for non-finalized periods + head_period = spec.compute_sync_committee_period_at_slot(head_bid.slot) + low_slot = max(test.lc_data_store.cache.tail_slot, old_finalized_bid.slot) + low_period = spec.compute_sync_committee_period_at_slot(low_slot) + bid = head_bid + for period in reversed(range(low_period, head_period + 1)): + period_end_slot = compute_start_slot_at_sync_committee_period(spec, period + 1) - 1 + bid = get_ancestor_of_block_id(test, bid, period_end_slot) + if bid is None or bid.slot < low_slot: + break + best = _get_light_client_data(test.lc_data_store, bid).current_period_best_update + if ( + best.spec is None + or sum(best.data.sync_aggregate.sync_committee_bits) < spec.MIN_SYNC_COMMITTEE_PARTICIPANTS + ): + test.lc_data_store.db.best_updates.pop(period, None) + else: + test.lc_data_store.db.best_updates[period] = best + + # Update latest light client data + head_data = _get_light_client_data(test.lc_data_store, head_bid) + signature_slot = head_data.latest_signature_slot + if signature_slot <= low_slot: + test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) + return + signature_bid = get_ancestor_of_block_id(test, head_bid, signature_slot) + if signature_bid is None or signature_bid.slot <= low_slot: + test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) + return + attested_bid = get_ancestor_of_block_id(test, signature_bid, signature_bid.slot - 1) + if attested_bid is None or attested_bid.slot < low_slot: + test.lc_data_store.cache.latest = ForkedLightClientFinalityUpdate(spec=None, data=None) + return + sync_aggregate = _sync_aggregate_for_block_id(test, signature_bid) + assert sync_aggregate is not None + test.lc_data_store.cache.latest = _create_lc_finality_update_from_lc_data( + test, attested_bid, signature_slot, sync_aggregate) + + +def _process_finalization_for_light_client(test, spec, finalized_bid, old_finalized_bid): + # Prune cached data that is no longer useful for creating future + # `LightClientUpdate` and `LightClientBootstrap` instances. + # This needs to be called whenever `finalized_checkpoint` changes. + finalized_slot = finalized_bid.slot + if finalized_slot < test.lc_data_store.cache.tail_slot: + return + + # Cache `LightClientBootstrap` for newly finalized epoch boundary blocks + first_new_slot = old_finalized_bid.slot + 1 + low_slot = max(first_new_slot, test.lc_data_store.cache.tail_slot) + boundary_slot = finalized_slot + while boundary_slot >= low_slot: + bid = _block_id_at_finalized_slot(test, boundary_slot) + if bid is None: + break + if bid.slot >= low_slot: + _create_lc_bootstrap(test, spec, bid) + boundary_slot = _next_epoch_boundary_slot(spec, bid.slot) + if boundary_slot < spec.SLOTS_PER_EPOCH: + break + boundary_slot = boundary_slot - spec.SLOTS_PER_EPOCH + + # Prune light client data that is no longer referrable by future updates + bids_to_delete = [] + for bid in test.lc_data_store.cache.data: + if bid.slot >= finalized_bid.slot: + continue + bids_to_delete.append(bid) + for bid in bids_to_delete: + _delete_light_client_data(test.lc_data_store, bid) + + +def get_light_client_bootstrap(test, block_root): # -> ForkedLightClientBootstrap + try: + header = test.lc_data_store.db.headers[block_root] + except KeyError: + return ForkedLightClientBootstrap(spec=None, data=None) + + slot = header.data.beacon.slot + period = header.spec.compute_sync_committee_period_at_slot(slot) + return ForkedLightClientBootstrap( + spec=header.spec, + data=header.spec.LightClientBootstrap( + header=header.data, + current_sync_committee=test.lc_data_store.db.sync_committees[period], + current_sync_committee_branch=test.lc_data_store.db.current_branches[slot], + ) + ) + + +def get_light_client_update_for_period(test, period): # -> ForkedLightClientUpdate + try: + return test.lc_data_store.db.best_updates[period] + except KeyError: + return ForkedLightClientUpdate(spec=None, data=None) + + +def get_light_client_finality_update(test): # -> ForkedLightClientFinalityUpdate + return test.lc_data_store.cache.latest + + +def get_light_client_optimistic_update(test): # -> ForkedLightClientOptimisticUpdate + finality_update = get_light_client_finality_update(test) + if finality_update.spec is None: + return ForkedLightClientOptimisticUpdate(spec=None, data=None) + return ForkedLightClientOptimisticUpdate( + spec=finality_update.spec, + data=finality_update.spec.LightClientOptimisticUpdate( + attested_header=finality_update.data.attested_header, + sync_aggregate=finality_update.data.sync_aggregate, + signature_slot=finality_update.data.signature_slot, + ), + ) + + +def setup_lc_data_collection_test(spec, state, phases=None): + assert spec.compute_slots_since_epoch_start(state.slot) == 0 + + test = LightClientDataCollectionTest( + steps=[], + files=set(), + phases=phases, + blocks={}, + finalized_block_roots={}, + states={}, + finalized_checkpoint_states={}, + latest_finalized_epoch=state.finalized_checkpoint.epoch, + latest_finalized_bid=BlockID( + slot=spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch), + root=state.finalized_checkpoint.root, + ), + historical_tail_slot=state.slot, + lc_data_store=LightClientDataStore( + spec=spec, + cache=LightClientDataCache( + data={}, + latest=ForkedLightClientFinalityUpdate(spec=None, data=None), + tail_slot=max(state.slot, spec.compute_start_slot_at_epoch(spec.config.ALTAIR_FORK_EPOCH)), + ), + db=LightClientDataDB( + headers={}, + current_branches={}, + sync_committees={}, + best_updates={}, + ), + ), + ) + bid = _state_to_block_id(state) + yield "initial_state", state + test.blocks[bid.root] = ForkedSignedBeaconBlock(spec=spec, data=spec.SignedBeaconBlock( + message=spec.BeaconBlock(state_root=state.hash_tree_root()), + )) + test.finalized_block_roots[bid.slot] = bid.root + test.states[state.hash_tree_root()] = ForkedBeaconState(spec=spec, data=state) + test.finalized_checkpoint_states[state.hash_tree_root()] = ForkedBeaconState(spec=spec, data=state) + _cache_lc_data( + test.lc_data_store, spec, state, bid, + current_period_best_update=ForkedLightClientUpdate(spec=None, data=None), + latest_signature_slot=spec.GENESIS_SLOT, + ) + _create_lc_bootstrap(test, spec, bid) + + return test + + +def finish_lc_data_collection_test(test): + yield "steps", test.steps + + +def _encode_lc_object(test, prefix, obj, slot, genesis_validators_root): + yield from [] # Consistently enable `yield from` syntax in calling tests + + file_name = f"{prefix}_{slot}_{encode_hex(obj.data.hash_tree_root())}" + if file_name not in test.files: + test.files.add(file_name) + yield file_name, obj.data + return { + "fork_digest": encode_hex(obj.spec.compute_fork_digest( + obj.spec.compute_fork_version(obj.spec.compute_epoch_at_slot(slot)), + genesis_validators_root, + )), + "data": file_name, + } + + +def add_new_block(test, spec, state, slot=None, num_sync_participants=0): + if slot is None: + slot = state.slot + 1 + assert slot > state.slot + parent_bid = _state_to_block_id(state) + + # Advance to target slot - 1 to ensure sync aggregate can be efficiently computed + if state.slot < slot - 1: + spec, state, _ = transition_across_forks(spec, state, slot - 1, phases=test.phases) + + # Compute sync aggregate, using: + # - sync committee based on target slot + # - fork digest based on target slot - 1 + # - signed data based on parent_bid.slot + # All three slots may be from different forks + sync_aggregate, signature_slot = get_sync_aggregate( + spec, state, num_participants=num_sync_participants, phases=test.phases) + assert signature_slot == slot + + # Apply final block with computed sync aggregate + spec, state, block = transition_across_forks( + spec, state, slot, phases=test.phases, with_block=True, sync_aggregate=sync_aggregate) + bid = _block_to_block_id(block) + test.blocks[bid.root] = ForkedSignedBeaconBlock(spec=spec, data=block) + test.states[block.message.state_root] = ForkedBeaconState(spec=spec, data=state) + _process_new_block_for_light_client(test, spec, state, block, parent_bid) + block_obj = yield from _encode_lc_object( + test, "block", ForkedSignedBeaconBlock(spec=spec, data=block), block.message.slot, + state.genesis_validators_root, + ) + test.steps.append({ + "new_block": block_obj + }) + return spec, state, bid + + +def select_new_head(test, spec, head_bid): + old_finalized_bid = test.latest_finalized_bid + _process_head_change_for_light_client(test, spec, head_bid, old_finalized_bid) + + # Process finalization + block = test.blocks[head_bid.root] + state = test.states[block.data.message.state_root] + if state.data.finalized_checkpoint.epoch != spec.GENESIS_EPOCH: + block = test.blocks[state.data.finalized_checkpoint.root] + bid = _block_to_block_id(block.data) + new_finalized_bid = bid + if new_finalized_bid.slot > old_finalized_bid.slot: + old_finalized_epoch = None + new_finalized_epoch = state.data.finalized_checkpoint.epoch + while bid.slot > test.latest_finalized_bid.slot: + test.finalized_block_roots[bid.slot] = bid.root + finalized_epoch = spec.compute_epoch_at_slot(bid.slot + spec.SLOTS_PER_EPOCH - 1) + if finalized_epoch != old_finalized_epoch: + state = test.states[block.data.message.state_root] + test.finalized_checkpoint_states[block.data.message.state_root] = state + old_finalized_epoch = finalized_epoch + block = test.blocks[block.data.message.parent_root] + bid = _block_to_block_id(block.data) + test.latest_finalized_epoch = new_finalized_epoch + test.latest_finalized_bid = new_finalized_bid + _process_finalization_for_light_client(test, spec, new_finalized_bid, old_finalized_bid) + + blocks_to_delete = [] + for block_root, block in test.blocks.items(): + if block.data.message.slot < new_finalized_bid.slot: + blocks_to_delete.append(block_root) + for block_root in blocks_to_delete: + del test.blocks[block_root] + states_to_delete = [] + for state_root, state in test.states.items(): + if state.data.slot < new_finalized_bid.slot: + states_to_delete.append(state_root) + for state_root in states_to_delete: + del test.states[state_root] + + yield from [] # Consistently enable `yield from` syntax in calling tests + + bootstraps = [] + for state in test.finalized_checkpoint_states.values(): + bid = _state_to_block_id(state.data) + entry = { + "block_root": encode_hex(bid.root), + } + bootstrap = get_light_client_bootstrap(test, bid.root) + if bootstrap.spec is not None: + bootstrap_obj = yield from _encode_lc_object( + test, "bootstrap", bootstrap, bootstrap.data.header.beacon.slot, + state.data.genesis_validators_root, + ) + entry["bootstrap"] = bootstrap_obj + bootstraps.append(entry) + + best_updates = [] + low_period = spec.compute_sync_committee_period_at_slot(test.lc_data_store.cache.tail_slot) + head_period = spec.compute_sync_committee_period_at_slot(head_bid.slot) + for period in range(low_period, head_period + 1): + entry = { + "period": int(period), + } + update = get_light_client_update_for_period(test, period) + if update.spec is not None: + update_obj = yield from _encode_lc_object( + test, "update", update, update.data.attested_header.beacon.slot, + state.data.genesis_validators_root, + ) + entry["update"] = update_obj + best_updates.append(entry) + + checks = { + "latest_finalized_checkpoint": { + "epoch": int(test.latest_finalized_epoch), + "root": encode_hex(test.latest_finalized_bid.root), + }, + "bootstraps": bootstraps, + "best_updates": best_updates, + } + finality_update = get_light_client_finality_update(test) + if finality_update.spec is not None: + finality_update_obj = yield from _encode_lc_object( + test, "finality_update", finality_update, finality_update.data.attested_header.beacon.slot, + state.data.genesis_validators_root, + ) + checks["latest_finality_update"] = finality_update_obj + optimistic_update = get_light_client_optimistic_update(test) + if optimistic_update.spec is not None: + optimistic_update_obj = yield from _encode_lc_object( + test, "optimistic_update", optimistic_update, optimistic_update.data.attested_header.beacon.slot, + state.data.genesis_validators_root, + ) + checks["latest_optimistic_update"] = optimistic_update_obj + + test.steps.append({ + "new_head": { + "head_block_root": encode_hex(head_bid.root), + "checks": checks, + } + }) + + +def run_lc_data_collection_test_multi_fork(spec, phases, state, fork_1, fork_2): + # Start test + test = yield from setup_lc_data_collection_test(spec, state, phases=phases) + + # Genesis block is post Altair and is finalized, so can be used as bootstrap + genesis_bid = BlockID(slot=state.slot, root=spec.BeaconBlock(state_root=state.hash_tree_root()).hash_tree_root()) + assert get_lc_bootstrap_block_id(get_light_client_bootstrap(test, genesis_bid.root).data) == genesis_bid + + # Shared history up to final epoch of period before `fork_1` + fork_1_epoch = getattr(phases[fork_1].config, fork_1.upper() + '_FORK_EPOCH') + fork_1_period = spec.compute_sync_committee_period(fork_1_epoch) + slot = compute_start_slot_at_sync_committee_period(spec, fork_1_period) - spec.SLOTS_PER_EPOCH + spec, state, bid = yield from add_new_block(test, spec, state, slot=slot, num_sync_participants=1) + yield from select_new_head(test, spec, bid) + assert get_light_client_bootstrap(test, bid.root).spec is None + slot_period = spec.compute_sync_committee_period_at_slot(slot) + if slot_period == 0: + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, 0).data) == genesis_bid + else: + for period in range(0, slot_period): + assert get_light_client_update_for_period(test, period).spec is None # attested period != signature period + state_period = spec.compute_sync_committee_period_at_slot(state.slot) + + # Branch A: Advance past `fork_2`, having blocks at slots 0 and 4 of each epoch + spec_a = spec + state_a = state + slot_a = state_a.slot + bids_a = [bid] + num_sync_participants_a = 1 + fork_2_epoch = getattr(phases[fork_2].config, fork_2.upper() + '_FORK_EPOCH') + while spec_a.get_current_epoch(state_a) <= fork_2_epoch: + attested_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + slot_a += 4 + signature_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + if signature_period != attested_period: + num_sync_participants_a = 0 + num_sync_participants_a += 1 + spec_a, state_a, bid_a = yield from add_new_block( + test, spec_a, state_a, slot=slot_a, num_sync_participants=num_sync_participants_a) + yield from select_new_head(test, spec_a, bid_a) + for bid in bids_a: + assert get_light_client_bootstrap(test, bid.root).spec is None + if attested_period == signature_period: + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] + else: + assert signature_period == attested_period + 1 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] + assert get_light_client_update_for_period(test, signature_period).spec is None + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bids_a[-1] + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bids_a[-1] + bids_a.append(bid_a) + + # Branch B: Advance past `fork_2`, having blocks at slots 1 and 5 of each epoch but no sync participation + spec_b = spec + state_b = state + slot_b = state_b.slot + bids_b = [bid] + while spec_b.get_current_epoch(state_b) <= fork_2_epoch: + slot_b += 4 + signature_period = spec_b.compute_sync_committee_period_at_slot(slot_b) + spec_b, state_b, bid_b = yield from add_new_block( + test, spec_b, state_b, slot=slot_b) + # Simulate that this does not become head yet, e.g., this branch was withheld + for bid in bids_b: + assert get_light_client_bootstrap(test, bid.root).spec is None + bids_b.append(bid_b) + + # Branch B: Another block that becomes head + attested_period = spec_b.compute_sync_committee_period_at_slot(slot_b) + slot_b += 1 + signature_period = spec_b.compute_sync_committee_period_at_slot(slot_b) + num_sync_participants_b = 1 + spec_b, state_b, bid_b = yield from add_new_block( + test, spec_b, state_b, slot=slot_b, num_sync_participants=num_sync_participants_b) + yield from select_new_head(test, spec_b, bid_b) + for bid in bids_b: + assert get_light_client_bootstrap(test, bid.root).spec is None + if attested_period == signature_period: + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_b[-1] + else: + assert signature_period == attested_period + 1 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_b[-2] + assert get_light_client_update_for_period(test, signature_period).spec is None + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bids_b[-1] + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bids_b[-1] + bids_b.append(bid_b) + + # All data for periods between the common ancestor of the two branches should have reorged. + # As there was no sync participation on branch B, that means it is deleted. + state_b_period = spec_b.compute_sync_committee_period_at_slot(state_b.slot) + for period in range(state_period + 1, state_b_period): + assert get_light_client_update_for_period(test, period).spec is None + + # Branch A: Another block, reorging branch B once more + attested_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + slot_a = slot_b + 1 + signature_period = spec_a.compute_sync_committee_period_at_slot(slot_a) + if signature_period != attested_period: + num_sync_participants_a = 0 + num_sync_participants_a += 1 + spec_a, state_a, bid_a = yield from add_new_block( + test, spec_a, state_a, slot=slot_a, num_sync_participants=num_sync_participants_a) + yield from select_new_head(test, spec_a, bid_a) + for bid in bids_a: + assert get_light_client_bootstrap(test, bid.root).spec is None + if attested_period == signature_period: + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] + else: + assert signature_period == attested_period + 1 + assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] + assert get_light_client_update_for_period(test, signature_period).spec is None + assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bids_a[-1] + assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bids_a[-1] + bids_a.append(bid_a) + + # Data has been restored + state_a_period = spec_a.compute_sync_committee_period_at_slot(state_a.slot) + for period in range(state_period + 1, state_a_period): + assert get_light_client_update_for_period(test, period).spec is not None + + # Finish test + yield from finish_lc_data_collection_test(test) From 24dffad1af31fe2dbda3b78a043de4b7445f9a2c Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Nov 2024 14:28:19 +0100 Subject: [PATCH 14/20] Link tests with generator --- tests/generators/light_client/main.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index 6534524fe3..a5775b1cbe 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -12,11 +12,23 @@ bellatrix_mods = altair_mods _new_capella_mods = {key: 'eth2spec.test.capella.light_client.test_' + key for key in [ + 'data_collection', 'single_merkle_proof', + 'sync', ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) - deneb_mods = capella_mods - electra_mods = deneb_mods + + _new_deneb_mods = {key: 'eth2spec.test.deneb.light_client.test_' + key for key in [ + 'data_collection', + 'sync', + ]} + deneb_mods = combine_mods(_new_deneb_mods, capella_mods) + + _new_electra_mods = {key: 'eth2spec.test.electra.light_client.test_' + key for key in [ + 'data_collection', + 'sync', + ]} + electra_mods = combine_mods(_new_electra_mods, deneb_mods) all_mods = { ALTAIR: altair_mods, From eaed600263d10c1e7f15f2a98d09fb2bfffd5a73 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Nov 2024 14:29:44 +0100 Subject: [PATCH 15/20] Lint --- .../test/capella/light_client/test_sync.py | 1 + .../light_client/test_data_collection.py | 1 + .../test/deneb/light_client/test_sync.py | 1 + .../test/electra/light_client/test_sync.py | 1 + .../helpers/light_client_data_collection.py | 24 ++++++++++++++----- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py index 3958900be5..99a56f96e0 100644 --- a/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py @@ -15,6 +15,7 @@ run_lc_sync_test_upgraded_store_with_legacy_data, ) + @with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) @spec_test @with_config_overrides({ diff --git a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py index 03b7286988..5e894a5d13 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py +++ b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py @@ -14,6 +14,7 @@ run_lc_data_collection_test_multi_fork, ) + @with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) @spec_test @with_config_overrides({ diff --git a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py index d19e1e0238..45a8ff2c8f 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py @@ -16,6 +16,7 @@ run_lc_sync_test_upgraded_store_with_legacy_data, ) + @with_phases(phases=[CAPELLA], other_phases=[DENEB]) @spec_test @with_config_overrides({ diff --git a/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py index 2b20552d6b..c37e8b21e1 100644 --- a/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py @@ -16,6 +16,7 @@ run_lc_sync_test_upgraded_store_with_legacy_data, ) + @with_phases(phases=[DENEB], other_phases=[ELECTRA]) @spec_test @with_config_overrides({ diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py b/tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py index d56ea05310..5de9b37c61 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client_data_collection.py @@ -816,10 +816,14 @@ def run_lc_data_collection_test_multi_fork(spec, phases, state, fork_1, fork_2): for bid in bids_a: assert get_light_client_bootstrap(test, bid.root).spec is None if attested_period == signature_period: - assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] + assert get_lc_update_attested_block_id( + get_light_client_update_for_period(test, attested_period).data, + ) == bids_a[-1] else: assert signature_period == attested_period + 1 - assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] + assert get_lc_update_attested_block_id( + get_light_client_update_for_period(test, attested_period).data, + ) == bids_a[-2] assert get_light_client_update_for_period(test, signature_period).spec is None assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bids_a[-1] assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bids_a[-1] @@ -851,10 +855,14 @@ def run_lc_data_collection_test_multi_fork(spec, phases, state, fork_1, fork_2): for bid in bids_b: assert get_light_client_bootstrap(test, bid.root).spec is None if attested_period == signature_period: - assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_b[-1] + assert get_lc_update_attested_block_id( + get_light_client_update_for_period(test, attested_period).data, + ) == bids_b[-1] else: assert signature_period == attested_period + 1 - assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_b[-2] + assert get_lc_update_attested_block_id( + get_light_client_update_for_period(test, attested_period).data, + ) == bids_b[-2] assert get_light_client_update_for_period(test, signature_period).spec is None assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bids_b[-1] assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bids_b[-1] @@ -879,10 +887,14 @@ def run_lc_data_collection_test_multi_fork(spec, phases, state, fork_1, fork_2): for bid in bids_a: assert get_light_client_bootstrap(test, bid.root).spec is None if attested_period == signature_period: - assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-1] + assert get_lc_update_attested_block_id( + get_light_client_update_for_period(test, attested_period).data, + ) == bids_a[-1] else: assert signature_period == attested_period + 1 - assert get_lc_update_attested_block_id(get_light_client_update_for_period(test, attested_period).data) == bids_a[-2] + assert get_lc_update_attested_block_id( + get_light_client_update_for_period(test, attested_period).data, + ) == bids_a[-2] assert get_light_client_update_for_period(test, signature_period).spec is None assert get_lc_update_attested_block_id(get_light_client_finality_update(test).data) == bids_a[-1] assert get_lc_update_attested_block_id(get_light_client_optimistic_update(test).data) == bids_a[-1] From 531a0b08862d3c7b937802e07479b2d4dc8764bb Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 27 Nov 2024 15:43:26 +0100 Subject: [PATCH 16/20] Fix module list --- tests/generators/light_client/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index a5775b1cbe..a6174b277d 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -12,7 +12,6 @@ bellatrix_mods = altair_mods _new_capella_mods = {key: 'eth2spec.test.capella.light_client.test_' + key for key in [ - 'data_collection', 'single_merkle_proof', 'sync', ]} From 12401a5be5867b7fe219a27954e5690a5bc5439e Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 28 Nov 2024 13:02:12 +0100 Subject: [PATCH 17/20] Move fork tests to origin rather than destination to fix issues --- .../test/altair/light_client/test_sync.py | 56 ++++++++++++++++++- .../light_client/__init__.py | 0 .../light_client/test_data_collection.py | 0 .../light_client/test_sync.py | 42 ++++++-------- .../light_client/test_data_collection.py | 0 .../test/capella/light_client/test_sync.py | 26 +++++---- tests/core/pyspec/eth2spec/test/context.py | 9 +++ .../test/deneb/light_client/test_sync.py | 36 ++---------- .../test/helpers/light_client_sync.py | 22 -------- tests/generators/light_client/main.py | 15 +++-- 10 files changed, 108 insertions(+), 98 deletions(-) rename tests/core/pyspec/eth2spec/test/{electra => bellatrix}/light_client/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/{deneb => bellatrix}/light_client/test_data_collection.py (100%) rename tests/core/pyspec/eth2spec/test/{electra => bellatrix}/light_client/test_sync.py (55%) rename tests/core/pyspec/eth2spec/test/{electra => capella}/light_client/test_data_collection.py (100%) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index 8000ceb799..1c77e648ab 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -1,13 +1,18 @@ from eth2spec.test.context import ( spec_state_test_with_matching_config, - with_presets, + spec_test, + with_all_phases_to, with_light_client, + with_matching_spec_config, + with_presets, + with_state, ) from eth2spec.test.helpers.attestations import ( next_slots_with_attestations, state_transition_with_full_block, ) from eth2spec.test.helpers.constants import ( + CAPELLA, DENEB, ELECTRA, MINIMAL, ) from eth2spec.test.helpers.light_client import ( @@ -352,3 +357,52 @@ def test_advance_finality_without_sync_committee(spec, state): # Finish test yield from finish_lc_sync_test(test) + + +def run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, fork): + # Start test (Legacy bootstrap with an upgraded store) + test = yield from setup_lc_sync_test(spec, state, phases[fork], phases) + + # Initial `LightClientUpdate` (check that the upgraded store can process it) + finalized_block = spec.SignedBeaconBlock() + finalized_block.message.state_root = state.hash_tree_root() + finalized_state = state.copy() + attested_block = state_transition_with_full_block(spec, state, True, True) + attested_state = state.copy() + sync_aggregate, _ = get_sync_aggregate(spec, state) + block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) + yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) + assert test.store.finalized_header.beacon.slot == finalized_state.slot + assert test.store.next_sync_committee == finalized_state.next_sync_committee + assert test.store.best_valid_update is None + assert test.store.optimistic_header.beacon.slot == attested_state.slot + + # Finish test + yield from finish_lc_sync_test(test) + + +@with_all_phases_to(CAPELLA, other_phases=[CAPELLA]) +@spec_test +@with_state +@with_matching_spec_config(emitted_fork=CAPELLA) +@with_presets([MINIMAL], reason="too slow") +def test_capella_store_with_legacy_data(spec, phases, state): + yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, CAPELLA) + + +@with_all_phases_to(DENEB, other_phases=[CAPELLA, DENEB]) +@spec_test +@with_state +@with_matching_spec_config(emitted_fork=DENEB) +@with_presets([MINIMAL], reason="too slow") +def test_deneb_store_with_legacy_data(spec, phases, state): + yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, DENEB) + + +@with_all_phases_to(ELECTRA, other_phases=[CAPELLA, DENEB, ELECTRA]) +@spec_test +@with_state +@with_matching_spec_config(emitted_fork=ELECTRA) +@with_presets([MINIMAL], reason="too slow") +def test_electra_store_with_legacy_data(spec, phases, state): + yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, ELECTRA) diff --git a/tests/core/pyspec/eth2spec/test/electra/light_client/__init__.py b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/electra/light_client/__init__.py rename to tests/core/pyspec/eth2spec/test/bellatrix/light_client/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/deneb/light_client/test_data_collection.py rename to tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_data_collection.py diff --git a/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_sync.py similarity index 55% rename from tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py rename to tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_sync.py index c37e8b21e1..81b44d8749 100644 --- a/tests/core/pyspec/eth2spec/test/electra/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/light_client/test_sync.py @@ -7,59 +7,49 @@ with_state, ) from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA, + BELLATRIX, CAPELLA, DENEB, ELECTRA, MINIMAL, ) from eth2spec.test.helpers.light_client_sync import ( run_lc_sync_test_multi_fork, run_lc_sync_test_single_fork, - run_lc_sync_test_upgraded_store_with_legacy_data, ) -@with_phases(phases=[DENEB], other_phases=[ELECTRA]) +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) @spec_test @with_config_overrides({ - 'ELECTRA_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 }, emit=False) @with_state -@with_matching_spec_config(emitted_fork=ELECTRA) +@with_matching_spec_config(emitted_fork=CAPELLA) @with_presets([MINIMAL], reason="too slow") -def test_electra_fork(spec, phases, state): - yield from run_lc_sync_test_single_fork(spec, phases, state, ELECTRA) +def test_capella_fork(spec, phases, state): + yield from run_lc_sync_test_single_fork(spec, phases, state, CAPELLA) -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB, ELECTRA]) +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) @spec_test @with_config_overrides({ 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 'DENEB_FORK_EPOCH': 4, - 'ELECTRA_FORK_EPOCH': 5, }, emit=False) @with_state -@with_matching_spec_config(emitted_fork=ELECTRA) +@with_matching_spec_config(emitted_fork=DENEB) @with_presets([MINIMAL], reason="too slow") -def test_capella_electra_fork(spec, phases, state): - yield from run_lc_sync_test_multi_fork(spec, phases, state, CAPELLA, ELECTRA) +def test_capella_deneb_fork(spec, phases, state): + yield from run_lc_sync_test_multi_fork(spec, phases, state, CAPELLA, DENEB) -@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB, ELECTRA]) @spec_test @with_config_overrides({ - 'DENEB_FORK_EPOCH': 3, # Test setup advances to epoch 2 - 'ELECTRA_FORK_EPOCH': 4, + 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'DENEB_FORK_EPOCH': 4, + 'ELECTRA_FORK_EPOCH': 5, }, emit=False) @with_state @with_matching_spec_config(emitted_fork=ELECTRA) @with_presets([MINIMAL], reason="too slow") -def test_deneb_electra_fork(spec, phases, state): - yield from run_lc_sync_test_multi_fork(spec, phases, state, DENEB, ELECTRA) - - -@with_phases(phases=[ALTAIR, BELLATRIX, CAPELLA, DENEB], other_phases=[CAPELLA, DENEB, ELECTRA]) -@spec_test -@with_state -@with_matching_spec_config(emitted_fork=ELECTRA) -@with_presets([MINIMAL], reason="too slow") -def test_electra_store_with_legacy_data(spec, phases, state): - yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, ELECTRA) +def test_capella_electra_fork(spec, phases, state): + yield from run_lc_sync_test_multi_fork(spec, phases, state, CAPELLA, ELECTRA) diff --git a/tests/core/pyspec/eth2spec/test/electra/light_client/test_data_collection.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/electra/light_client/test_data_collection.py rename to tests/core/pyspec/eth2spec/test/capella/light_client/test_data_collection.py diff --git a/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py index 99a56f96e0..faa727d6d2 100644 --- a/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/capella/light_client/test_sync.py @@ -7,31 +7,35 @@ with_state, ) from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, CAPELLA, + CAPELLA, DENEB, ELECTRA, MINIMAL, ) from eth2spec.test.helpers.light_client_sync import ( + run_lc_sync_test_multi_fork, run_lc_sync_test_single_fork, - run_lc_sync_test_upgraded_store_with_legacy_data, ) -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@with_phases(phases=[CAPELLA], other_phases=[DENEB]) @spec_test @with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'DENEB_FORK_EPOCH': 3, # Test setup advances to epoch 2 }, emit=False) @with_state -@with_matching_spec_config(emitted_fork=CAPELLA) +@with_matching_spec_config(emitted_fork=DENEB) @with_presets([MINIMAL], reason="too slow") -def test_capella_fork(spec, phases, state): - yield from run_lc_sync_test_single_fork(spec, phases, state, CAPELLA) +def test_deneb_fork(spec, phases, state): + yield from run_lc_sync_test_single_fork(spec, phases, state, DENEB) -@with_phases(phases=[ALTAIR, BELLATRIX], other_phases=[CAPELLA]) +@with_phases(phases=[CAPELLA], other_phases=[DENEB, ELECTRA]) @spec_test +@with_config_overrides({ + 'DENEB_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'ELECTRA_FORK_EPOCH': 4, +}, emit=False) @with_state -@with_matching_spec_config(emitted_fork=CAPELLA) +@with_matching_spec_config(emitted_fork=ELECTRA) @with_presets([MINIMAL], reason="too slow") -def test_capella_store_with_legacy_data(spec, phases, state): - yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, CAPELLA) +def test_deneb_electra_fork(spec, phases, state): + yield from run_lc_sync_test_multi_fork(spec, phases, state, DENEB, ELECTRA) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 8b2e8de6d3..f2298d297b 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -436,6 +436,15 @@ def with_all_phases_from_except(earliest_phase, except_phases=None): return with_all_phases_from(earliest_phase, [phase for phase in ALL_PHASES if phase not in except_phases]) +def with_all_phases_to(next_phase, all_phases=ALL_PHASES): + """ + A decorator factory for running a tests with every phase except the ones listed + """ + def decorator(fn): + return with_phases([phase for phase in all_phases if is_post_fork(next_phase, phase)])(fn) + return decorator + + def with_all_phases_except(exclusion_phases): """ A decorator factory for running a tests with every phase except the ones listed diff --git a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py index 45a8ff2c8f..2a2b4db118 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/deneb/light_client/test_sync.py @@ -7,45 +7,21 @@ with_state, ) from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, CAPELLA, DENEB, + DENEB, ELECTRA, MINIMAL, ) from eth2spec.test.helpers.light_client_sync import ( - run_lc_sync_test_multi_fork, run_lc_sync_test_single_fork, - run_lc_sync_test_upgraded_store_with_legacy_data, ) -@with_phases(phases=[CAPELLA], other_phases=[DENEB]) +@with_phases(phases=[DENEB], other_phases=[ELECTRA]) @spec_test @with_config_overrides({ - 'DENEB_FORK_EPOCH': 3, # Test setup advances to epoch 2 + 'ELECTRA_FORK_EPOCH': 3, # Test setup advances to epoch 2 }, emit=False) @with_state -@with_matching_spec_config(emitted_fork=DENEB) +@with_matching_spec_config(emitted_fork=ELECTRA) @with_presets([MINIMAL], reason="too slow") -def test_deneb_fork(spec, phases, state): - yield from run_lc_sync_test_single_fork(spec, phases, state, DENEB) - - -@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_config_overrides({ - 'CAPELLA_FORK_EPOCH': 3, # Test setup advances to epoch 2 - 'DENEB_FORK_EPOCH': 4, -}, emit=False) -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_capella_deneb_fork(spec, phases, state): - yield from run_lc_sync_test_multi_fork(spec, phases, state, CAPELLA, DENEB) - - -@with_phases(phases=[ALTAIR, BELLATRIX, CAPELLA], other_phases=[CAPELLA, DENEB]) -@spec_test -@with_state -@with_matching_spec_config(emitted_fork=DENEB) -@with_presets([MINIMAL], reason="too slow") -def test_deneb_store_with_legacy_data(spec, phases, state): - yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, DENEB) +def test_electra_fork(spec, phases, state): + yield from run_lc_sync_test_single_fork(spec, phases, state, ELECTRA) diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py b/tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py index e64b0a2eca..54a5c0f970 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client_sync.py @@ -318,25 +318,3 @@ def run_lc_sync_test_multi_fork(spec, phases, state, fork_1, fork_2): # Finish test yield from finish_lc_sync_test(test) - - -def run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, fork): - # Start test (Legacy bootstrap with an upgraded store) - test = yield from setup_lc_sync_test(spec, state, phases[fork], phases) - - # Initial `LightClientUpdate` (check that the upgraded store can process it) - finalized_block = spec.SignedBeaconBlock() - finalized_block.message.state_root = state.hash_tree_root() - finalized_state = state.copy() - attested_block = state_transition_with_full_block(spec, state, True, True) - attested_state = state.copy() - sync_aggregate, _ = get_sync_aggregate(spec, state) - block = state_transition_with_full_block(spec, state, True, True, sync_aggregate=sync_aggregate) - yield from emit_update(test, spec, state, block, attested_state, attested_block, finalized_block, phases=phases) - assert test.store.finalized_header.beacon.slot == finalized_state.slot - assert test.store.next_sync_committee == finalized_state.next_sync_committee - assert test.store.best_valid_update is None - assert test.store.optimistic_header.beacon.slot == attested_state.slot - - # Finish test - yield from finish_lc_sync_test(test) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index a6174b277d..e362c6b4c0 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -9,25 +9,24 @@ 'sync', 'update_ranking', ]} - bellatrix_mods = altair_mods + + _new_bellatrix_mods = {key: 'eth2spec.test.bellatrix.light_client.test_' + key for key in [ + 'data_collection', + ]} + bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) _new_capella_mods = {key: 'eth2spec.test.capella.light_client.test_' + key for key in [ + 'data_collection', 'single_merkle_proof', 'sync', ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) _new_deneb_mods = {key: 'eth2spec.test.deneb.light_client.test_' + key for key in [ - 'data_collection', 'sync', ]} deneb_mods = combine_mods(_new_deneb_mods, capella_mods) - - _new_electra_mods = {key: 'eth2spec.test.electra.light_client.test_' + key for key in [ - 'data_collection', - 'sync', - ]} - electra_mods = combine_mods(_new_electra_mods, deneb_mods) + electra_mods = deneb_mods all_mods = { ALTAIR: altair_mods, From 30bed615ffde18429dc349ee07b7fbcc715b9a79 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 28 Nov 2024 13:06:19 +0100 Subject: [PATCH 18/20] Add missing mod --- tests/generators/light_client/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index e362c6b4c0..6420382240 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -12,6 +12,7 @@ _new_bellatrix_mods = {key: 'eth2spec.test.bellatrix.light_client.test_' + key for key in [ 'data_collection', + 'sync', ]} bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) From a52a82c11e1e8ab544b5ecbb4be7297ed6b3a164 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 28 Nov 2024 14:36:58 +0100 Subject: [PATCH 19/20] Extend decorator factory to support `other_phases` --- tests/core/pyspec/eth2spec/test/context.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index f2298d297b..16149bb861 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -436,12 +436,15 @@ def with_all_phases_from_except(earliest_phase, except_phases=None): return with_all_phases_from(earliest_phase, [phase for phase in ALL_PHASES if phase not in except_phases]) -def with_all_phases_to(next_phase, all_phases=ALL_PHASES): +def with_all_phases_to(next_phase, other_phases=None, all_phases=ALL_PHASES): """ - A decorator factory for running a tests with every phase except the ones listed + A decorator factory for running a tests with every phase up to and excluding the one listed """ def decorator(fn): - return with_phases([phase for phase in all_phases if is_post_fork(next_phase, phase)])(fn) + return with_phases( + [phase for phase in all_phases if is_post_fork(next_phase, phase)], + other_phases=other_phases, + )(fn) return decorator From 09e8f013105e40f487bfbd060f2a5732d9fd1ebe Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 28 Nov 2024 15:17:00 +0100 Subject: [PATCH 20/20] Make `from` -> `to` bounds explicit --- .../eth2spec/test/altair/light_client/test_sync.py | 10 +++++----- tests/core/pyspec/eth2spec/test/context.py | 10 +++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index 1c77e648ab..15437f0959 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -1,7 +1,7 @@ from eth2spec.test.context import ( spec_state_test_with_matching_config, spec_test, - with_all_phases_to, + with_all_phases_from_to, with_light_client, with_matching_spec_config, with_presets, @@ -12,7 +12,7 @@ state_transition_with_full_block, ) from eth2spec.test.helpers.constants import ( - CAPELLA, DENEB, ELECTRA, + ALTAIR, CAPELLA, DENEB, ELECTRA, MINIMAL, ) from eth2spec.test.helpers.light_client import ( @@ -381,7 +381,7 @@ def run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, fork): yield from finish_lc_sync_test(test) -@with_all_phases_to(CAPELLA, other_phases=[CAPELLA]) +@with_all_phases_from_to(ALTAIR, CAPELLA, other_phases=[CAPELLA]) @spec_test @with_state @with_matching_spec_config(emitted_fork=CAPELLA) @@ -390,7 +390,7 @@ def test_capella_store_with_legacy_data(spec, phases, state): yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, CAPELLA) -@with_all_phases_to(DENEB, other_phases=[CAPELLA, DENEB]) +@with_all_phases_from_to(ALTAIR, DENEB, other_phases=[CAPELLA, DENEB]) @spec_test @with_state @with_matching_spec_config(emitted_fork=DENEB) @@ -399,7 +399,7 @@ def test_deneb_store_with_legacy_data(spec, phases, state): yield from run_lc_sync_test_upgraded_store_with_legacy_data(spec, phases, state, DENEB) -@with_all_phases_to(ELECTRA, other_phases=[CAPELLA, DENEB, ELECTRA]) +@with_all_phases_from_to(ALTAIR, ELECTRA, other_phases=[CAPELLA, DENEB, ELECTRA]) @spec_test @with_state @with_matching_spec_config(emitted_fork=ELECTRA) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 16149bb861..a90190287d 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -436,13 +436,17 @@ def with_all_phases_from_except(earliest_phase, except_phases=None): return with_all_phases_from(earliest_phase, [phase for phase in ALL_PHASES if phase not in except_phases]) -def with_all_phases_to(next_phase, other_phases=None, all_phases=ALL_PHASES): +def with_all_phases_from_to(from_phase, to_phase, other_phases=None, all_phases=ALL_PHASES): """ - A decorator factory for running a tests with every phase up to and excluding the one listed + A decorator factory for running a tests with every phase + from a given start phase up to and excluding a given end phase """ def decorator(fn): return with_phases( - [phase for phase in all_phases if is_post_fork(next_phase, phase)], + [phase for phase in all_phases if ( + phase != to_phase and is_post_fork(to_phase, phase) + and is_post_fork(phase, from_phase) + )], other_phases=other_phases, )(fn) return decorator