Migrate beacon state (#8833)

* migrate beacon state

* fix historical summary schema

* fix ref test compilation

* fix ref tests

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/altair/fork/ForkUpgradeTestExecutor.java

@@ -14,6 +14,7 @@
 package tech.pegasys.teku.reference.altair.fork;
 
 import static tech.pegasys.teku.infrastructure.ssz.SszDataAssert.assertThatSszData;
+import static tech.pegasys.teku.spec.SpecMilestone.BELLATRIX;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableMap;
@@ -44,14 +45,26 @@ public void runTest(final TestDefinition testDefinition) throws Throwable {
   }
 
   private void processUpgrade(final TestDefinition testDefinition, final SpecMilestone milestone) {
-    final SpecVersion spec = testDefinition.getSpec().getGenesisSpec();
+    final SpecMilestone previousMilestone = milestone.getPreviousMilestone();
+    final SpecVersion previousMilestoneSpecVersion =
+        testDefinition.getSpec().forMilestone(previousMilestone);
     final BeaconStateSchema<?, ?> fromMilestoneSchema =
         switch (milestone) {
-          case ALTAIR -> BeaconStateSchemaPhase0.create(spec.getConfig());
-          case BELLATRIX -> BeaconStateSchemaAltair.create(spec.getConfig());
-          case CAPELLA -> BeaconStateSchemaBellatrix.create(spec.getConfig());
-          case DENEB -> BeaconStateSchemaCapella.create(spec.getConfig());
-          case ELECTRA -> BeaconStateSchemaDeneb.create(spec.getConfig());
+          case ALTAIR -> BeaconStateSchemaPhase0.create(previousMilestoneSpecVersion.getConfig());
+          case BELLATRIX ->
+              BeaconStateSchemaAltair.create(previousMilestoneSpecVersion.getConfig());
+          case CAPELLA ->
+              BeaconStateSchemaBellatrix.create(
+                  previousMilestoneSpecVersion.getConfig(),
+                  previousMilestoneSpecVersion.getSchemaDefinitions().getSchemaRegistry());
+          case DENEB ->
+              BeaconStateSchemaCapella.create(
+                  previousMilestoneSpecVersion.getConfig(),
+                  previousMilestoneSpecVersion.getSchemaDefinitions().getSchemaRegistry());
+          case ELECTRA ->
+              BeaconStateSchemaDeneb.create(
+                  previousMilestoneSpecVersion.getConfig(),
+                  previousMilestoneSpecVersion.getSchemaDefinitions().getSchemaRegistry());
           default ->
               throw new IllegalStateException(
                   "Unhandled fork upgrade for test "

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/versions/bellatrix/BeaconStateSchemaBellatrix.java

@@ -14,6 +14,7 @@
 package tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.bellatrix;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.EXECUTION_PAYLOAD_HEADER_SCHEMA;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.List;
@@ -24,37 +25,37 @@
 import tech.pegasys.teku.infrastructure.ssz.sos.SszField;
 import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode;
 import tech.pegasys.teku.spec.config.SpecConfig;
-import tech.pegasys.teku.spec.config.SpecConfigBellatrix;
 import tech.pegasys.teku.spec.datastructures.execution.versions.bellatrix.ExecutionPayloadHeaderSchemaBellatrix;
 import tech.pegasys.teku.spec.datastructures.state.SyncCommittee.SyncCommitteeSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.common.AbstractBeaconStateSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.common.BeaconStateFields;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.altair.BeaconStateSchemaAltair;
+import tech.pegasys.teku.spec.schemas.registry.SchemaRegistry;
 
 public class BeaconStateSchemaBellatrix
     extends AbstractBeaconStateSchema<BeaconStateBellatrix, MutableBeaconStateBellatrix> {
 
   public static final int LATEST_EXECUTION_PAYLOAD_HEADER_FIELD_INDEX = 24;
 
   @VisibleForTesting
-  BeaconStateSchemaBellatrix(final SpecConfig specConfig) {
-    super("BeaconStateBellatrix", getUniqueFields(specConfig), specConfig);
+  BeaconStateSchemaBellatrix(final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    super("BeaconStateBellatrix", getUniqueFields(specConfig, schemaRegistry), specConfig);
   }
 
-  public static BeaconStateSchemaBellatrix create(final SpecConfig specConfig) {
-    return new BeaconStateSchemaBellatrix(specConfig);
+  public static BeaconStateSchemaBellatrix create(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    return new BeaconStateSchemaBellatrix(specConfig, schemaRegistry);
   }
 
-  public static List<SszField> getUniqueFields(final SpecConfig specConfig) {
+  public static List<SszField> getUniqueFields(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
     final List<SszField> newFields =
         List.of(
             new SszField(
                 LATEST_EXECUTION_PAYLOAD_HEADER_FIELD_INDEX,
                 BeaconStateFields.LATEST_EXECUTION_PAYLOAD_HEADER,
-                () ->
-                    new ExecutionPayloadHeaderSchemaBellatrix(
-                        SpecConfigBellatrix.required(specConfig))));
+                () -> schemaRegistry.get(EXECUTION_PAYLOAD_HEADER_SCHEMA)));
 
     return Stream.concat(
             BeaconStateSchemaAltair.getUniqueFields(specConfig).stream(), newFields.stream())

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/versions/capella/BeaconStateSchemaCapella.java

@@ -14,7 +14,7 @@
 package tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.capella;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.bellatrix.BeaconStateSchemaBellatrix.LATEST_EXECUTION_PAYLOAD_HEADER_FIELD_INDEX;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.HISTORICAL_SUMMARIES_SCHEMA;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.List;
@@ -27,14 +27,14 @@
 import tech.pegasys.teku.infrastructure.ssz.sos.SszField;
 import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode;
 import tech.pegasys.teku.spec.config.SpecConfig;
-import tech.pegasys.teku.spec.config.SpecConfigCapella;
 import tech.pegasys.teku.spec.datastructures.execution.versions.capella.ExecutionPayloadHeaderSchemaCapella;
 import tech.pegasys.teku.spec.datastructures.state.SyncCommittee;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.common.AbstractBeaconStateSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.common.BeaconStateFields;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.bellatrix.BeaconStateSchemaBellatrix;
 import tech.pegasys.teku.spec.datastructures.state.versions.capella.HistoricalSummary;
+import tech.pegasys.teku.spec.schemas.registry.SchemaRegistry;
 
 public class BeaconStateSchemaCapella
     extends AbstractBeaconStateSchema<BeaconStateCapella, MutableBeaconStateCapella> {
@@ -43,23 +43,12 @@ public class BeaconStateSchemaCapella
   public static final int HISTORICAL_SUMMARIES_FIELD_INDEX = 27;
 
   @VisibleForTesting
-  BeaconStateSchemaCapella(final SpecConfig specConfig) {
-    super("BeaconStateCapella", getUniqueFields(specConfig), specConfig);
+  BeaconStateSchemaCapella(final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    super("BeaconStateCapella", getUniqueFields(specConfig, schemaRegistry), specConfig);
   }
 
-  public static List<SszField> getUniqueFields(final SpecConfig specConfig) {
-    final HistoricalSummary.HistoricalSummarySchema historicalSummarySchema =
-        new HistoricalSummary.HistoricalSummarySchema();
-
-    final List<SszField> updatedFields =
-        List.of(
-            new SszField(
-                LATEST_EXECUTION_PAYLOAD_HEADER_FIELD_INDEX,
-                BeaconStateFields.LATEST_EXECUTION_PAYLOAD_HEADER,
-                () ->
-                    new ExecutionPayloadHeaderSchemaCapella(
-                        SpecConfigCapella.required(specConfig))));
-
+  public static List<SszField> getUniqueFields(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
     final List<SszField> newFields =
         List.of(
             new SszField(
@@ -73,18 +62,11 @@ public static List<SszField> getUniqueFields(final SpecConfig specConfig) {
             new SszField(
                 HISTORICAL_SUMMARIES_FIELD_INDEX,
                 BeaconStateFields.HISTORICAL_SUMMARIES,
-                () ->
-                    SszListSchema.create(
-                        historicalSummarySchema, specConfig.getHistoricalRootsLimit())));
+                () -> schemaRegistry.get(HISTORICAL_SUMMARIES_SCHEMA)));
 
     return Stream.concat(
-            BeaconStateSchemaBellatrix.getUniqueFields(specConfig).stream(), newFields.stream())
-        .map(
-            field ->
-                updatedFields.stream()
-                    .filter(updatedField -> updatedField.getIndex() == field.getIndex())
-                    .findFirst()
-                    .orElse(field))
+            BeaconStateSchemaBellatrix.getUniqueFields(specConfig, schemaRegistry).stream(),
+            newFields.stream())
         .toList();
   }
 
@@ -126,8 +108,9 @@ public MutableBeaconStateCapella createBuilder() {
     return new MutableBeaconStateCapellaImpl(createEmptyBeaconStateImpl(), true);
   }
 
-  public static BeaconStateSchemaCapella create(final SpecConfig specConfig) {
-    return new BeaconStateSchemaCapella(specConfig);
+  public static BeaconStateSchemaCapella create(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    return new BeaconStateSchemaCapella(specConfig, schemaRegistry);
   }
 
   public static BeaconStateSchemaCapella required(final BeaconStateSchema<?, ?> schema) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/versions/deneb/BeaconStateSchemaDeneb.java

@@ -14,7 +14,6 @@
 package tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.deneb;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.bellatrix.BeaconStateSchemaBellatrix.LATEST_EXECUTION_PAYLOAD_HEADER_FIELD_INDEX;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.List;
@@ -25,39 +24,26 @@
 import tech.pegasys.teku.infrastructure.ssz.sos.SszField;
 import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode;
 import tech.pegasys.teku.spec.config.SpecConfig;
-import tech.pegasys.teku.spec.config.SpecConfigDeneb;
 import tech.pegasys.teku.spec.datastructures.execution.versions.deneb.ExecutionPayloadHeaderSchemaDeneb;
 import tech.pegasys.teku.spec.datastructures.state.SyncCommittee;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.common.AbstractBeaconStateSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.common.BeaconStateFields;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.capella.BeaconStateSchemaCapella;
 import tech.pegasys.teku.spec.datastructures.state.versions.capella.HistoricalSummary;
+import tech.pegasys.teku.spec.schemas.registry.SchemaRegistry;
 
 public class BeaconStateSchemaDeneb
     extends AbstractBeaconStateSchema<BeaconStateDeneb, MutableBeaconStateDeneb> {
 
   @VisibleForTesting
-  BeaconStateSchemaDeneb(final SpecConfig specConfig) {
-    super("BeaconStateDeneb", getUniqueFields(specConfig), specConfig);
+  BeaconStateSchemaDeneb(final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    super("BeaconStateDeneb", getUniqueFields(specConfig, schemaRegistry), specConfig);
   }
 
-  public static List<SszField> getUniqueFields(final SpecConfig specConfig) {
-    final List<SszField> updatedFields =
-        List.of(
-            new SszField(
-                LATEST_EXECUTION_PAYLOAD_HEADER_FIELD_INDEX,
-                BeaconStateFields.LATEST_EXECUTION_PAYLOAD_HEADER,
-                () -> new ExecutionPayloadHeaderSchemaDeneb(SpecConfigDeneb.required(specConfig))));
-
-    return BeaconStateSchemaCapella.getUniqueFields(specConfig).stream()
-        .map(
-            field ->
-                updatedFields.stream()
-                    .filter(updatedField -> updatedField.getIndex() == field.getIndex())
-                    .findFirst()
-                    .orElse(field))
-        .toList();
+  public static List<SszField> getUniqueFields(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    return BeaconStateSchemaCapella.getUniqueFields(specConfig, schemaRegistry).stream().toList();
   }
 
   @SuppressWarnings("unchecked")
@@ -92,8 +78,9 @@ public MutableBeaconStateDeneb createBuilder() {
     return new MutableBeaconStateDenebImpl(createEmptyBeaconStateImpl(), true);
   }
 
-  public static BeaconStateSchemaDeneb create(final SpecConfig specConfig) {
-    return new BeaconStateSchemaDeneb(specConfig);
+  public static BeaconStateSchemaDeneb create(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    return new BeaconStateSchemaDeneb(specConfig, schemaRegistry);
   }
 
   public static BeaconStateSchemaDeneb required(final BeaconStateSchema<?, ?> schema) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/versions/electra/BeaconStateSchemaElectra.java

@@ -14,6 +14,9 @@
 package tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.electra;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.PENDING_CONSOLIDATIONS_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.PENDING_DEPOSITS_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.PENDING_PARTIAL_WITHDRAWALS_SCHEMA;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.List;
@@ -26,7 +29,6 @@
 import tech.pegasys.teku.infrastructure.ssz.sos.SszField;
 import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode;
 import tech.pegasys.teku.spec.config.SpecConfig;
-import tech.pegasys.teku.spec.config.SpecConfigElectra;
 import tech.pegasys.teku.spec.datastructures.execution.versions.deneb.ExecutionPayloadHeaderSchemaDeneb;
 import tech.pegasys.teku.spec.datastructures.state.SyncCommittee;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateSchema;
@@ -37,6 +39,7 @@
 import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingConsolidation;
 import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingDeposit;
 import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingPartialWithdrawal;
+import tech.pegasys.teku.spec.schemas.registry.SchemaRegistry;
 
 public class BeaconStateSchemaElectra
     extends AbstractBeaconStateSchema<BeaconStateElectra, MutableBeaconStateElectra> {
@@ -51,19 +54,12 @@ public class BeaconStateSchemaElectra
   public static final int PENDING_CONSOLIDATIONS_FIELD_INDEX = 36;
 
   @VisibleForTesting
-  BeaconStateSchemaElectra(final SpecConfig specConfig) {
-    super("BeaconStateElectra", getUniqueFields(specConfig), specConfig);
+  BeaconStateSchemaElectra(final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    super("BeaconStateElectra", getUniqueFields(specConfig, schemaRegistry), specConfig);
   }
 
-  private static List<SszField> getUniqueFields(final SpecConfig specConfig) {
-    final PendingDeposit.PendingDepositSchema pendingDepositSchema =
-        new PendingDeposit.PendingDepositSchema();
-    final PendingPartialWithdrawal.PendingPartialWithdrawalSchema pendingPartialWithdrawalSchema =
-        new PendingPartialWithdrawal.PendingPartialWithdrawalSchema();
-    final SpecConfigElectra specConfigElectra = SpecConfigElectra.required(specConfig);
-    final PendingConsolidation.PendingConsolidationSchema pendingConsolidationSchema =
-        new PendingConsolidation.PendingConsolidationSchema();
-
+  private static List<SszField> getUniqueFields(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
     final List<SszField> newFields =
         List.of(
             new SszField(
@@ -93,26 +89,19 @@ private static List<SszField> getUniqueFields(final SpecConfig specConfig) {
             new SszField(
                 PENDING_DEPOSITS_FIELD_INDEX,
                 BeaconStateFields.PENDING_DEPOSITS,
-                () ->
-                    SszListSchema.create(
-                        pendingDepositSchema, specConfigElectra.getPendingDepositsLimit())),
+                () -> schemaRegistry.get(PENDING_DEPOSITS_SCHEMA)),
             new SszField(
                 PENDING_PARTIAL_WITHDRAWALS_FIELD_INDEX,
                 BeaconStateFields.PENDING_PARTIAL_WITHDRAWALS,
-                () ->
-                    SszListSchema.create(
-                        pendingPartialWithdrawalSchema,
-                        specConfigElectra.getPendingPartialWithdrawalsLimit())),
+                () -> schemaRegistry.get(PENDING_PARTIAL_WITHDRAWALS_SCHEMA)),
             new SszField(
                 PENDING_CONSOLIDATIONS_FIELD_INDEX,
                 BeaconStateFields.PENDING_CONSOLIDATIONS,
-                () ->
-                    SszListSchema.create(
-                        pendingConsolidationSchema,
-                        specConfigElectra.getPendingConsolidationsLimit())));
+                () -> schemaRegistry.get(PENDING_CONSOLIDATIONS_SCHEMA)));
 
     return Stream.concat(
-            BeaconStateSchemaDeneb.getUniqueFields(specConfig).stream(), newFields.stream())
+            BeaconStateSchemaDeneb.getUniqueFields(specConfig, schemaRegistry).stream(),
+            newFields.stream())
         .toList();
   }
 
@@ -148,8 +137,9 @@ public MutableBeaconStateElectra createBuilder() {
     return new MutableBeaconStateElectraImpl(createEmptyBeaconStateImpl(), true);
   }
 
-  public static BeaconStateSchemaElectra create(final SpecConfig specConfig) {
-    return new BeaconStateSchemaElectra(specConfig);
+  public static BeaconStateSchemaElectra create(
+      final SpecConfig specConfig, final SchemaRegistry schemaRegistry) {
+    return new BeaconStateSchemaElectra(specConfig, schemaRegistry);
   }
 
   public static BeaconStateSchemaElectra required(final BeaconStateSchema<?, ?> schema) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/altair/forktransition/AltairStateUpgrade.java

@@ -60,9 +60,7 @@ public BeaconStateAltair upgrade(final BeaconState preState) {
     final UInt64 epoch = beaconStateAccessors.getCurrentEpoch(preState);
     final int validatorCount = preState.getValidators().size();
 
-    return schemaDefinitions
-        .getBeaconStateSchema()
-        .createEmpty()
+    return BeaconStateAltair.required(schemaDefinitions.getBeaconStateSchema().createEmpty())
         .updatedAltair(
             state -> {
               BeaconStateFields.copyCommonFieldsFromSource(state, preState);

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/bellatrix/forktransition/BellatrixStateUpgrade.java

@@ -43,9 +43,7 @@ public BeaconStateBellatrix upgrade(final BeaconState preState) {
     final UInt64 epoch = beaconStateAccessors.getCurrentEpoch(preState);
     BeaconStateAltair preStateAltair = BeaconStateAltair.required(preState);
 
-    return schemaDefinitions
-        .getBeaconStateSchema()
-        .createEmpty()
+    return BeaconStateBellatrix.required(schemaDefinitions.getBeaconStateSchema().createEmpty())
         .updatedBellatrix(
             state -> {
               BeaconStateFields.copyCommonFieldsFromSource(state, preState);

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/capella/forktransition/CapellaStateUpgrade.java

@@ -47,9 +47,7 @@ public CapellaStateUpgrade(
   public BeaconStateCapella upgrade(final BeaconState preState) {
     final UInt64 epoch = beaconStateAccessors.getCurrentEpoch(preState);
     BeaconStateBellatrix preStateBellatrix = BeaconStateBellatrix.required(preState);
-    return schemaDefinitions
-        .getBeaconStateSchema()
-        .createEmpty()
+    return BeaconStateCapella.required(schemaDefinitions.getBeaconStateSchema().createEmpty())
         .updatedCapella(
             state -> {
               BeaconStateFields.copyCommonFieldsFromSource(state, preState);

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/forktransition/DenebStateUpgrade.java

@@ -45,9 +45,7 @@ public DenebStateUpgrade(
   public BeaconStateDeneb upgrade(final BeaconState preState) {
     final UInt64 epoch = beaconStateAccessors.getCurrentEpoch(preState);
     final BeaconStateCapella preStateCapella = BeaconStateCapella.required(preState);
-    return schemaDefinitions
-        .getBeaconStateSchema()
-        .createEmpty()
+    return BeaconStateDeneb.required(schemaDefinitions.getBeaconStateSchema().createEmpty())
         .updatedDeneb(
             state -> {
               BeaconStateFields.copyCommonFieldsFromSource(state, preState);