Add some custom fc tests

sigp · Feb 9, 2023 · f35d44b · f35d44b
1 parent 0261cda
commit f35d44b
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,7 @@ perf.data*
 /bin
 genesis.ssz
 /clippy.toml
+*.DS_Store
 
 # IntelliJ
 /*.iml
diff --git a/testing/ef_tests/.gitignore b/testing/ef_tests/.gitignore
@@ -1,3 +1,4 @@
 /consensus-spec-tests
 .accessed_file_log.txt
 /bls12-381-tests
+/custom-fc-tests
diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile
@@ -2,6 +2,18 @@ TESTS_TAG := v1.2.0
 TESTS = general minimal mainnet
 TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
 
+# Custom fork choice tests from https://github.com/adiasg/consensus-specs
+#
+# Since these tests are private, we can't download them in this Makefile
+# without adding an API key into the process, so we're just downloading
+# the files manually. These files can be obtained from:
+# https://github.com/adiasg/consensus-specs/pull/18#issuecomment-1411070772
+CUSTOM_FC_TESTS_TAG = 076551b
+CUSTOM_FC_TESTS_MAINNET_ARCHIVE = $(CUSTOM_FC_TESTS_TAG)-mainnet.tar.gz
+CUSTOM_FC_TESTS_MINIMAL_ARCHIVE = $(CUSTOM_FC_TESTS_TAG)-minimal.tar.gz
+CUSTOM_FC_TESTS_REPO_NAME := custom-fc-tests
+CUSTOM_FC_TESTS_OUTPUT_DIR := ./$(CUSTOM_FC_TESTS_REPO_NAME)
+
 REPO_NAME := consensus-spec-tests
 OUTPUT_DIR := ./$(REPO_NAME)
 BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG)
@@ -33,6 +45,11 @@ $(BLS_OUTPUT_DIR):
 %-$(TESTS_TAG).tar.gz:
 	$(WGET) $(BASE_URL)/$*.tar.gz -O $@
 
+custom-fc-tests:
+	mkdir $(CUSTOM_FC_TESTS_OUTPUT_DIR)
+	tar -xzf $(CUSTOM_FC_TESTS_MAINNET_ARCHIVE) -C $(CUSTOM_FC_TESTS_OUTPUT_DIR);
+	tar -xzf $(CUSTOM_FC_TESTS_MINIMAL_ARCHIVE) -C $(CUSTOM_FC_TESTS_OUTPUT_DIR);
+
 clean-test-files:
 	rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)
 

diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs
@@ -18,6 +18,10 @@ pub trait Handler {
 
     fn handler_name(&self) -> String;
 
+    fn tests_dir(&self) -> String {
+        "consensus-spec-tests".to_string()
+    }
+
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         Self::Case::is_enabled_for_fork(fork_name)
     }
@@ -38,7 +42,7 @@ pub trait Handler {
         let fork_name_str = fork_name.to_string();
 
         let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-            .join("consensus-spec-tests")
+            .join(&self.tests_dir())
             .join("tests")
             .join(Self::config_name())
             .join(&fork_name_str)
@@ -499,13 +503,15 @@ impl<E: EthSpec + TypeName> Handler for FinalityHandler<E> {
 }
 
 pub struct ForkChoiceHandler<E> {
+    tests_dir: String,
     handler_name: String,
     _phantom: PhantomData<E>,
 }
 
 impl<E: EthSpec> ForkChoiceHandler<E> {
-    pub fn new(handler_name: &str) -> Self {
+    pub fn new(tests_dir: &str, handler_name: &str) -> Self {
         Self {
+            tests_dir: tests_dir.into(),
             handler_name: handler_name.into(),
             _phantom: PhantomData,
         }
@@ -523,6 +529,10 @@ impl<E: EthSpec + TypeName> Handler for ForkChoiceHandler<E> {
         "fork_choice"
     }
 
+    fn tests_dir(&self) -> String {
+        self.tests_dir.clone()
+    }
+
     fn handler_name(&self) -> String {
         self.handler_name.clone()
     }

diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs
@@ -1,8 +1,11 @@
-#![cfg(feature = "ef_tests")]
+// #![cfg(feature = "ef_tests")]
 
 use ef_tests::*;
 use types::*;
 
+const CONSENSUS_SPEC_TESTS: &str = "consensus-spec-tests";
+const CUSTOM_FC_TESTS: &str = "custom-fc-tests";
+
 // Check that the hand-computed multiplications on EthSpec are correctly computed.
 // This test lives here because one is most likely to muck these up during a spec update.
 fn check_typenum_values<E: EthSpec>() {
@@ -424,30 +427,41 @@ fn finality() {
     FinalityHandler::<MainnetEthSpec>::default().run();
 }
 
-#[test]
-fn fork_choice_get_head() {
-    ForkChoiceHandler::<MinimalEthSpec>::new("get_head").run();
-    ForkChoiceHandler::<MainnetEthSpec>::new("get_head").run();
-}
+macro_rules! fork_choice_tests {
+    ($mod: ident, $tests_dir: ident) => {
+        mod $mod {
+            use super::*;
 
-#[test]
-fn fork_choice_on_block() {
-    ForkChoiceHandler::<MinimalEthSpec>::new("on_block").run();
-    ForkChoiceHandler::<MainnetEthSpec>::new("on_block").run();
-}
+            #[test]
+            fn fork_choice_get_head() {
+                ForkChoiceHandler::<MinimalEthSpec>::new($tests_dir, "get_head").run();
+                ForkChoiceHandler::<MainnetEthSpec>::new($tests_dir, "get_head").run();
+            }
 
-#[test]
-fn fork_choice_on_merge_block() {
-    ForkChoiceHandler::<MinimalEthSpec>::new("on_merge_block").run();
-    ForkChoiceHandler::<MainnetEthSpec>::new("on_merge_block").run();
-}
+            #[test]
+            fn fork_choice_on_block() {
+                ForkChoiceHandler::<MinimalEthSpec>::new($tests_dir, "on_block").run();
+                ForkChoiceHandler::<MainnetEthSpec>::new($tests_dir, "on_block").run();
+            }
 
-#[test]
-fn fork_choice_ex_ante() {
-    ForkChoiceHandler::<MinimalEthSpec>::new("ex_ante").run();
-    ForkChoiceHandler::<MainnetEthSpec>::new("ex_ante").run();
+            #[test]
+            fn fork_choice_on_merge_block() {
+                ForkChoiceHandler::<MinimalEthSpec>::new($tests_dir, "on_merge_block").run();
+                ForkChoiceHandler::<MainnetEthSpec>::new($tests_dir, "on_merge_block").run();
+            }
+
+            #[test]
+            fn fork_choice_ex_ante() {
+                ForkChoiceHandler::<MinimalEthSpec>::new($tests_dir, "ex_ante").run();
+                ForkChoiceHandler::<MainnetEthSpec>::new($tests_dir, "ex_ante").run();
+            }
+        }
+    };
 }
 
+fork_choice_tests!(fork_choice_standard, CONSENSUS_SPEC_TESTS);
+fork_choice_tests!(fork_choice_custom, CUSTOM_FC_TESTS);
+
 #[test]
 fn optimistic_sync() {
     OptimisticSyncHandler::<MinimalEthSpec>::default().run();