Merge branch 'leaderboard-submissions' into leaderboard-submissions-test

.github/workflows/docs.yml

@@ -9,6 +9,21 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      # ==== python, leaderboard data ====
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Format leaderboard results for web
+        run: python leaderboard-submissions/webfmt.py
+
       # ==== vue ====
       - name: Set up Node.js
         uses: actions/setup-node@v4
@@ -29,17 +44,6 @@ jobs:
         run: cp -R leaderboard/dist/. docs/_extra/leaderboard
 
       # ==== sphinx ====
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-
       - name: Build docs
         working-directory: ./docs
         run: sphinx-build -T -E -W --keep-going -b html "." "_build/html"

leaderboard-submissions/gh-print-new-results.py

@@ -12,13 +12,14 @@
     with open(fp) as f:
         results = json.load(f)
 
+    metadata = results["metadata"]
     cb = results["closedbook"]
     ob = results["openbook"]
     ep = results["evidenceprovided"]
 
     print(
-        f"# {results['name']}\n"
-        f"*[{results['citation']}]({results['url']})*\n\n"
+        f"# {metadata['name']}\n"
+        f"*[{metadata['citation']}]({metadata['url']})*\n\n"
         "## Closed Book\n\n"
         f"- **Loose**: {cb['acc']['loose']:.3}\n"
         f"- **Strict**: {cb['acc']['strict']:.3}\n"

leaderboard-submissions/hydrate.py

@@ -6,8 +6,8 @@
 from pathlib import Path
 from typing import List, Literal, Optional
 
-import fanoutqa
-from fanoutqa.eval import evaluate
+# import fanoutqa
+# from fanoutqa.eval import evaluate
 
 # prevent manipulation of results - the results must be generated by this script or else the hash will not match
 LEADERBOARD_SALT = os.getenv("LEADERBOARD_SALT", "supersecret").encode()
@@ -141,14 +141,14 @@ def eval_submission(metadata_fp: Path, check_result: CheckResult):
     """Read in the answers and generations and eval them all, then write the results file."""
     # TODO test remove me and uncomment below
     dummy = {
-        "acc": {"loose": 0., "strict": 0.},
+        "acc": {"loose": 0.0, "strict": 0.0},
         "rouge": {
-            "rouge1": {"precision": 0., "recall": 0., "fscore": 0.},
-            "rouge2": {"precision": 0., "recall": 0., "fscore": 0.},
-            "rougeL": {"precision": 0., "recall": 0., "fscore": 0.},
+            "rouge1": {"precision": 0.0, "recall": 0.0, "fscore": 0.0},
+            "rouge2": {"precision": 0.0, "recall": 0.0, "fscore": 0.0},
+            "rougeL": {"precision": 0.0, "recall": 0.0, "fscore": 0.0},
         },
-        "bleurt": 0.,
-        "gpt": 0.,
+        "bleurt": 0.0,
+        "gpt": 0.0,
     }
     closedbook_results = openbook_results = evidenceprovided_results = dummy
 
@@ -168,7 +168,7 @@ def eval_submission(metadata_fp: Path, check_result: CheckResult):
 
     # hash the results to prevent score manipulation
     results_hash = hashlib.sha256()
-    results_hash.update(check_result.submission_hash.digest())
+    results_hash.update(check_result.submission_hash.hexdigest().encode())
     results_hash.update(str(closedbook_results).encode())
     results_hash.update(str(openbook_results).encode())
     results_hash.update(str(evidenceprovided_results).encode())
@@ -180,12 +180,14 @@ def eval_submission(metadata_fp: Path, check_result: CheckResult):
         # SHA256(_submission_hash, cb_results, ob_results, ep_results, salt)
         "_results_hash": results_hash.hexdigest(),
         # metadata
-        "name": check_result.metadata.name,
-        "authors": check_result.metadata.authors,
-        "url": check_result.metadata.url,
-        "citation": check_result.metadata.citation,
-        "type": check_result.metadata.type,
-        "context": check_result.metadata.context,
+        "metadata": {
+            "name": check_result.metadata.name,
+            "authors": check_result.metadata.authors,
+            "url": check_result.metadata.url,
+            "citation": check_result.metadata.citation,
+            "type": check_result.metadata.type,
+            "context": check_result.metadata.context,
+        },
         # results
         "closedbook": closedbook_results,
         "openbook": openbook_results,

leaderboard-submissions/webfmt.py

@@ -0,0 +1,39 @@
+"""Copy the results in leaderboard-submissions/results/ to leaderboard/src/data in the expected format for the web."""
+
+import json
+from pathlib import Path
+
+SUBMISSIONS_ROOT = Path(__file__).parent
+REPO_ROOT = SUBMISSIONS_ROOT.parent
+METADATA_PATH = SUBMISSIONS_ROOT / "metadata"
+RESULTS_PATH = SUBMISSIONS_ROOT / "results"
+WEB_DATA_PATH = REPO_ROOT / "leaderboard/src/data"
+
+
+def main():
+    results_closed = []
+    results_open = []
+    results_provided = []
+
+    for fp in RESULTS_PATH.glob("*.json"):
+        with open(fp) as f:
+            result = json.load(f)
+
+        cb = {**result["metadata"], **result["closedbook"]}
+        ob = {**result["metadata"], **result["openbook"]}
+        ep = {**result["metadata"], **result["evidenceprovided"]}
+
+        results_closed.append(cb)
+        results_open.append(ob)
+        results_provided.append(ep)
+
+    with open(WEB_DATA_PATH / "web-closedbook.json", "w") as f:
+        json.dump(results_closed, f, indent=2)
+    with open(WEB_DATA_PATH / "web-openbook.json", "w") as f:
+        json.dump(results_open, f, indent=2)
+    with open(WEB_DATA_PATH / "web-wiki-provided.json", "w") as f:
+        json.dump(results_provided, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()