From d7b6831ceb071195f4c4612ee96babef1be82410 Mon Sep 17 00:00:00 2001 From: mina-parham Date: Sat, 25 Jan 2025 04:24:29 -0500 Subject: [PATCH 1/3] Updated ClimateFEVER dataset with new version --- .../Retrieval/eng/ClimateFEVERRetrieval.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py index d60b7a3817..12c8ce9f6c 100644 --- a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py @@ -72,3 +72,39 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval): primaryClass={cs.CL} }""", ) + + +class ClimateFEVERv2(AbsTaskRetrieval): + metadata = TaskMetadata( + name="ClimateFEVER.v2", + description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. ", + reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html", + dataset={ + "path": "Mina76/climate-fever-v2", + "revision": "9dd886701f63b066f5d6b82cd061a6e13befdbd4", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=None, + sample_creation=None, + bibtex_citation="""@misc{diggelmann2021climatefever, + title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, + author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, + year={2021}, + eprint={2012.00614}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", + prompt={ + "query": "Given a claim about climate change, retrieve documents that support or refute the claim" + }, + ) From 42634aeb558fafd99035e8bc1afcc1324474f35a Mon Sep 17 00:00:00 2001 From: mina-parham Date: Sat, 25 Jan 2025 16:19:12 -0500 Subject: [PATCH 2/3] Adds Fill in the empty metadata. --- .../Retrieval/eng/ClimateFEVERRetrieval.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py index 12c8ce9f6c..06d5e91ab8 100644 --- a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py @@ -80,8 +80,8 @@ class ClimateFEVERv2(AbsTaskRetrieval): description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. ", reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html", dataset={ - "path": "Mina76/climate-fever-v2", - "revision": "9dd886701f63b066f5d6b82cd061a6e13befdbd4", + "path": "Mina76/climate-fever", + "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4", }, type="Retrieval", category="s2p", @@ -89,13 +89,13 @@ class ClimateFEVERv2(AbsTaskRetrieval): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", - date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=("2020-12-11"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@misc{diggelmann2021climatefever, title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, From 5511f499abfe99d77214f6ffd1a67ab249e386bc Mon Sep 17 00:00:00 2001 From: mina-parham Date: Sat, 25 Jan 2025 16:39:29 -0500 Subject: [PATCH 3/3] Updates the date tuple --- mteb/models/gme_models.py | 1 - mteb/models/lens_models.py | 8 +------- mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py | 2 +- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/mteb/models/gme_models.py b/mteb/models/gme_models.py index 53476403bc..804dfbc84d 100644 --- a/mteb/models/gme_models.py +++ b/mteb/models/gme_models.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from functools import partial from mteb.model_meta import ModelMeta diff --git a/mteb/models/lens_models.py b/mteb/models/lens_models.py index 2fe54b26a1..380724e53e 100644 --- a/mteb/models/lens_models.py +++ b/mteb/models/lens_models.py @@ -1,12 +1,6 @@ from __future__ import annotations -from functools import partial - -import torch - -from mteb.encoder_interface import PromptType -from mteb.model_meta import ModelMeta, sentence_transformers_loader -from mteb.models.instruct_wrapper import instruct_wrapper +from mteb.model_meta import ModelMeta from .bge_models import bge_full_data diff --git a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py index 06d5e91ab8..92a489c016 100644 --- a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py @@ -89,7 +89,7 @@ class ClimateFEVERv2(AbsTaskRetrieval): eval_splits=["test"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", - date=("2020-12-11"), + date=("2020-12-11", "2020-12-11"), domains=["Academic"], task_subtypes=["Question answering"], license="cc-by-sa-4.0",