Record task start and end times for Acute-Evals (#4208)

* Record Acute-Eval task start and end times * Ignore time_taken column * Test work * Try to fix broken test * Another test * Keep testing * [WIP] Fix crowdsourcing CI check * Revert * Record Acute-Eval task start and end times * Ignore time_taken column * Test work * Try to fix broken test * Another test * Keep testing * Revert * Dummy * Just pull in upstream Mephisto change
facebookresearch · Jan 6, 2022 · d23a623 · d23a623
1 parent 2698017
commit d23a623
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 18 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -171,7 +171,7 @@ commands:
             command: |
               cd ..
               git clone [email protected]:facebookresearch/Mephisto.git Mephisto
-              cd Mephisto; git checkout v0.4.1 -b stable
+              cd Mephisto; git checkout v0.4.2 -b stable
               pip install -r requirements.txt
               python setup.py develop
               # `echo` so that ENTER will be pressed at the prompt

diff --git a/parlai/crowdsourcing/tasks/acute_eval/util.py b/parlai/crowdsourcing/tasks/acute_eval/util.py
@@ -6,7 +6,7 @@
 
 import hashlib
 import os
-from typing import Iterable, List, Tuple, Union
+from typing import Iterable, List, Optional, Tuple, Union
 
 import pandas as pd
 from pytest_regressions.data_regression import DataRegressionFixture
@@ -114,6 +114,7 @@ def test_full_csv(
             results_folder=outputs['results_folder'],
             file_suffix='full.csv',
             dataframe_regression=dataframe_regression,
+            drop_columns=['task_start', 'time_taken'],
         )
 
     def test_grid_csv(
@@ -169,11 +170,21 @@ def _check_dataframe(
         results_folder: str,
         file_suffix: str,
         dataframe_regression: DataFrameRegressionFixture,
+        drop_columns: Optional[list] = None,
     ):
+        """
+        Check a dataframe for correctness.
+
+        Check the dataframe stored at the file with suffix file_suffix in the folder
+        results_folder. Pass in drop_column to indicate columns that shouldn't be
+        checked because they will vary across runs (for instance, timestamp columns).
+        """
+        if drop_columns is None:
+            drop_columns = []
         file_path = self._get_matching_file_path(
             results_folder=results_folder, file_suffix=file_suffix
         )
-        df = pd.read_csv(file_path)
+        df = pd.read_csv(file_path).drop(columns=drop_columns)
         dataframe_regression.check(data_frame=df)
 
     def _check_file_contents(