diff --git a/.circleci/config.yml b/.circleci/config.yml index ccec5378d5e..fe97a362e5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -167,7 +167,7 @@ commands: command: | cd .. git clone git@github.com:facebookresearch/Mephisto.git Mephisto - cd Mephisto; git checkout v0.3.4 -b stable + cd Mephisto; git checkout v0.4.0 -b stable pip install -r requirements.txt python setup.py develop # `echo` so that ENTER will be pressed at the prompt diff --git a/parlai/crowdsourcing/README.md b/parlai/crowdsourcing/README.md index a06d8c013c6..b329e8d19d9 100644 --- a/parlai/crowdsourcing/README.md +++ b/parlai/crowdsourcing/README.md @@ -6,11 +6,11 @@ Code for crowdsourcing tasks that use Mephisto. See the [Mephisto quick start gu ## Running tasks -Tasks are launched by calling the appropriate run script: for instance, an ACUTE-Eval run can be launched with `python parlai/crowdsourcing/tasks/acute_eval/run.py`, followed by any appropriate flags. All run parameters are set using [Hydra](https://github.com/facebookresearch/hydra): append the flag `-c job` to your run command to see a list of all available parameters, grouped by their package name (`mephisto.blueprint`, `mephisto.task`, etc.), which determines how they are called. Each run script has a YAML file of default parameters that will be loaded, found in the `conf/` subfolder of each task. +Tasks are launched by calling the appropriate run script: for instance, an ACUTE-Eval run can be launched with `python parlai/crowdsourcing/tasks/acute_eval/run.py`, followed by any appropriate flags. All run parameters are set using [Hydra](https://github.com/facebookresearch/hydra): append the flag `-c job` to your run command to see a list of all available parameters, grouped by their package name (`mephisto.blueprint`, `mephisto.task`, etc.), which determines how they are called. Each run script has a YAML file of default parameters that will be loaded, found in the `hydra_configs/conf/` subfolder of each task. ### Specifying your own YAML file - The easiest way to specify a different YAML file is to create a new file, say, `my_params.yaml`, in the `conf/` subfolder of the task. Then, you can launch HITs with `python ${TASK_FOLDER}/run.py conf=my_params`. + The easiest way to specify a different YAML file is to create a new file, say, `my_params.yaml`, in the `hydra_configs/conf/` subfolder of the task. Then, you can launch HITs with `python ${TASK_FOLDER}/run.py conf=my_params`. You also can specify a path to a YAML file existing *outside* of `${TASK_FOLDER}`: you will need to have your YAML file stored at a location `${CUSTOM_FOLDER}/conf/my_params.yaml`, and then you can add a `--config-dir ${CUSTOM_FOLDER}` string to the launch command above. diff --git a/parlai/crowdsourcing/projects/multisession_chat/human_eval/conf/example.yaml b/parlai/crowdsourcing/projects/multisession_chat/human_eval/hydra_configs/conf/example.yaml similarity index 100% rename from parlai/crowdsourcing/projects/multisession_chat/human_eval/conf/example.yaml rename to parlai/crowdsourcing/projects/multisession_chat/human_eval/hydra_configs/conf/example.yaml diff --git a/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py b/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py index 6308e8d11b7..f31a90e07f6 100644 --- a/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py +++ b/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py @@ -56,7 +56,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name='scriptconfig', module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: check_override_opt(cfg.mephisto) run_task(cfg=cfg, task_directory=TASK_DIRECTORY, world_module=None) diff --git a/parlai/crowdsourcing/projects/wizard_of_internet/conf/dev.yaml b/parlai/crowdsourcing/projects/wizard_of_internet/hydra_configs/conf/dev.yaml similarity index 100% rename from parlai/crowdsourcing/projects/wizard_of_internet/conf/dev.yaml rename to parlai/crowdsourcing/projects/wizard_of_internet/hydra_configs/conf/dev.yaml diff --git a/parlai/crowdsourcing/projects/wizard_of_internet/run.py b/parlai/crowdsourcing/projects/wizard_of_internet/run.py index f3d42a076ce..0e1bd120c44 100644 --- a/parlai/crowdsourcing/projects/wizard_of_internet/run.py +++ b/parlai/crowdsourcing/projects/wizard_of_internet/run.py @@ -273,7 +273,7 @@ def add_banned_words_frontend_conf(task_state, fpath: str = None): task_state.task_config['bannedWords'] = banned_words -@hydra.main(config_name='scriptconfig') +@hydra.main(config_path="hydra_configs", config_name='scriptconfig') def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) world_opt = get_world_opt(cfg) diff --git a/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py b/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py index 002e93b17e5..1ed9af767e4 100644 --- a/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py +++ b/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py @@ -570,7 +570,7 @@ class TestScriptConfig(MTurkRunScriptConfig): register_script_config(name='fast_acute_scriptconfig', module=TestScriptConfig) -@hydra.main(config_name="fast_acute_scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="fast_acute_scriptconfig") def main(cfg: DictConfig) -> None: runner = FastAcuteExecutor(cfg) diff --git a/parlai/crowdsourcing/tasks/acute_eval/conf/base_fast_acute.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/base_fast_acute.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/acute_eval/conf/base_fast_acute.yaml rename to parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/base_fast_acute.yaml diff --git a/parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml rename to parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example.yaml diff --git a/parlai/crowdsourcing/tasks/acute_eval/conf/example_fast_acute.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example_fast_acute.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/acute_eval/conf/example_fast_acute.yaml rename to parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example_fast_acute.yaml diff --git a/parlai/crowdsourcing/tasks/acute_eval/run.py b/parlai/crowdsourcing/tasks/acute_eval/run.py index 76d3e585ff3..7cd0aabddc1 100644 --- a/parlai/crowdsourcing/tasks/acute_eval/run.py +++ b/parlai/crowdsourcing/tasks/acute_eval/run.py @@ -53,7 +53,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name='scriptconfig', module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) print(f'*** RUN ID: {cfg.mephisto.task.task_name} ***') diff --git a/parlai/crowdsourcing/tasks/model_chat/conf/example.yaml b/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/model_chat/conf/example.yaml rename to parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example.yaml diff --git a/parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml b/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example_image_chat.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/model_chat/conf/example_image_chat.yaml rename to parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example_image_chat.yaml diff --git a/parlai/crowdsourcing/tasks/model_chat/run.py b/parlai/crowdsourcing/tasks/model_chat/run.py index 1551cfda5c0..31d371ca0ef 100644 --- a/parlai/crowdsourcing/tasks/model_chat/run.py +++ b/parlai/crowdsourcing/tasks/model_chat/run.py @@ -44,7 +44,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name='scriptconfig', module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: run_task(cfg=cfg, task_directory=TASK_DIRECTORY, world_module=world_module) diff --git a/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py b/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py index 7d093ace25d..3349fd0b0d9 100644 --- a/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py +++ b/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py @@ -46,7 +46,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name='scriptconfig', module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: run_task(cfg=cfg, task_directory=TASK_DIRECTORY, world_module=world_module) diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml b/parlai/crowdsourcing/tasks/qa_data_collection/hydra_configs/conf/example.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/qa_data_collection/conf/example.yaml rename to parlai/crowdsourcing/tasks/qa_data_collection/hydra_configs/conf/example.yaml diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/run.py b/parlai/crowdsourcing/tasks/qa_data_collection/run.py index b328dab06f4..65727bf6e04 100644 --- a/parlai/crowdsourcing/tasks/qa_data_collection/run.py +++ b/parlai/crowdsourcing/tasks/qa_data_collection/run.py @@ -63,7 +63,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name="scriptconfig", module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/turn_annotations_static/conf/example.yaml rename to parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example.yaml diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example_in_flight_qa.yaml similarity index 100% rename from parlai/crowdsourcing/tasks/turn_annotations_static/conf/example_in_flight_qa.yaml rename to parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example_in_flight_qa.yaml diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/run.py b/parlai/crowdsourcing/tasks/turn_annotations_static/run.py index 555ca397322..c984ad35cea 100644 --- a/parlai/crowdsourcing/tasks/turn_annotations_static/run.py +++ b/parlai/crowdsourcing/tasks/turn_annotations_static/run.py @@ -45,7 +45,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name='scriptconfig', module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: run_static_task(cfg=cfg, task_directory=TASK_DIRECTORY) diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py b/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py index c595c53024a..8812cddbe57 100644 --- a/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py +++ b/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py @@ -49,7 +49,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name='scriptconfig', module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: run_static_task(cfg=cfg, task_directory=TASK_DIRECTORY) diff --git a/parlai/crowdsourcing/utils/tests.py b/parlai/crowdsourcing/utils/tests.py index 8245ef7ddf7..2f179cbf074 100644 --- a/parlai/crowdsourcing/utils/tests.py +++ b/parlai/crowdsourcing/utils/tests.py @@ -80,7 +80,9 @@ def _set_up_config( relative_task_directory = os.path.relpath( task_directory, os.path.dirname(__file__) ) - relative_config_path = os.path.join(relative_task_directory, 'conf') + relative_config_path = os.path.join( + relative_task_directory, 'hydra_configs', 'conf' + ) if overrides is None: overrides = [] with initialize(config_path=relative_config_path): @@ -88,6 +90,7 @@ def _set_up_config( config_name="example", overrides=[ f'+mephisto.blueprint._blueprint_type={blueprint_type}', + f'+mephisto.blueprint.link_task_source=False', f'+mephisto/architect=mock', f'+mephisto/provider=mock', f'+task_dir={task_directory}', diff --git a/projects/safety_recipes/human_safety_evaluation/conf/example.yaml b/projects/safety_recipes/human_safety_evaluation/hydra_configs/conf/example.yaml similarity index 100% rename from projects/safety_recipes/human_safety_evaluation/conf/example.yaml rename to projects/safety_recipes/human_safety_evaluation/hydra_configs/conf/example.yaml diff --git a/projects/safety_recipes/human_safety_evaluation/run.py b/projects/safety_recipes/human_safety_evaluation/run.py index 1e3ff52634f..90b550ba7c2 100644 --- a/projects/safety_recipes/human_safety_evaluation/run.py +++ b/projects/safety_recipes/human_safety_evaluation/run.py @@ -38,7 +38,7 @@ class ScriptConfig(MTurkRunScriptConfig): register_script_config(name='scriptconfig', module=ScriptConfig) -@hydra.main(config_name="scriptconfig") +@hydra.main(config_path="hydra_configs", config_name="scriptconfig") def main(cfg: DictConfig) -> None: run_static_task(cfg=cfg, task_directory=BASE_TASK_DIRECTORY) diff --git a/requirements.txt b/requirements.txt index b27b6f90db6..35935dc3a4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,13 +11,13 @@ flake8==3.7.8 iopath~=0.1.8 gitdb2==2.0.5 GitPython==3.0.3 -hydra-core~=1.0.6 +hydra-core~=1.1.0 ipython==7.19.0 torch>=1.4.0 joblib==0.14.1 nltk==3.4.5 numpy==1.17.5 -omegaconf<2.1.0 +omegaconf~=2.1.1 pandas==1.1.1 pytest_regressions==2.1.1 pytest==5.3.2