From 1566bed64b2a1ea548753c57f7362c875b82c6b4 Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Sun, 23 Feb 2025 13:36:36 -0500 Subject: [PATCH 01/14] Update _config_utils.py allow for config and ctc to be set to None --- mne_bids_pipeline/_config_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index 39837d97a..4f46bf975 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -463,10 +463,8 @@ def get_mf_cal_fname( ).match()[0] mf_cal_fpath = bids_path.meg_calibration_fpath if mf_cal_fpath is None: - raise ValueError( - "Could not determine Maxwell Filter Calibration file from BIDS " - f"definition for file {bids_path}." - ) + msg = "WARNING: Could not determine Maxwell Filter Calibration file from BIDS. Set to None." + logger.info(**gen_log_kwargs(message=msg)) else: mf_cal_fpath = pathlib.Path(config.mf_cal_fname).expanduser().absolute() if not mf_cal_fpath.exists(): @@ -491,7 +489,8 @@ def get_mf_ctc_fname( root=config.bids_root, ).meg_crosstalk_fpath if mf_ctc_fpath is None: - raise ValueError("Could not find Maxwell Filter cross-talk file.") + msg = "WARNING: Could not find Maxwell Filter cross-talk file. Set to None." + logger.info(**gen_log_kwargs(message=msg)) else: mf_ctc_fpath = pathlib.Path(config.mf_ctc_fname).expanduser().absolute() if not mf_ctc_fpath.exists(): From c6f65075ccae42bf3786227edb0b3b162ff0a1db Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Mon, 24 Feb 2025 12:18:42 -0500 Subject: [PATCH 02/14] include None types for bids paths --- mne_bids_pipeline/_config_utils.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index 4f46bf975..07f242fe2 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -452,7 +452,7 @@ def sanitize_cond_name(cond: str) -> str: def get_mf_cal_fname( *, config: SimpleNamespace, subject: str, session: str | None -) -> pathlib.Path: +) -> pathlib.Path | None: if config.mf_cal_fname is None: bids_path = BIDSPath( subject=subject, @@ -460,10 +460,12 @@ def get_mf_cal_fname( suffix="meg", datatype="meg", root=config.bids_root, - ).match()[0] - mf_cal_fpath = bids_path.meg_calibration_fpath - if mf_cal_fpath is None: - msg = "WARNING: Could not determine Maxwell Filter Calibration file from BIDS. Set to None." + ).match() + if len(bids_path) > 0: + mf_cal_fpath = bids_path[0].meg_calibration_fpath + else: + mf_cal_fpath = None + msg = "warning: Could not determine Maxwell Filter Calibration file from BIDS. Set to None." logger.info(**gen_log_kwargs(message=msg)) else: mf_cal_fpath = pathlib.Path(config.mf_cal_fname).expanduser().absolute() @@ -473,23 +475,26 @@ def get_mf_cal_fname( f"file at {str(mf_cal_fpath)}." ) - assert isinstance(mf_cal_fpath, pathlib.Path), type(mf_cal_fpath) + assert isinstance(mf_cal_fpath, pathlib.Path | None), type(mf_cal_fpath) return mf_cal_fpath def get_mf_ctc_fname( *, config: SimpleNamespace, subject: str, session: str | None -) -> pathlib.Path: +) -> pathlib.Path | None: if config.mf_ctc_fname is None: - mf_ctc_fpath = BIDSPath( + bids_path = BIDSPath( subject=subject, session=session, suffix="meg", datatype="meg", root=config.bids_root, - ).meg_crosstalk_fpath - if mf_ctc_fpath is None: - msg = "WARNING: Could not find Maxwell Filter cross-talk file. Set to None." + ).match() + if len(bids_path) > 0: + mf_ctc_fpath = bids_path[0].meg_crosstalk_fpath + else: + mf_ctc_fpath = None + msg = "warning: Could not find Maxwell Filter cross-talk file. Set to None." logger.info(**gen_log_kwargs(message=msg)) else: mf_ctc_fpath = pathlib.Path(config.mf_ctc_fname).expanduser().absolute() @@ -498,7 +503,7 @@ def get_mf_ctc_fname( f"Could not find Maxwell Filter cross-talk file at {str(mf_ctc_fpath)}." ) - assert isinstance(mf_ctc_fpath, pathlib.Path), type(mf_ctc_fpath) + assert isinstance(mf_ctc_fpath, pathlib.Path | None), type(mf_ctc_fpath) return mf_ctc_fpath From 0337945703256dcea7e8a4ae77bf68f1322ffa09 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 17:19:09 +0000 Subject: [PATCH 03/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne_bids_pipeline/_config_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index 07f242fe2..fe189a7f9 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -463,7 +463,7 @@ def get_mf_cal_fname( ).match() if len(bids_path) > 0: mf_cal_fpath = bids_path[0].meg_calibration_fpath - else: + else: mf_cal_fpath = None msg = "warning: Could not determine Maxwell Filter Calibration file from BIDS. Set to None." logger.info(**gen_log_kwargs(message=msg)) @@ -492,7 +492,7 @@ def get_mf_ctc_fname( ).match() if len(bids_path) > 0: mf_ctc_fpath = bids_path[0].meg_crosstalk_fpath - else: + else: mf_ctc_fpath = None msg = "warning: Could not find Maxwell Filter cross-talk file. Set to None." logger.info(**gen_log_kwargs(message=msg)) From fd97f2e09f87ef55ca03ea87b0e8e2d5cf958622 Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Mon, 24 Feb 2025 14:05:37 -0500 Subject: [PATCH 04/14] remove `mf_cal_fname` and `mf_ctc_fname` from `in_files` --- mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py index 14e255974..4e084521f 100644 --- a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py +++ b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py @@ -281,9 +281,6 @@ def get_input_fnames_maxwell_filter( ) _update_for_splits(in_files, key, single=True) - # standard files - in_files["mf_cal_fname"] = cfg.mf_cal_fname - in_files["mf_ctc_fname"] = cfg.mf_ctc_fname return in_files @@ -380,8 +377,8 @@ def run_maxwell_filter( apply_msg += " to" mf_kws = dict( - calibration=in_files.pop("mf_cal_fname"), - cross_talk=in_files.pop("mf_ctc_fname"), + calibration=cfg.mf_cal_fname, + cross_talk=cfg.mf_ctc_fname, st_duration=cfg.mf_st_duration, st_correlation=cfg.mf_st_correlation, origin=cfg.mf_head_origin, From b39c19688c75e1be1f81e0043af027904a541c9d Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Mon, 24 Feb 2025 14:08:50 -0500 Subject: [PATCH 05/14] wrap text --- mne_bids_pipeline/_config_utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index fe189a7f9..b3c523df1 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -465,7 +465,10 @@ def get_mf_cal_fname( mf_cal_fpath = bids_path[0].meg_calibration_fpath else: mf_cal_fpath = None - msg = "warning: Could not determine Maxwell Filter Calibration file from BIDS. Set to None." + msg = ( + "warning: Could not determine Maxwell Filter Calibration file " + "from BIDS. Set to None." + ) logger.info(**gen_log_kwargs(message=msg)) else: mf_cal_fpath = pathlib.Path(config.mf_cal_fname).expanduser().absolute() @@ -494,7 +497,10 @@ def get_mf_ctc_fname( mf_ctc_fpath = bids_path[0].meg_crosstalk_fpath else: mf_ctc_fpath = None - msg = "warning: Could not find Maxwell Filter cross-talk file. Set to None." + msg = ( + "warning: Could not find Maxwell Filter cross-talk file " + "from BIDS. Set to None." + ) logger.info(**gen_log_kwargs(message=msg)) else: mf_ctc_fpath = pathlib.Path(config.mf_ctc_fname).expanduser().absolute() From ff827dce7c4cd42c6b0e998456d9d9ff905fb5af Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Mon, 24 Feb 2025 14:20:36 -0500 Subject: [PATCH 06/14] update documentation --- docs/source/dev.md.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/dev.md.inc b/docs/source/dev.md.inc index b682b779d..6a1779f0d 100644 --- a/docs/source/dev.md.inc +++ b/docs/source/dev.md.inc @@ -28,6 +28,7 @@ - Fix bug where the ``config.proc`` parameter was not used properly during forward model creation (#1014 by @larsoner) - Fix bug where emptyroom recordings containing EEG channels would crash the pipeline during maxwell filtering (#1040 by @drammock) +- `mf_ctc_fname` and `mf_cal_fname` can now be set to `None` (#1057 by @harrisonritz) ### :books: Documentation From 6c247b3ac3e8f7fe30461cc5bcc3c0592ed3016c Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Mon, 24 Feb 2025 15:53:12 -0500 Subject: [PATCH 07/14] use in_files if paths are provided; update 01_data_quality to match --- .../steps/preprocessing/_01_data_quality.py | 14 ++++++++++++++ .../steps/preprocessing/_03_maxfilter.py | 11 +++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index e3030e13c..c93fe1eaf 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -60,6 +60,14 @@ def get_input_fnames_data_quality( add_bads=False, ) ) + + # set calibration and crosstalk files (if provided) + if _do_mf_autobad(cfg=cfg): + if cfg.mf_cal_fname is not None: + in_files["mf_cal_fname"] = cfg.mf_cal_fname + if cfg.mf_ctc_fname is not None: + in_files["mf_ctc_fname"] = cfg.mf_ctc_fname + return in_files @@ -88,6 +96,7 @@ def assess_data_quality( bids_path_ref_in = None msg, _ = _read_raw_msg(bids_path_in=bids_path_in, run=run, task=task) logger.info(**gen_log_kwargs(message=msg)) + if run is None and task == "noise": raw = import_er_data( cfg=cfg, @@ -111,6 +120,11 @@ def assess_data_quality( auto_noisy_chs: list[str] | None = None auto_flat_chs: list[str] | None = None if _do_mf_autobad(cfg=cfg): + + # use calibration and crosstalk files (if provided) + cfg.mf_cal_fname = in_files.pop("mf_cal_fname", None) + cfg.mf_ctc_fname = in_files.pop("mf_ctc_fname", None) + ( auto_noisy_chs, auto_flat_chs, diff --git a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py index 4e084521f..86154a81e 100644 --- a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py +++ b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py @@ -281,6 +281,13 @@ def get_input_fnames_maxwell_filter( ) _update_for_splits(in_files, key, single=True) + # set calibration and crosstalk files (if provided) + if cfg.mf_cal_fname is not None: + in_files["mf_cal_fname"] = cfg.mf_cal_fname + if cfg.mf_ctc_fname is not None: + in_files["mf_ctc_fname"] = cfg.mf_ctc_fname + + return in_files @@ -377,8 +384,8 @@ def run_maxwell_filter( apply_msg += " to" mf_kws = dict( - calibration=cfg.mf_cal_fname, - cross_talk=cfg.mf_ctc_fname, + calibration=in_files.pop("mf_cal_fname", None), + cross_talk=in_files.pop("mf_ctc_fname", None), st_duration=cfg.mf_st_duration, st_correlation=cfg.mf_st_correlation, origin=cfg.mf_head_origin, From b54ad7d4552208bd2ec3ffc78d148567e7e481f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 20:53:35 +0000 Subject: [PATCH 08/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne_bids_pipeline/steps/preprocessing/_01_data_quality.py | 7 +++---- mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py index c93fe1eaf..1385b3102 100644 --- a/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py +++ b/mne_bids_pipeline/steps/preprocessing/_01_data_quality.py @@ -60,14 +60,14 @@ def get_input_fnames_data_quality( add_bads=False, ) ) - + # set calibration and crosstalk files (if provided) if _do_mf_autobad(cfg=cfg): - if cfg.mf_cal_fname is not None: + if cfg.mf_cal_fname is not None: in_files["mf_cal_fname"] = cfg.mf_cal_fname if cfg.mf_ctc_fname is not None: in_files["mf_ctc_fname"] = cfg.mf_ctc_fname - + return in_files @@ -120,7 +120,6 @@ def assess_data_quality( auto_noisy_chs: list[str] | None = None auto_flat_chs: list[str] | None = None if _do_mf_autobad(cfg=cfg): - # use calibration and crosstalk files (if provided) cfg.mf_cal_fname = in_files.pop("mf_cal_fname", None) cfg.mf_ctc_fname = in_files.pop("mf_ctc_fname", None) diff --git a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py index 86154a81e..30fd0db4e 100644 --- a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py +++ b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py @@ -282,12 +282,11 @@ def get_input_fnames_maxwell_filter( _update_for_splits(in_files, key, single=True) # set calibration and crosstalk files (if provided) - if cfg.mf_cal_fname is not None: + if cfg.mf_cal_fname is not None: in_files["mf_cal_fname"] = cfg.mf_cal_fname if cfg.mf_ctc_fname is not None: in_files["mf_ctc_fname"] = cfg.mf_ctc_fname - return in_files From b773e65465df77216598c385222ed9e70b7e9acb Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Mon, 24 Feb 2025 16:53:41 -0500 Subject: [PATCH 09/14] fix indenting --- mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py index 30fd0db4e..507ecf008 100644 --- a/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py +++ b/mne_bids_pipeline/steps/preprocessing/_03_maxfilter.py @@ -281,11 +281,11 @@ def get_input_fnames_maxwell_filter( ) _update_for_splits(in_files, key, single=True) - # set calibration and crosstalk files (if provided) - if cfg.mf_cal_fname is not None: - in_files["mf_cal_fname"] = cfg.mf_cal_fname - if cfg.mf_ctc_fname is not None: - in_files["mf_ctc_fname"] = cfg.mf_ctc_fname + # set calibration and crosstalk files (if provided) + if cfg.mf_cal_fname is not None: + in_files["mf_cal_fname"] = cfg.mf_cal_fname + if cfg.mf_ctc_fname is not None: + in_files["mf_ctc_fname"] = cfg.mf_ctc_fname return in_files From 12df21c0eaa183eae8feaa36ab095aca6af1b5cd Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Tue, 25 Feb 2025 16:52:58 -0500 Subject: [PATCH 10/14] added: `mf_cal_missing` and `mf_ctc_missing`; new test: `ds000248_mf` --- docs/source/dev.md.inc | 3 +- mne_bids_pipeline/_config.py | 14 ++++ mne_bids_pipeline/_config_utils.py | 72 ++++++++++++++----- .../tests/configs/config_ds000248_mf.py | 60 ++++++++++++++++ mne_bids_pipeline/tests/test_run.py | 3 + 5 files changed, 132 insertions(+), 20 deletions(-) create mode 100644 mne_bids_pipeline/tests/configs/config_ds000248_mf.py diff --git a/docs/source/dev.md.inc b/docs/source/dev.md.inc index 6a1779f0d..a3b7f6a6f 100644 --- a/docs/source/dev.md.inc +++ b/docs/source/dev.md.inc @@ -6,6 +6,7 @@ - New config option [`allow_missing_sessions`][mne_bids_pipeline._config.allow_missing_sessions] allows to continue when not all sessions are present for all subjects. (#1000 by @drammock) - New config option [`mf_extra_kws`][mne_bids_pipeline._config.mf_extra_kws] passes additional keyword arguments to `mne.preprocessing.maxwell_filter`. (#1038 by @drammock) - New value `"twa"` for config option [`mf_destination`][mne_bids_pipeline._config.mf_destination], to use the time-weighted average head position across runs as the destination position. (#1043 and #1055 by @drammock) +- New config option [`mf_cal_missing`][mne_bids_pipeline._config.mf_cal_missing] and [`mf_ctc_missing`][mne_bids_pipeline._config.mf_ctc_missing] for handling missing calibration and cross-talk files ['raise', 'warn', 'ignore'] (#1057 by @harrisonritz) ### :warning: Behavior changes @@ -28,7 +29,7 @@ - Fix bug where the ``config.proc`` parameter was not used properly during forward model creation (#1014 by @larsoner) - Fix bug where emptyroom recordings containing EEG channels would crash the pipeline during maxwell filtering (#1040 by @drammock) -- `mf_ctc_fname` and `mf_cal_fname` can now be set to `None` (#1057 by @harrisonritz) + ### :books: Documentation diff --git a/mne_bids_pipeline/_config.py b/mne_bids_pipeline/_config.py index 7cf4c3d55..b0b7ecee9 100644 --- a/mne_bids_pipeline/_config.py +++ b/mne_bids_pipeline/_config.py @@ -704,6 +704,13 @@ ``` """ # noqa : E501 +mf_cal_missing: Literal["ignore", "warn", "raise"] = "raise" +""" +How to handle the situation where the Maxwell Filter calibration file is +missing. Possible options are to ignore the missing file, issue a warning, +or raise an error. +""" + mf_ctc_fname: str | None = None """ Path to the Maxwell Filter cross-talk file. If `None`, the recommended @@ -719,6 +726,13 @@ ``` """ # noqa : E501 +mf_ctc_missing: Literal["ignore", "warn", "raise"] = "raise" +""" +How to handle the situation where the Maxwell Filter cross-talk file is +missing. Possible options are to ignore the missing file, issue a warning, +or raise an error. +""" + mf_esss: int = 0 """ Number of extended SSS (eSSS) basis projectors to use from empty-room data. diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index b3c523df1..66e2ef825 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -464,19 +464,35 @@ def get_mf_cal_fname( if len(bids_path) > 0: mf_cal_fpath = bids_path[0].meg_calibration_fpath else: - mf_cal_fpath = None - msg = ( - "warning: Could not determine Maxwell Filter Calibration file " - "from BIDS. Set to None." - ) - logger.info(**gen_log_kwargs(message=msg)) + if config.mf_cal_missing == 'raise': + raise ValueError( + "Could not determine Maxwell Filter cross-talk file from BIDS." + ) + else: + mf_cal_fpath = None + if config.mf_cal_missing == 'warn': + msg = ( + "WARNING: Could not find Maxwell Filter cross-talk file " + "from BIDS. Set to None." + ) + logger.info(**gen_log_kwargs(message=msg)) else: mf_cal_fpath = pathlib.Path(config.mf_cal_fname).expanduser().absolute() if not mf_cal_fpath.exists(): - raise ValueError( - f"Could not find Maxwell Filter Calibration " - f"file at {str(mf_cal_fpath)}." - ) + if config.mf_cal_missing == 'raise': + raise ValueError( + f"Could not find Maxwell Filter Calibration " + f"file at {str(config.mf_cal_fname)}." + ) + else: + mf_cal_fpath = None + if config.mf_cal_missing == 'warn': + msg = ( + "WARNING: Could not find Maxwell Filter calibration file " + f"at {str(config.mf_cal_fname)}. Set to None." + ) + logger.info(**gen_log_kwargs(message=msg)) + assert isinstance(mf_cal_fpath, pathlib.Path | None), type(mf_cal_fpath) return mf_cal_fpath @@ -496,18 +512,36 @@ def get_mf_ctc_fname( if len(bids_path) > 0: mf_ctc_fpath = bids_path[0].meg_crosstalk_fpath else: - mf_ctc_fpath = None - msg = ( - "warning: Could not find Maxwell Filter cross-talk file " - "from BIDS. Set to None." - ) - logger.info(**gen_log_kwargs(message=msg)) + if config.mf_ctc_missing == 'raise': + raise ValueError( + "Could not determine Maxwell Filter cross-talk file from BIDS." + ) + else: + mf_ctc_fpath = None + if config.mf_ctc_missing == 'warn': + msg = ( + "WARNING: Could not find Maxwell Filter cross-talk file " + "from BIDS. Set to None." + ) + logger.info(**gen_log_kwargs(message=msg)) + else: mf_ctc_fpath = pathlib.Path(config.mf_ctc_fname).expanduser().absolute() if not mf_ctc_fpath.exists(): - raise ValueError( - f"Could not find Maxwell Filter cross-talk file at {str(mf_ctc_fpath)}." - ) + if config.mf_ctc_missing == 'raise': + raise ValueError( + f"Could not find Maxwell Filter cross-talk file " + f"at {str(config.mf_ctc_fname)}." + ) + else: + mf_ctc_fpath = None + if config.mf_ctc_missing == 'warn': + msg = ( + "WARNING: Could not find Maxwell Filter cross-talk file " + f"at {str(config.mf_ctc_fname)}. Set to None." + ) + logger.info(**gen_log_kwargs(message=msg)) + assert isinstance(mf_ctc_fpath, pathlib.Path | None), type(mf_ctc_fpath) return mf_ctc_fpath diff --git a/mne_bids_pipeline/tests/configs/config_ds000248_mf.py b/mne_bids_pipeline/tests/configs/config_ds000248_mf.py new file mode 100644 index 000000000..fc3cb214f --- /dev/null +++ b/mne_bids_pipeline/tests/configs/config_ds000248_mf.py @@ -0,0 +1,60 @@ +"""MNE Sample Data: M/EEG combined processing.""" + +import mne +import mne_bids + +bids_root = "~/mne_data/ds000248" +deriv_root = "~/mne_data/derivatives/mne-bids-pipeline/ds000248_mf" +subjects_dir = f"{bids_root}/derivatives/freesurfer/subjects" + +subjects = ["01"] +rename_events = {"Smiley": "Emoji", "Button": "Switch"} +conditions = ["Auditory", "Visual", "Auditory/Left", "Auditory/Right"] +epochs_metadata_query = "index > 0" # Just for testing! +contrasts = [("Visual", "Auditory"), ("Auditory/Right", "Auditory/Left")] + +time_frequency_conditions = ["Auditory", "Visual"] + +ch_types = ["meg", "eeg"] +mf_reference_run = "01" +find_flat_channels_meg = True +find_noisy_channels_meg = True +use_maxwell_filter = True + +# ADJUST THESE TO TEST ERROR HANDLING ------------------------------------------ +mf_cal_fname = None +mf_cal_missing = "warn" + +mf_ctc_fname = f"{bids_root}/wrong_ctc.fif" +mf_ctc_missing = "warn" +# ----------------------------------------------------------------------------- + +def noise_cov(bp: mne_bids.BIDSPath) -> mne.Covariance: + """Estimate the noise covariance.""" + # Use pre-stimulus period as noise source + bp = bp.copy().update(suffix="epo") + if not bp.fpath.exists(): + bp.update(split="01") + epo = mne.read_epochs(bp) + cov = mne.compute_covariance(epo, rank="info", tmax=0) + return cov + + +spatial_filter = "ssp" +n_proj_eog = dict(n_mag=1, n_grad=1, n_eeg=1) +n_proj_ecg = dict(n_mag=1, n_grad=1, n_eeg=0) +ssp_meg = "combined" +ecg_proj_from_average = True +eog_proj_from_average = False +epochs_decim = 4 + +bem_mri_images = "FLASH" +recreate_bem = True + +n_jobs = 2 + + +def mri_t1_path_generator(bids_path: mne_bids.BIDSPath) -> mne_bids.BIDSPath: + """Return the path to a T1 image.""" + # don't really do any modifications – just for testing! + return bids_path diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index a0f65c2a4..2dcdcdf96 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -88,6 +88,9 @@ class _TestOptionsT(TypedDict, total=False): "ds000248_no_mri": { "steps": ("preprocessing", "sensor", "source"), }, + "ds000248_mf": { + "stage": ("preprocessing"), + }, "ds001810": { "steps": ("preprocessing", "preprocessing", "sensor"), }, From f59057b1b53a1c9d2d52496dbdd4d1a13603718e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 21:53:21 +0000 Subject: [PATCH 11/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne_bids_pipeline/_config_utils.py | 22 +++++++++---------- .../tests/configs/config_ds000248_mf.py | 3 ++- mne_bids_pipeline/tests/test_run.py | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index 66e2ef825..e702de534 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -464,13 +464,13 @@ def get_mf_cal_fname( if len(bids_path) > 0: mf_cal_fpath = bids_path[0].meg_calibration_fpath else: - if config.mf_cal_missing == 'raise': + if config.mf_cal_missing == "raise": raise ValueError( "Could not determine Maxwell Filter cross-talk file from BIDS." ) else: mf_cal_fpath = None - if config.mf_cal_missing == 'warn': + if config.mf_cal_missing == "warn": msg = ( "WARNING: Could not find Maxwell Filter cross-talk file " "from BIDS. Set to None." @@ -479,20 +479,19 @@ def get_mf_cal_fname( else: mf_cal_fpath = pathlib.Path(config.mf_cal_fname).expanduser().absolute() if not mf_cal_fpath.exists(): - if config.mf_cal_missing == 'raise': + if config.mf_cal_missing == "raise": raise ValueError( f"Could not find Maxwell Filter Calibration " f"file at {str(config.mf_cal_fname)}." - ) + ) else: mf_cal_fpath = None - if config.mf_cal_missing == 'warn': + if config.mf_cal_missing == "warn": msg = ( "WARNING: Could not find Maxwell Filter calibration file " f"at {str(config.mf_cal_fname)}. Set to None." ) logger.info(**gen_log_kwargs(message=msg)) - assert isinstance(mf_cal_fpath, pathlib.Path | None), type(mf_cal_fpath) return mf_cal_fpath @@ -512,13 +511,13 @@ def get_mf_ctc_fname( if len(bids_path) > 0: mf_ctc_fpath = bids_path[0].meg_crosstalk_fpath else: - if config.mf_ctc_missing == 'raise': + if config.mf_ctc_missing == "raise": raise ValueError( "Could not determine Maxwell Filter cross-talk file from BIDS." ) else: mf_ctc_fpath = None - if config.mf_ctc_missing == 'warn': + if config.mf_ctc_missing == "warn": msg = ( "WARNING: Could not find Maxwell Filter cross-talk file " "from BIDS. Set to None." @@ -528,20 +527,19 @@ def get_mf_ctc_fname( else: mf_ctc_fpath = pathlib.Path(config.mf_ctc_fname).expanduser().absolute() if not mf_ctc_fpath.exists(): - if config.mf_ctc_missing == 'raise': + if config.mf_ctc_missing == "raise": raise ValueError( f"Could not find Maxwell Filter cross-talk file " f"at {str(config.mf_ctc_fname)}." - ) + ) else: mf_ctc_fpath = None - if config.mf_ctc_missing == 'warn': + if config.mf_ctc_missing == "warn": msg = ( "WARNING: Could not find Maxwell Filter cross-talk file " f"at {str(config.mf_ctc_fname)}. Set to None." ) logger.info(**gen_log_kwargs(message=msg)) - assert isinstance(mf_ctc_fpath, pathlib.Path | None), type(mf_ctc_fpath) return mf_ctc_fpath diff --git a/mne_bids_pipeline/tests/configs/config_ds000248_mf.py b/mne_bids_pipeline/tests/configs/config_ds000248_mf.py index fc3cb214f..1e12db34e 100644 --- a/mne_bids_pipeline/tests/configs/config_ds000248_mf.py +++ b/mne_bids_pipeline/tests/configs/config_ds000248_mf.py @@ -25,10 +25,11 @@ mf_cal_fname = None mf_cal_missing = "warn" -mf_ctc_fname = f"{bids_root}/wrong_ctc.fif" +mf_ctc_fname = f"{bids_root}/wrong_ctc.fif" mf_ctc_missing = "warn" # ----------------------------------------------------------------------------- + def noise_cov(bp: mne_bids.BIDSPath) -> mne.Covariance: """Estimate the noise covariance.""" # Use pre-stimulus period as noise source diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index 2dcdcdf96..8354e8f1e 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -90,7 +90,7 @@ class _TestOptionsT(TypedDict, total=False): }, "ds000248_mf": { "stage": ("preprocessing"), - }, + }, "ds001810": { "steps": ("preprocessing", "preprocessing", "sensor"), }, From 11f2b0ea0382905474f04c8f8f5be1c23eaa33b5 Mon Sep 17 00:00:00 2001 From: Harrison Ritz Date: Tue, 25 Feb 2025 18:43:34 -0500 Subject: [PATCH 12/14] stage --> steps? --- mne_bids_pipeline/tests/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index 8354e8f1e..79a7f9011 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -89,7 +89,7 @@ class _TestOptionsT(TypedDict, total=False): "steps": ("preprocessing", "sensor", "source"), }, "ds000248_mf": { - "stage": ("preprocessing"), + "steps": ("preprocessing"), }, "ds001810": { "steps": ("preprocessing", "preprocessing", "sensor"), From 60571f26b672312f9548f38296fb7ad001fea971 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 26 Feb 2025 09:51:37 -0600 Subject: [PATCH 13/14] refactor error/warning messages --- mne_bids_pipeline/_config_utils.py | 44 ++++++++++-------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index e702de534..02021de4d 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -453,6 +453,7 @@ def sanitize_cond_name(cond: str) -> str: def get_mf_cal_fname( *, config: SimpleNamespace, subject: str, session: str | None ) -> pathlib.Path | None: + msg = "Could not find Maxwell Filter calibration file {where}." if config.mf_cal_fname is None: bids_path = BIDSPath( subject=subject, @@ -464,33 +465,24 @@ def get_mf_cal_fname( if len(bids_path) > 0: mf_cal_fpath = bids_path[0].meg_calibration_fpath else: + msg = msg.format(where=f"from BIDSPath {bids_path}") if config.mf_cal_missing == "raise": - raise ValueError( - "Could not determine Maxwell Filter cross-talk file from BIDS." - ) + raise ValueError(msg) else: mf_cal_fpath = None if config.mf_cal_missing == "warn": - msg = ( - "WARNING: Could not find Maxwell Filter cross-talk file " - "from BIDS. Set to None." - ) + msg = f"WARNING: {msg} Set to None." logger.info(**gen_log_kwargs(message=msg)) else: mf_cal_fpath = pathlib.Path(config.mf_cal_fname).expanduser().absolute() if not mf_cal_fpath.exists(): + msg = msg.format(where=f"at {str(config.mf_cal_fname)}") if config.mf_cal_missing == "raise": - raise ValueError( - f"Could not find Maxwell Filter Calibration " - f"file at {str(config.mf_cal_fname)}." - ) + raise ValueError(msg) else: mf_cal_fpath = None if config.mf_cal_missing == "warn": - msg = ( - "WARNING: Could not find Maxwell Filter calibration file " - f"at {str(config.mf_cal_fname)}. Set to None." - ) + msg = f"WARNING: {msg} Set to None." logger.info(**gen_log_kwargs(message=msg)) assert isinstance(mf_cal_fpath, pathlib.Path | None), type(mf_cal_fpath) @@ -500,6 +492,7 @@ def get_mf_cal_fname( def get_mf_ctc_fname( *, config: SimpleNamespace, subject: str, session: str | None ) -> pathlib.Path | None: + msg = "Could not find Maxwell Filter cross-talk file {where}." if config.mf_ctc_fname is None: bids_path = BIDSPath( subject=subject, @@ -511,34 +504,25 @@ def get_mf_ctc_fname( if len(bids_path) > 0: mf_ctc_fpath = bids_path[0].meg_crosstalk_fpath else: + msg = msg.format(where=f"from BIDSPath {bids_path}") if config.mf_ctc_missing == "raise": - raise ValueError( - "Could not determine Maxwell Filter cross-talk file from BIDS." - ) + raise ValueError(msg) else: mf_ctc_fpath = None if config.mf_ctc_missing == "warn": - msg = ( - "WARNING: Could not find Maxwell Filter cross-talk file " - "from BIDS. Set to None." - ) + msg = f"WARNING: {msg} Set to None." logger.info(**gen_log_kwargs(message=msg)) else: mf_ctc_fpath = pathlib.Path(config.mf_ctc_fname).expanduser().absolute() if not mf_ctc_fpath.exists(): + msg = msg.format(where=f"at {str(config.mf_ctc_fname)}") if config.mf_ctc_missing == "raise": - raise ValueError( - f"Could not find Maxwell Filter cross-talk file " - f"at {str(config.mf_ctc_fname)}." - ) + raise ValueError(msg) else: mf_ctc_fpath = None if config.mf_ctc_missing == "warn": - msg = ( - "WARNING: Could not find Maxwell Filter cross-talk file " - f"at {str(config.mf_ctc_fname)}. Set to None." - ) + msg = f"WARNING: {msg} Set to None." logger.info(**gen_log_kwargs(message=msg)) assert isinstance(mf_ctc_fpath, pathlib.Path | None), type(mf_ctc_fpath) From d2676527a0ec26d0d3ee24c1e0523f56f1bd2416 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 26 Feb 2025 11:55:51 -0500 Subject: [PATCH 14/14] Apply suggestions from code review Co-authored-by: Daniel McCloy --- docs/source/dev.md.inc | 2 +- mne_bids_pipeline/_config.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/dev.md.inc b/docs/source/dev.md.inc index a3b7f6a6f..b30400b0c 100644 --- a/docs/source/dev.md.inc +++ b/docs/source/dev.md.inc @@ -6,7 +6,7 @@ - New config option [`allow_missing_sessions`][mne_bids_pipeline._config.allow_missing_sessions] allows to continue when not all sessions are present for all subjects. (#1000 by @drammock) - New config option [`mf_extra_kws`][mne_bids_pipeline._config.mf_extra_kws] passes additional keyword arguments to `mne.preprocessing.maxwell_filter`. (#1038 by @drammock) - New value `"twa"` for config option [`mf_destination`][mne_bids_pipeline._config.mf_destination], to use the time-weighted average head position across runs as the destination position. (#1043 and #1055 by @drammock) -- New config option [`mf_cal_missing`][mne_bids_pipeline._config.mf_cal_missing] and [`mf_ctc_missing`][mne_bids_pipeline._config.mf_ctc_missing] for handling missing calibration and cross-talk files ['raise', 'warn', 'ignore'] (#1057 by @harrisonritz) +- New config options [`mf_cal_missing`][mne_bids_pipeline._config.mf_cal_missing] and [`mf_ctc_missing`][mne_bids_pipeline._config.mf_ctc_missing] for handling missing calibration and cross-talk files (#1057 by @harrisonritz) ### :warning: Behavior changes diff --git a/mne_bids_pipeline/_config.py b/mne_bids_pipeline/_config.py index b0b7ecee9..ad7f7bc26 100644 --- a/mne_bids_pipeline/_config.py +++ b/mne_bids_pipeline/_config.py @@ -706,9 +706,9 @@ mf_cal_missing: Literal["ignore", "warn", "raise"] = "raise" """ -How to handle the situation where the Maxwell Filter calibration file is -missing. Possible options are to ignore the missing file, issue a warning, -or raise an error. +How to handle the situation where the MEG device's fine calibration file is missing. +Possible options are to ignore the missing file (as may be appropriate for OPM data), +issue a warning, or raise an error. """ mf_ctc_fname: str | None = None @@ -728,9 +728,9 @@ mf_ctc_missing: Literal["ignore", "warn", "raise"] = "raise" """ -How to handle the situation where the Maxwell Filter cross-talk file is -missing. Possible options are to ignore the missing file, issue a warning, -or raise an error. +How to handle the situation where the MEG device's cross-talk file is missing. Possible +options are to ignore the missing file (as may be appropriate for OPM data), issue a +warning, or raise an error (appropriate for data from Electa/Neuromag/MEGIN systems). """ mf_esss: int = 0