From c8f478ba33b2d0eed9e326b543476403e54dcd03 Mon Sep 17 00:00:00 2001 From: Mathieu Scheltienne Date: Fri, 23 Jun 2023 15:22:33 +0200 Subject: [PATCH 01/26] Reset the channel unit multiplication factor on unit change (#11750) --- mne/channels/channels.py | 4 +++- mne/channels/tests/test_channels.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mne/channels/channels.py b/mne/channels/channels.py index 7b70daadf34..0f306b7b85e 100644 --- a/mne/channels/channels.py +++ b/mne/channels/channels.py @@ -36,7 +36,7 @@ _on_missing, legacy, ) -from ..io.constants import FIFF +from ..io.constants import FIFF, _ch_unit_mul_named from ..io.meas_info import ( anonymize_info, Info, @@ -442,6 +442,8 @@ def set_channel_types(self, mapping, *, on_unit_change="warn", verbose=None): if this_change not in unit_changes: unit_changes[this_change] = list() unit_changes[this_change].append(ch_name) + # reset unit multiplication factor since the unit has now changed + self.info["chs"][c_ind]["unit_mul"] = _ch_unit_mul_named[0] self.info["chs"][c_ind]["unit"] = _human2unit[ch_type] if ch_type in ["eeg", "seeg", "ecog", "dbs"]: coil_type = FIFF.FIFFV_COIL_EEG diff --git a/mne/channels/tests/test_channels.py b/mne/channels/tests/test_channels.py index 2b719b7e3af..c0868d7a7e3 100644 --- a/mne/channels/tests/test_channels.py +++ b/mne/channels/tests/test_channels.py @@ -38,7 +38,7 @@ read_raw_kit, RawArray, ) -from mne.io.constants import FIFF +from mne.io.constants import FIFF, _ch_unit_mul_named from mne import ( pick_types, pick_channels, @@ -223,6 +223,13 @@ def test_set_channel_types(): ch_types = {raw.ch_names[0]: "misc"} pytest.raises(ValueError, raw.set_channel_types, ch_types) + # test reset of channel units on unit change + idx = raw.ch_names.index("EEG 003") + raw.info["chs"][idx]["unit_mul"] = _ch_unit_mul_named[-6] + assert raw.info["chs"][idx]["unit_mul"] == -6 + raw.set_channel_types({"EEG 003": "misc"}, on_unit_change="ignore") + assert raw.info["chs"][idx]["unit_mul"] == 0 + def test_get_builtin_ch_adjacencies(): """Test retrieving the names of all built-in FieldTrip neighbors.""" From ffa3fb50f3fcf4029d874d490a53c7d142c034ab Mon Sep 17 00:00:00 2001 From: George O'Neill Date: Fri, 23 Jun 2023 18:30:54 +0100 Subject: [PATCH 02/26] [MNT] Add support for FIL data without sensor positions (#11733) --- doc/changes/latest.inc | 1 + mne/io/fil/fil.py | 27 +++++++++++++++++---------- mne/io/fil/tests/test_fil.py | 25 +++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index a73a5b20f40..0d7acbc8b9d 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -38,6 +38,7 @@ Bugs - Fix hanging interpreter with matplotlib figures using ``mne/viz/_mpl_figure.py`` in spyder console and jupyter notebooks (:gh:`11696` by `Mathieu Scheltienne`_) - Fix bug with overlapping text for :meth:`mne.Evoked.plot` (:gh:`11698` by `Alex Rockhill`_) - For :func:`mne.io.read_raw_eyelink`, the default value of the ``gap_description`` parameter is now ``'BAD_ACQ_SKIP'``, following MNE convention (:gh:`11719` by `Scott Huberty`_) +- Fix bug with :func:`mne.io.read_raw_fil` where datasets without sensor positions would not import (:gh:`11733` by `George O'Neill`_) - Allow int-like for the argument ``id`` of `~mne.make_fixed_length_events` (:gh:`11748` by `Mathieu Scheltienne`_) API changes diff --git a/mne/io/fil/fil.py b/mne/io/fil/fil.py index b5b3a96d9c3..c4593aa6c99 100644 --- a/mne/io/fil/fil.py +++ b/mne/io/fil/fil.py @@ -90,9 +90,7 @@ def __init__(self, binfile, precision="single", preload=False): files = _get_file_names(binfile) chans = _from_tsv(files["chans"]) - chanpos = _from_tsv(files["positions"]) nchans = len(chans["name"]) - nlocs = len(chanpos["name"]) nsamples = _determine_nsamples(files["bin"], nchans, precision) - 1 sample_info["nsamples"] = nsamples @@ -101,13 +99,21 @@ def __init__(self, binfile, precision="single", preload=False): chans["pos"] = [None] * nchans chans["ori"] = [None] * nchans - - for ii in range(0, nlocs): - idx = chans["name"].index(chanpos["name"][ii]) - tmp = np.array([chanpos["Px"][ii], chanpos["Py"][ii], chanpos["Pz"][ii]]) - chans["pos"][idx] = tmp.astype(np.float64) - tmp = np.array([chanpos["Ox"][ii], chanpos["Oy"][ii], chanpos["Oz"][ii]]) - chans["ori"][idx] = tmp.astype(np.float64) + if files["positions"].is_file(): + chanpos = _from_tsv(files["positions"]) + nlocs = len(chanpos["name"]) + for ii in range(0, nlocs): + idx = chans["name"].index(chanpos["name"][ii]) + tmp = np.array( + [chanpos["Px"][ii], chanpos["Py"][ii], chanpos["Pz"][ii]] + ) + chans["pos"][idx] = tmp.astype(np.float64) + tmp = np.array( + [chanpos["Ox"][ii], chanpos["Oy"][ii], chanpos["Oz"][ii]] + ) + chans["ori"][idx] = tmp.astype(np.float64) + else: + warn("No sensor position information found.") with open(files["meg"], "r") as fid: meg = json.load(fid) @@ -180,7 +186,8 @@ def _convert_channel_info(chans): """Convert the imported _channels.tsv into the chs element of raw.info.""" nmeg = nstim = nmisc = nref = 0 - units, sf = _get_pos_units(chans["pos"]) + if not all(p is None for p in chans["pos"]): + _, sf = _get_pos_units(chans["pos"]) chs = list() for ii in range(len(chans["name"])): diff --git a/mne/io/fil/tests/test_fil.py b/mne/io/fil/tests/test_fil.py index 0788fd17667..6b4f68dc2bb 100644 --- a/mne/io/fil/tests/test_fil.py +++ b/mne/io/fil/tests/test_fil.py @@ -2,10 +2,13 @@ # # License: BSD-3-Clause -from numpy import isnan, empty +from numpy import isnan, empty, array from numpy.testing import assert_array_equal, assert_array_almost_equal +from os import remove import pytest +import shutil + from mne.datasets import testing from mne.io import read_raw_fil @@ -127,7 +130,7 @@ def _fil_sensorpos(raw_test, raw_mat): @testing.requires_testing_data -def test_fil_all(): +def test_fil_complete(): """Test FIL reader, match to known answers from .mat file.""" binname = fil_path / "sub-noise_ses-001_task-noise220622_run-001_meg.bin" matname = fil_path / "sub-noise_ses-001_task-noise220622_run-001_fieldtrip.mat" @@ -140,3 +143,21 @@ def test_fil_all(): _fil_megmag(raw, mat) _fil_stim(raw, mat) _fil_sensorpos(raw, mat) + + +@testing.requires_testing_data +def test_fil_no_positions(tmp_path): + """Test FIL reader in cases where a position file is missing.""" + test_path = tmp_path / "FIL" + shutil.copytree(fil_path, test_path) + + posname = test_path / "sub-noise_ses-001_task-noise220622_run-001_positions.tsv" + binname = test_path / "sub-noise_ses-001_task-noise220622_run-001_meg.bin" + + remove(posname) + + with pytest.warns(RuntimeWarning, match="No sensor position.*"): + raw = read_raw_fil(binname) + chs = raw.info["chs"] + locs = array([ch["loc"][:] for ch in chs]) + assert isnan(locs).all() From bad786df12c4c14512cb1a27276a688912131a16 Mon Sep 17 00:00:00 2001 From: Scott Huberty <52462026+scott-huberty@users.noreply.github.com> Date: Fri, 23 Jun 2023 13:42:53 -0400 Subject: [PATCH 03/26] ENH, FIX: Make eyelink occular annotations "channel aware", and call blink annotations "BAD" (#11746) Co-authored-by: Eric Larson --- doc/changes/latest.inc | 2 + mne/io/eyelink/_utils.py | 237 ++++++++++++++ mne/io/eyelink/eyelink.py | 306 ++---------------- mne/io/eyelink/tests/test_eyelink.py | 65 ++-- tutorials/io/70_reading_eyetracking_data.py | 6 +- .../preprocessing/90_eyetracking_data.py | 30 +- 6 files changed, 345 insertions(+), 301 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 0d7acbc8b9d..dc14c20b09b 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -27,6 +27,7 @@ Enhancements - When failing to locate a file, we now print the full path in quotation marks to help spot accidentally added trailing spaces (:gh:`11718` by `Richard Höchenberger`_) - Add standard montage lookup table for ``easycap-M43`` (:gh:`11744` by :newcontrib:`Diptyajit Das`) - Added :class:`mne.preprocessing.eyetracking.Calibration` to store eye-tracking calibration info, and :func:`mne.preprocessing.eyetracking.read_eyelink_calibration` to read calibration data from EyeLink systems (:gh:`11719` by `Scott Huberty`_) +- Ocular :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` are now channel aware. This means if the left eye blinked, the associated annotation will store this in the ``'ch_names'`` key. (:gh:`11746` by `Scott Huberty`_) Bugs ~~~~ @@ -40,6 +41,7 @@ Bugs - For :func:`mne.io.read_raw_eyelink`, the default value of the ``gap_description`` parameter is now ``'BAD_ACQ_SKIP'``, following MNE convention (:gh:`11719` by `Scott Huberty`_) - Fix bug with :func:`mne.io.read_raw_fil` where datasets without sensor positions would not import (:gh:`11733` by `George O'Neill`_) - Allow int-like for the argument ``id`` of `~mne.make_fixed_length_events` (:gh:`11748` by `Mathieu Scheltienne`_) +- blink :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` now begin with ``'BAD_'``, i.e. ``'BAD_blink'``, because ocular data are missing during blinks. (:gh:`11746` by `Scott Huberty`_) API changes ~~~~~~~~~~~ diff --git a/mne/io/eyelink/_utils.py b/mne/io/eyelink/_utils.py index 3e6cf76e2fe..f5bb910e059 100644 --- a/mne/io/eyelink/_utils.py +++ b/mne/io/eyelink/_utils.py @@ -5,6 +5,243 @@ import re import numpy as np +from ...utils import _check_pandas_installed, _validate_type + + +def _isfloat(token): + """Boolean test for whether string can be of type float. + + Parameters + ---------- + token : str + Single element from tokens list. + """ + _validate_type(token, str, "token") + try: + float(token) + except ValueError: + return False + else: + return True + + +def _convert_types(tokens): + """Convert the type of each token in list. + + The tokens input is a list of string elements. + Posix timestamp strings can be integers, eye gaze position and + pupil size can be floats. flags token ("...") remains as string. + Missing eye/head-target data (indicated by '.' or 'MISSING_DATA') + are replaced by np.nan. + + Parameters + ---------- + Tokens : list + List of string elements. + + Returns + ------- + Tokens list with elements of various types. + """ + return [ + int(token) + if token.isdigit() # execute this before _isfloat() + else float(token) + if _isfloat(token) + else np.nan + if token in (".", "MISSING_DATA") + else token # remains as string + for token in tokens + ] + + +def _parse_line(line): + """Parse tab delminited string from eyelink ASCII file. + + Takes a tab deliminited string from eyelink file, + splits it into a list of tokens, and converts the type + for each token in the list. + """ + tokens = line.split() + return _convert_types(tokens) + + +def _is_sys_msg(line): + """Flag lines from eyelink ASCII file that contain a known system message. + + Some lines in eyelink files are system outputs usually + only meant for Eyelinks DataViewer application to read. + These shouldn't need to be parsed. + + Parameters + ---------- + line : string + single line from Eyelink asc file + + Returns + ------- + bool : + True if any of the following strings that are + known to indicate a system message are in the line + + Notes + ----- + Examples of eyelink system messages: + - ;Sess:22Aug22;Tria:1;Tri2:False;ESNT:182BFE4C2F4; + - ;NTPT:182BFE55C96;SMSG:__NTP_CLOCK_SYNC__;DIFF:-1; + - !V APLAYSTART 0 1 library/audio + - !MODE RECORD CR 500 2 1 R + """ + return "!V" in line or "!MODE" in line or ";" in line + + +def _get_sfreq(rec_info): + """Get sampling frequency from Eyelink ASCII file. + + Parameters + ---------- + rec_info : list + the first list in self._event_lines['SAMPLES']. + The sfreq occurs after RATE: i.e. [..., RATE, 1000, ...]. + + Returns + ------- + sfreq : int | float + """ + return rec_info[rec_info.index("RATE") + 1] + + +def _sort_by_time(df, col="time"): + df.sort_values(col, ascending=True, inplace=True) + df.reset_index(drop=True, inplace=True) + + +def _convert_times(df, first_samp, col="time"): + """Set initial time to 0, converts from ms to seconds in place. + + Parameters + ---------- + df pandas.DataFrame: + One of the dataframes in the self.dataframes dict. + + first_samp int: + timestamp of the first sample of the recording. This should + be the first sample of the first recording block. + col str (default 'time'): + column name to sort pandas.DataFrame by + + Notes + ----- + Each sample in an Eyelink file has a posix timestamp string. + Subtracts the "first" sample's timestamp from each timestamp. + The "first" sample is inferred to be the first sample of + the first recording block, i.e. the first "START" line. + """ + _sort_by_time(df, col) + for col in df.columns: + if col.endswith("time"): # 'time' and 'end_time' cols + df[col] -= first_samp + df[col] /= 1000 + if col in ["duration", "offset"]: + df[col] /= 1000 + + +def _fill_times( + df, + sfreq, + time_col="time", +): + """Fill missing timestamps if there are multiple recording blocks. + + Parameters + ---------- + df : pandas.DataFrame: + dataframe of the eyetracking data samples, BEFORE + _convert_times() is applied to the dataframe + + sfreq : int | float: + sampling frequency of the data + + time_col : str (default 'time'): + name of column with the timestamps (e.g. 9511881, 9511882, ...) + + Returns + ------- + %(df_return)s + + Notes + ----- + After _parse_recording_blocks, Files with multiple recording blocks will + have missing timestamps for the duration of the period between the blocks. + This would cause the occular annotations (i.e. blinks) to not line up with + the signal. + """ + pd = _check_pandas_installed() + + first, last = df[time_col].iloc[[0, -1]] + step = 1000 / sfreq + df[time_col] = df[time_col].astype(float) + new_times = pd.DataFrame( + np.arange(first, last + step / 2, step), columns=[time_col] + ) + return pd.merge_asof( + new_times, df, on=time_col, direction="nearest", tolerance=step / 10 + ) + + +def _find_overlaps(df, max_time=0.05): + """Merge left/right eye events with onset/offset diffs less than max_time. + + df : pandas.DataFrame + Pandas DataFrame with occular events (fixations, saccades, blinks) + max_time : float (default 0.05) + Time in seconds. Defaults to .05 (50 ms) + + Returns + ------- + DataFrame: %(df_return)s + :class:`pandas.DataFrame` specifying overlapped eye events, if any + Notes + ----- + The idea is to cumulative sum the boolean values for rows with onset and + offset differences (against the previous row) that are greater than the + max_time. If onset and offset diffs are less than max_time then no_overlap + will become False. Alternatively, if either the onset or offset diff is + greater than max_time, no_overlap becomes True. Cumulatively summing over + these boolean values will leave rows with no_overlap == False unchanged + and hence with the same group number. + """ + pd = _check_pandas_installed() + + df = df.copy() + df["overlap_start"] = df.sort_values("time")["time"].diff().lt(max_time) + + df["overlap_end"] = df["end_time"].diff().abs().lt(max_time) + + df["no_overlap"] = ~(df["overlap_end"] & df["overlap_start"]) + df["group"] = df["no_overlap"].cumsum() + + # now use groupby on 'group'. If one left and one right eye in group + # the new start/end times are the mean of the two eyes + ovrlp = pd.concat( + [ + pd.DataFrame(g[1].drop(columns="eye").mean()).T + if (len(g[1]) == 2) and (len(g[1].eye.unique()) == 2) + else g[1] # not an overlap, return group unchanged + for g in df.groupby("group") + ] + ) + # overlapped events get a "both" value in the "eye" col + if "eye" in ovrlp.columns: + ovrlp["eye"] = ovrlp["eye"].fillna("both") + else: + ovrlp["eye"] = "both" + tmp_cols = ["overlap_start", "overlap_end", "no_overlap", "group"] + return ovrlp.drop(columns=tmp_cols).reset_index(drop=True) + + +# Used by read_eyelinke_calibration + def _find_recording_start(lines): """Return the first START line in an SR Research EyeLink ASCII file. diff --git a/mne/io/eyelink/eyelink.py b/mne/io/eyelink/eyelink.py index 5321ddc136d..73a16554f96 100644 --- a/mne/io/eyelink/eyelink.py +++ b/mne/io/eyelink/eyelink.py @@ -10,6 +10,14 @@ from pathlib import Path import numpy as np +from ._utils import ( + _convert_times, + _fill_times, + _find_overlaps, + _get_sfreq, + _is_sys_msg, + _parse_line, +) # helper functions from ..constants import FIFF from ..base import BaseRaw from ..meas_info import create_info @@ -45,247 +53,6 @@ } -def _isfloat(token): - """Boolean test for whether string can be of type float. - - Parameters - ---------- - token : str - Single element from tokens list. - """ - if isinstance(token, str): - try: - float(token) - return True - except ValueError: - return False - else: - raise ValueError( - "input should be a string," f" but {token} is of type {type(token)}" - ) - - -def _convert_types(tokens): - """Convert the type of each token in list. - - The tokens input is a list of string elements. - Posix timestamp strings can be integers, eye gaze position and - pupil size can be floats. flags token ("...") remains as string. - Missing eye/head-target data (indicated by '.' or 'MISSING_DATA') - are replaced by np.nan. - - Parameters - ---------- - Tokens : list - List of string elements. - - Returns - ------- - Tokens list with elements of various types. - """ - return [ - int(token) - if token.isdigit() # execute this before _isfloat() - else float(token) - if _isfloat(token) - else np.nan - if token in (".", "MISSING_DATA") - else token # remains as string - for token in tokens - ] - - -def _parse_line(line): - """Parse tab delminited string from eyelink ASCII file. - - Takes a tab deliminited string from eyelink file, - splits it into a list of tokens, and converts the type - for each token in the list. - """ - if len(line): - tokens = line.split() - return _convert_types(tokens) - else: - raise ValueError("line is empty, nothing to parse") - - -def _is_sys_msg(line): - """Flag lines from eyelink ASCII file that contain a known system message. - - Some lines in eyelink files are system outputs usually - only meant for Eyelinks DataViewer application to read. - These shouldn't need to be parsed. - - Parameters - ---------- - line : string - single line from Eyelink asc file - - Returns - ------- - bool : - True if any of the following strings that are - known to indicate a system message are in the line - - Notes - ----- - Examples of eyelink system messages: - - ;Sess:22Aug22;Tria:1;Tri2:False;ESNT:182BFE4C2F4; - - ;NTPT:182BFE55C96;SMSG:__NTP_CLOCK_SYNC__;DIFF:-1; - - !V APLAYSTART 0 1 library/audio - - !MODE RECORD CR 500 2 1 R - """ - return any(["!V" in line, "!MODE" in line, ";" in line]) - - -def _get_sfreq(rec_info): - """Get sampling frequency from Eyelink ASCII file. - - Parameters - ---------- - rec_info : list - the first list in self._event_lines['SAMPLES']. - The sfreq occurs after RATE: i.e. [..., RATE, 1000, ...]. - - Returns - ------- - sfreq : int | float - """ - for i, token in enumerate(rec_info): - if token == "RATE": - # sfreq is the first token after RATE - return rec_info[i + 1] - - -def _sort_by_time(df, col="time"): - df.sort_values(col, ascending=True, inplace=True) - df.reset_index(drop=True, inplace=True) - - -def _convert_times(df, first_samp, col="time"): - """Set initial time to 0, converts from ms to seconds in place. - - Parameters - ---------- - df pandas.DataFrame: - One of the dataframes in the self.dataframes dict. - - first_samp int: - timestamp of the first sample of the recording. This should - be the first sample of the first recording block. - col str (default 'time'): - column name to sort pandas.DataFrame by - - Notes - ----- - Each sample in an Eyelink file has a posix timestamp string. - Subtracts the "first" sample's timestamp from each timestamp. - The "first" sample is inferred to be the first sample of - the first recording block, i.e. the first "START" line. - """ - _sort_by_time(df, col) - for col in df.columns: - if col.endswith("time"): # 'time' and 'end_time' cols - df[col] -= first_samp - df[col] /= 1000 - if col in ["duration", "offset"]: - df[col] /= 1000 - - -def _fill_times( - df, - sfreq, - time_col="time", -): - """Fill missing timestamps if there are multiple recording blocks. - - Parameters - ---------- - df : pandas.DataFrame: - dataframe of the eyetracking data samples, BEFORE - _convert_times() is applied to the dataframe - - sfreq : int | float: - sampling frequency of the data - - time_col : str (default 'time'): - name of column with the timestamps (e.g. 9511881, 9511882, ...) - - Returns - ------- - %(df_return)s - - Notes - ----- - After _parse_recording_blocks, Files with multiple recording blocks will - have missing timestamps for the duration of the period between the blocks. - This would cause the occular annotations (i.e. blinks) to not line up with - the signal. - """ - pd = _check_pandas_installed() - - first, last = df[time_col].iloc[[0, -1]] - step = 1000 / sfreq - df[time_col] = df[time_col].astype(float) - new_times = pd.DataFrame( - np.arange(first, last + step / 2, step), columns=[time_col] - ) - return pd.merge_asof( - new_times, df, on=time_col, direction="nearest", tolerance=step / 10 - ) - - -def _find_overlaps(df, max_time=0.05): - """Merge left/right eye events with onset/offset diffs less than max_time. - - df : pandas.DataFrame - Pandas DataFrame with occular events (fixations, saccades, blinks) - max_time : float (default 0.05) - Time in seconds. Defaults to .05 (50 ms) - - Returns - ------- - DataFrame: %(df_return)s - :class:`pandas.DataFrame` specifying overlapped eye events, if any - Notes - ----- - The idea is to cumulative sum the boolean values for rows with onset and - offset differences (against the previous row) that are greater than the - max_time. If onset and offset diffs are less than max_time then no_overlap - will become False. Alternatively, if either the onset or offset diff is - greater than max_time, no_overlap becomes True. Cumulatively summing over - these boolean values will leave rows with no_overlap == False unchanged - and hence with the same group number. - """ - pd = _check_pandas_installed() - - df = df.copy() - df["overlap_start"] = df.sort_values("time")["time"].diff().lt(max_time) - - df["overlap_end"] = df["end_time"].diff().abs().lt(max_time) - - df["no_overlap"] = ~(df["overlap_end"] & df["overlap_start"]) - df["group"] = df["no_overlap"].cumsum() - - # now use groupby on 'group'. If one left and one right eye in group - # the new start/end times are the mean of the two eyes - ovrlp = pd.concat( - [ - pd.DataFrame(g[1].drop(columns="eye").mean()).T - if (len(g[1]) == 2) and (len(g[1].eye.unique()) == 2) - else g[1] # not an overlap, return group unchanged - for g in df.groupby("group") - ] - ) - # overlapped events get a "both" value in the "eye" col - if "eye" in ovrlp.columns: - ovrlp["eye"] = ovrlp["eye"].fillna("both") - else: - ovrlp["eye"] = "both" - tmp_cols = ["overlap_start", "overlap_end", "no_overlap", "group"] - return ovrlp.drop(columns=tmp_cols).reset_index(drop=True) - - @fill_doc def read_raw_eyelink( fname, @@ -354,14 +121,6 @@ def read_raw_eyelink( ``'BAD_ACQ_SKIP'``. """ fname = _check_fname(fname, overwrite="read", must_exist=True, name="fname") - extension = fname.suffix - if extension not in ".asc": - raise ValueError( - "This reader can only read eyelink .asc files." - f" Got extension {extension} instead. consult EyeLink" - " manual for converting EyeLink data format (.edf)" - " files to .asc format." - ) raw_eyelink = RawEyelink( fname, @@ -635,21 +394,12 @@ def _get_recording_datetime(self): if line.startswith("** DATE:"): dt_str = line.replace("** DATE:", "").strip() fmt = "%a %b %d %H:%M:%S %Y" - try: - # Eyelink measdate timestamps are timezone naive. - # Force datetime to be in UTC. - # Even though dt is probably in local time zone. - dt_naive = datetime.strptime(dt_str, fmt) - dt_aware = dt_naive.replace(tzinfo=tz) - self._meas_date = dt_aware - except Exception: - msg = ( - "Extraction of measurement date failed." - " Please report this as a github issue." - " The date is being set to None" - ) - logger.warning(msg) - break + # Eyelink measdate timestamps are timezone naive. + # Force datetime to be in UTC. + # Even though dt is probably in local time zone. + dt_naive = datetime.strptime(dt_str, fmt) + dt_aware = dt_naive.replace(tzinfo=tz) + self._meas_date = dt_aware def _href_to_radian(self, opposite, f=15_000): """Convert HREF eyegaze samples to radians. @@ -906,6 +656,18 @@ def _make_gap_annots(self, key="recording_blocks"): def _make_eyelink_annots(self, df_dict, create_annots, apply_offsets): """Create Annotations for each df in self.dataframes.""" + eye_ch_map = { + "L": ("xpos_left", "ypos_left", "pupil_left"), + "R": ("xpos_right", "ypos_right", "pupil_right"), + "both": ( + "xpos_left", + "ypos_left", + "pupil_left", + "xpos_right", + "ypos_right", + "pupil_right", + ), + } valid_descs = ["blinks", "saccades", "fixations", "messages"] msg = ( "create_annotations must be True or a list containing one or" @@ -929,18 +691,18 @@ def _make_eyelink_annots(self, df_dict, create_annots, apply_offsets): onsets = df["time"] durations = df["duration"] # Create annotations for both eyes - descriptions = f"{key[:-1]}_" + df["eye"] # i.e "blink_r" + descriptions = key[:-1] # i.e "blink", "fixation", "saccade" + if key == "blinks": + descriptions = "BAD_" + descriptions + ch_names = df["eye"].map(eye_ch_map).tolist() this_annot = Annotations( - onset=onsets, duration=durations, description=descriptions + onset=onsets, + duration=durations, + description=descriptions, + ch_names=ch_names, ) elif (key in ["messages"]) and (key in descs): if apply_offsets: - if df["offset"].isnull().all(): - logger.warning( - "There are no offsets for the messages" - f" in {self.fname}. Not applying any" - " offset" - ) # If df['offset] is all NaNs, time is not changed onsets = df["time"] + df["offset"].fillna(0) else: diff --git a/mne/io/eyelink/tests/test_eyelink.py b/mne/io/eyelink/tests/test_eyelink.py index c16970a26dc..ddc2b4b7a1a 100644 --- a/mne/io/eyelink/tests/test_eyelink.py +++ b/mne/io/eyelink/tests/test_eyelink.py @@ -8,6 +8,19 @@ from mne.io.pick import _DATA_CH_TYPES_SPLIT from mne.utils import _check_pandas_installed, requires_pandas +MAPPING = { + "left": ["xpos_left", "ypos_left", "pupil_left"], + "right": ["xpos_right", "ypos_right", "pupil_right"], + "both": [ + "xpos_left", + "ypos_left", + "pupil_left", + "xpos_right", + "ypos_right", + "pupil_right", + ], +} + testing_path = data_path(download=False) fname = testing_path / "eyetrack" / "test_eyelink.asc" fname_href = testing_path / "eyetrack" / "test_eyelink_HREF.asc" @@ -26,21 +39,42 @@ def test_eyetrack_not_data_ch(): @requires_testing_data @requires_pandas @pytest.mark.parametrize( - "fname, create_annotations, find_overlaps", + "fname, create_annotations, find_overlaps, apply_offsets", [ - (fname, False, False), - (fname, False, False), - (fname, True, False), - (fname, True, True), - (fname, ["fixations", "saccades", "blinks"], True), + (fname, False, False, False), + ( + fname, + False, + False, + False, + ), + ( + fname, + True, + False, + False, + ), + ( + fname, + True, + True, + True, + ), + ( + fname, + ["fixations", "saccades", "blinks"], + True, + False, + ), ], ) -def test_eyelink(fname, create_annotations, find_overlaps): +def test_eyelink(fname, create_annotations, find_overlaps, apply_offsets): """Test reading eyelink asc files.""" raw = read_raw_eyelink( fname, create_annotations=create_annotations, find_overlaps=find_overlaps, + apply_offsets=apply_offsets, ) # First, tests that shouldn't change based on function arguments @@ -73,19 +107,14 @@ def test_eyelink(fname, create_annotations, find_overlaps): ) # There is a blink in this data at 8.9 seconds cond = (df["time_in_sec"] > 8.899) & (df["time_in_sec"] < 8.95) - assert df[cond]["description"].values[0].startswith("blink") - if find_overlaps is True: - df = raw.annotations.to_data_frame() - # these should both be True so long as _find_overlaps is not - # majorly refactored. - assert "blink_L" in df["description"].unique() - assert "blink_both" in df["description"].unique() + assert df[cond]["description"].values[0].startswith("BAD_blink") + + # Check that the annotation ch_names are set correctly + assert np.array_equal(raw.annotations[0]["ch_names"], MAPPING["both"]) + if isinstance(create_annotations, list) and find_overlaps: # the last pytest parametrize condition should hit this - df = raw.annotations.to_data_frame() - # Rows 0, 1, 2 should be 'fixation_both', 'saccade_both', 'blink_both' - for i, label in zip([0, 1, 2], ["fixation", "saccade", "blink"]): - assert df["description"].iloc[i] == f"{label}_both" + assert np.array_equal(raw.annotations[0]["ch_names"], MAPPING["both"]) @requires_testing_data diff --git a/tutorials/io/70_reading_eyetracking_data.py b/tutorials/io/70_reading_eyetracking_data.py index 0283ab7b52c..880fa1169e3 100644 --- a/tutorials/io/70_reading_eyetracking_data.py +++ b/tutorials/io/70_reading_eyetracking_data.py @@ -46,10 +46,10 @@ Supported measurement types from Eyelink files include eye position, pupil size, saccadic velocity, resolution, and head position (for recordings -collected in remote mode). Eyelink files often report occular events (blinks, +collected in remote mode). Eyelink files often report ocular events (blinks, saccades, and fixations), MNE will store these events as `mne.Annotations`. -For More information on the various measurement types that can be present in -Eyelink files, read below. +Blink annotation descriptions will be ``'BAD_blink'``. For more information +on the various measurement types that can be present in Eyelink files. read below. Eye Position Data ----------------- diff --git a/tutorials/preprocessing/90_eyetracking_data.py b/tutorials/preprocessing/90_eyetracking_data.py index 07b6846f768..b394cb72e2c 100644 --- a/tutorials/preprocessing/90_eyetracking_data.py +++ b/tutorials/preprocessing/90_eyetracking_data.py @@ -22,13 +22,6 @@ # First we will load an eye tracker recording from SR research's proprietary # ``'.asc'`` file format. # -# By default, Eyelink files will output events for occular events (blinks, -# saccades, fixations), and experiment messages. MNE will store these events -# as `mne.Annotations`. If we are only interested in certain event types from -# the Eyelink file, we can select for these using the ``'create_annotations'`` -# argument of `mne.io.read_raw_eyelink`. Here, we will only create annotations -# for blinks, and experiment messages. -# # The info structure tells us we loaded a monocular recording with 2 # ``'eyegaze'``, channels (X/Y), 1 ``'pupil'`` channel, and 1 ``'stim'`` # channel. @@ -43,6 +36,26 @@ raw = read_raw_eyelink(eyelink_fname, create_annotations=["blinks", "messages"]) raw.crop(tmin=0, tmax=146) +# %% +# Ocular annotations +# ------------------ +# By default, Eyelink files will output events for ocular events (blinks, +# saccades, fixations), and experiment messages. MNE will store these events +# as `mne.Annotations`. Ocular annotations contain channel information, in the +# ``'ch_names'``` key. This means that we can see which eye an ocular event occurred in: + +print(raw.annotations[0]) # a blink in the right eye + +# %% +# If we are only interested in certain event types from +# the Eyelink file, we can select for these using the ``'create_annotations'`` +# argument of `mne.io.read_raw_eyelink`. above, we only created annotations +# for blinks, and experiment messages. +# +# Note that ``'blink'`` annotations are read in as ``'BAD_blink'``, and MNE will treat +# these as bad segments of data. This means that blink periods will be dropped during +# epoching by default. + # %% # Checking the calibration # ------------------------ @@ -99,7 +112,8 @@ # `mne.Annotations`. # # In the example data, the DIN channel contains the onset of light flashes on -# the screen. We now extract these events to visualize the pupil response. +# the screen. We now extract these events to visualize the pupil response. We will use +# these later in this tutorial. events = find_events(raw, "DIN", shortest_event=1, min_duration=0.02, uint_cast=True) event_dict = {"flash": 3} From ee767ddffbcde5869e4740548120c9055c8e41ed Mon Sep 17 00:00:00 2001 From: Gennadiy <7503709+Genuster@users.noreply.github.com> Date: Fri, 23 Jun 2023 20:57:13 +0300 Subject: [PATCH 04/26] Fix resampling to the identical frequency (#11736) --- doc/changes/latest.inc | 1 + mne/filter.py | 18 ++++++++++++++++-- mne/io/base.py | 10 +++++++--- mne/io/tests/test_raw.py | 8 ++++++++ mne/source_estimate.py | 7 ++++++- mne/tests/test_evoked.py | 8 ++++++++ mne/tests/test_source_estimate.py | 9 +++++++++ 7 files changed, 55 insertions(+), 6 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index dc14c20b09b..31fa7af0bad 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -24,6 +24,7 @@ Current (1.5.dev0) Enhancements ~~~~~~~~~~~~ - Add ``cmap`` argument for the :func:`mne.viz.plot_sensors` (:gh:`11720` by :newcontrib:`Gennadiy Belonosov`) +- Return unmodified instance if new sampling frequency is identical to the original in :meth:`mne.io.Raw.resample`, :meth:`mne.Epochs.resample`, :meth:`mne.Evoked.resample` and :meth:`mne.SourceEstimate.resample` (:gh:`11736` by :newcontrib:`Gennadiy Belonosov`) - When failing to locate a file, we now print the full path in quotation marks to help spot accidentally added trailing spaces (:gh:`11718` by `Richard Höchenberger`_) - Add standard montage lookup table for ``easycap-M43`` (:gh:`11744` by :newcontrib:`Diptyajit Das`) - Added :class:`mne.preprocessing.eyetracking.Calibration` to store eye-tracking calibration info, and :func:`mne.preprocessing.eyetracking.read_eyelink_calibration` to read calibration data from EyeLink systems (:gh:`11719` by `Scott Huberty`_) diff --git a/mne/filter.py b/mne/filter.py index 5277a1fd502..40d7e8b38ab 100644 --- a/mne/filter.py +++ b/mne/filter.py @@ -2435,6 +2435,18 @@ def float_array(c): ) +def _check_resamp_noop(sfreq, o_sfreq, rtol=1e-6): + if np.isclose(sfreq, o_sfreq, atol=0, rtol=rtol): + logger.info( + ( + f"Sampling frequency of the instance is already {sfreq}, " + "returning unmodified." + ) + ) + return True + return False + + class FilterMixin: """Object for Epoch/Evoked filtering.""" @@ -2684,10 +2696,12 @@ def resample( # mne.io.base.BaseRaw overrides this method assert isinstance(self, (BaseEpochs, Evoked)) - _check_preload(self, "inst.resample") - sfreq = float(sfreq) o_sfreq = self.info["sfreq"] + if _check_resamp_noop(sfreq, o_sfreq): + return self + + _check_preload(self, "inst.resample") self._data = resample( self._data, sfreq, o_sfreq, npad, window=window, n_jobs=n_jobs, pad=pad ) diff --git a/mne/io/base.py b/mne/io/base.py index f44dce8daa5..9c1ea43a98d 100644 --- a/mne/io/base.py +++ b/mne/io/base.py @@ -1316,6 +1316,13 @@ def resample( ``self.load_data()``, but this increases memory requirements. The resulting raw object will have the data loaded into memory. """ + from ..filter import _check_resamp_noop + + sfreq = float(sfreq) + o_sfreq = float(self.info["sfreq"]) + if _check_resamp_noop(sfreq, o_sfreq): + return self + # When no event object is supplied, some basic detection of dropped # events is performed to generate a warning. Finding events can fail # for a variety of reasons, e.g. if no stim channel is present or it is @@ -1327,9 +1334,6 @@ def resample( except Exception: pass - sfreq = float(sfreq) - o_sfreq = float(self.info["sfreq"]) - offsets = np.concatenate(([0], np.cumsum(self._raw_lengths))) # set up stim channel processing diff --git a/mne/io/tests/test_raw.py b/mne/io/tests/test_raw.py index 27ac8001b5b..9e11202a2d1 100644 --- a/mne/io/tests/test_raw.py +++ b/mne/io/tests/test_raw.py @@ -972,3 +972,11 @@ def test_get_data_tmin_tmax(): with pytest.raises(TypeError, match="tmax must be .* float"): raw.get_data(tmax=[1, 2]) + + +def test_resamp_noop(): + """Tests resampling doesn't affect data if sfreq is identical.""" + raw = read_raw_fif(raw_fname) + data_before = raw.get_data() + data_after = raw.resample(sfreq=raw.info["sfreq"]).get_data() + assert_array_equal(data_before, data_after) diff --git a/mne/source_estimate.py b/mne/source_estimate.py index 9df886c15de..6bf9934f9ea 100644 --- a/mne/source_estimate.py +++ b/mne/source_estimate.py @@ -849,11 +849,16 @@ def resample(self, sfreq, npad="auto", window="boxcar", n_jobs=None, verbose=Non Note that the sample rate of the original data is inferred from tstep. """ + from .filter import _check_resamp_noop + + o_sfreq = 1.0 / self.tstep + if _check_resamp_noop(sfreq, o_sfreq): + return self + # resampling in sensor instead of source space gives a somewhat # different result, so we don't allow it self._remove_kernel_sens_data_() - o_sfreq = 1.0 / self.tstep data = self.data if data.dtype == np.float32: data = data.astype(np.float64) diff --git a/mne/tests/test_evoked.py b/mne/tests/test_evoked.py index 2298f99f2fb..08be432d67a 100644 --- a/mne/tests/test_evoked.py +++ b/mne/tests/test_evoked.py @@ -410,6 +410,14 @@ def test_evoked_resample(tmp_path): assert ave_new.info["lowpass"] == 25.0 +def test_evoked_resamp_noop(): + """Tests resampling doesn't affect data if sfreq is identical.""" + ave = read_evokeds(fname, 0) + data_before = ave.data + data_after = ave.resample(sfreq=ave.info["sfreq"]).data + assert_array_equal(data_before, data_after) + + def test_evoked_filter(): """Test filtering evoked data.""" # this is mostly a smoke test as the Epochs and raw tests are more complete diff --git a/mne/tests/test_source_estimate.py b/mne/tests/test_source_estimate.py index a2576f0d0a3..3a48df0edba 100644 --- a/mne/tests/test_source_estimate.py +++ b/mne/tests/test_source_estimate.py @@ -619,6 +619,15 @@ def test_stc_methods(): assert_array_almost_equal(stc_new.data, stc.data, 5) +@testing.requires_testing_data +def test_stc_resamp_noop(): + """Tests resampling doesn't affect data if sfreq is identical.""" + stc = read_source_estimate(fname_stc) + data_before = stc.data + data_after = stc.resample(sfreq=1.0 / stc.tstep).data + assert_array_equal(data_before, data_after) + + @testing.requires_testing_data def test_center_of_mass(): """Test computing the center of mass on an stc.""" From 1388d8440cd8347f8c86bdf2f3e91fae195d0359 Mon Sep 17 00:00:00 2001 From: Clemens Brunner Date: Fri, 23 Jun 2023 23:52:48 +0200 Subject: [PATCH 05/26] Plot epochs.events by default (#11445) Co-authored-by: Daniel McCloy --- doc/changes/latest.inc | 1 + mne/tests/test_epochs.py | 2 +- mne/viz/epochs.py | 49 ++++++++++++------ mne/viz/tests/test_epochs.py | 98 ++++++++++++++++++++++++------------ mne/viz/utils.py | 2 +- 5 files changed, 104 insertions(+), 48 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 31fa7af0bad..66fac3f4fa4 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -47,4 +47,5 @@ Bugs API changes ~~~~~~~~~~~ - The ``baseline`` argument can now be array-like (e.g. ``list``, ``tuple``, ``np.ndarray``, ...) instead of only a ``tuple`` (:gh:`11713` by `Clemens Brunner`_) +- The ``events`` and ``event_id`` parameters of `:meth:`Epochs.plot() ` now accept boolean values; see docstring for details (:gh:`11445` by `Daniel McCloy`_ and `Clemens Brunner`_) - Deprecated ``gap_description`` keyword argument of :func:`mne.io.read_raw_eyelink`, which will be removed in mne version 1.6, in favor of using :meth:`mne.Annotations.rename` (:gh:`11719` by `Scott Huberty`_) diff --git a/mne/tests/test_epochs.py b/mne/tests/test_epochs.py index 4c248e2d2dd..a7e339f55ec 100644 --- a/mne/tests/test_epochs.py +++ b/mne/tests/test_epochs.py @@ -3309,7 +3309,7 @@ def test_array_epochs(tmp_path, browser_backend): assert_array_equal(epochs.events, epochs2.events) # plotting - epochs[0].plot() + epochs[0].plot(events=False) # indexing assert_array_equal(np.unique(epochs["1"].events[:, 2]), np.array([1])) diff --git a/mne/viz/epochs.py b/mne/viz/epochs.py index c41f13b2416..9cf9f785e65 100644 --- a/mne/viz/epochs.py +++ b/mne/viz/epochs.py @@ -806,10 +806,11 @@ def plot_epochs( title : str | None The title of the window. If None, the event names (from ``epochs.event_id``) will be displayed. Defaults to None. - events : None | array, shape (n_events, 3) + events : bool | array, shape (n_events, 3) Events to show with vertical bars. You can use `~mne.viz.plot_events` as a legend for the colors. By default, the coloring scheme is the - same. Defaults to ``None``. + same. ``True`` plots ``epochs.events``. Defaults to ``False`` (do not + plot events). .. warning:: If the epochs have been resampled, the events no longer align with the data. @@ -858,12 +859,12 @@ def plot_epochs( .. versionadded:: 0.24.0 epoch_colors : list of (n_epochs) list (of n_channels) | None Colors to use for individual epochs. If None, use default colors. - event_id : dict | None - Dictionary of event labels (e.g. 'aud_l') as keys and associated event - integers as values. Useful when ``events`` contains event numbers not - present in ``epochs.event_id`` (e.g., because of event subselection). - Values in ``event_id`` will take precedence over those in - ``epochs.event_id`` when there are overlapping keys. + event_id : bool | dict + Determines to label the event markers on the plot. If ``True``, uses + ``epochs.event_id``. If ``False``, uses integer event codes instead of IDs. + If a ``dict`` is passed, uses its *keys* as event labels on the plot for + entries whose *values* are integer codes for events being drawn. Ignored if + ``events=False``. .. versionadded:: 0.20 %(group_by_browse)s @@ -918,8 +919,16 @@ def plot_epochs( unit_scalings = _handle_default("scalings", None) decim, picks_data = _handle_decim(epochs.info.copy(), decim, None) noise_cov = _check_cov(noise_cov, epochs.info) - event_id_rev = {v: k for k, v in (event_id or {}).items()} _check_option("group_by", group_by, ("selection", "position", "original", "type")) + # handle event labels + if not event_id: + event_id = dict() + else: + if not hasattr(event_id, "keys"): + event_id = dict() + # TODO: when min py=3.9, change to `epochs.event_id | event_id` + event_id = dict(**epochs.event_id, **event_id) + event_id_rev = {v: k for k, v in event_id.items()} # validate epoch_colors _validate_type(epoch_colors, (list, None), "epoch_colors") if epoch_colors is not None: @@ -946,12 +955,26 @@ def plot_epochs( boundary_times = np.arange(len(epochs) + 1) * len(epochs.times) / sfreq # events - if events is not None: + if events is None: + warn( + "The current default events=None is deprecated and will change to " + "events=True in MNE 1.6. Set events=False to suppress this warning.", + category=FutureWarning, + ) + events = False + if events is False: + event_nums = None + event_times = None + else: # True or ndarray + if events is True: # use epochs.events + events = epochs.events event_nums = events[:, 2] event_samps = events[:, 0] epoch_n_samps = len(epochs.times) # handle overlapping epochs (each event may show up in multiple places) - boundaries = epochs.events[:, [0]] + np.array([-1, 1]) * epochs.time_as_index(0) + boundaries = epochs.events[:, [0]] + np.array([-1, 1]) * epochs.time_as_index( + [0, epochs.tmax] + ) in_bounds = np.logical_and( boundaries[:, [0]] <= event_samps, event_samps < boundaries[:, [1]] ) @@ -973,9 +996,7 @@ def plot_epochs( event_numbers.extend([num] * len(_ixs)) event_nums = np.array(event_numbers) event_times = np.array(event_times) - else: - event_nums = None - event_times = None + event_color_dict = _make_event_color_dict(event_color, events, event_id) # determine trace order diff --git a/mne/viz/tests/test_epochs.py b/mne/viz/tests/test_epochs.py index c87755e37b7..d911c5ba8c9 100644 --- a/mne/viz/tests/test_epochs.py +++ b/mne/viz/tests/test_epochs.py @@ -16,11 +16,14 @@ from mne.event import make_fixed_length_events from mne.viz import plot_drop_log +# TODO: deprecation cycle handling, remove this and all `**ev` after 1.5 release +ev = dict(events=False) + def test_plot_epochs_not_preloaded(epochs_unloaded, browser_backend): """Test plotting non-preloaded epochs.""" assert epochs_unloaded._data is None - epochs_unloaded.plot() + epochs_unloaded.plot(**ev) assert epochs_unloaded._data is None @@ -29,7 +32,7 @@ def test_plot_epochs_basic(epochs, epochs_full, noise_cov_io, capsys, browser_ba assert len(epochs.events) == 1 with epochs.info._unlock(): epochs.info["lowpass"] = 10.0 # allow heavy decim during plotting - fig = epochs.plot(scalings=None, title="Epochs") + fig = epochs.plot(**ev, scalings=None, title="Epochs") ticks = fig._get_ticklabels("x") assert ticks == ["2"] browser_backend._close_all() @@ -38,38 +41,38 @@ def test_plot_epochs_basic(epochs, epochs_full, noise_cov_io, capsys, browser_ba assert noise_cov_io["bads"] == [] assert epochs.info["bads"] == [] # all good with pytest.warns(RuntimeWarning, match="projection"): - epochs.plot(noise_cov=noise_cov_io) + epochs.plot(**ev, noise_cov=noise_cov_io) browser_backend._close_all() # add a channel to the epochs.info['bads'] epochs.info["bads"] = [epochs.ch_names[0]] with pytest.warns(RuntimeWarning, match="projection"): - epochs.plot(noise_cov=noise_cov_io) + epochs.plot(**ev, noise_cov=noise_cov_io) browser_backend._close_all() # add a channel to cov['bads'] noise_cov_io["bads"] = [epochs.ch_names[1]] with pytest.warns(RuntimeWarning, match="projection"): - epochs.plot(noise_cov=noise_cov_io) + epochs.plot(**ev, noise_cov=noise_cov_io) browser_backend._close_all() # have a data channel missing from the covariance noise_cov_io["names"] = noise_cov_io["names"][:306] noise_cov_io["data"] = noise_cov_io["data"][:306][:306] with pytest.warns(RuntimeWarning, match="projection"): - epochs.plot(noise_cov=noise_cov_io) + epochs.plot(**ev, noise_cov=noise_cov_io) browser_backend._close_all() # other options - fig = epochs[0].plot(picks=[0, 2, 3], scalings=None) + fig = epochs[0].plot(**ev, picks=[0, 2, 3], scalings=None) fig._fake_keypress("escape") with pytest.raises(ValueError, match="No appropriate channels found"): - epochs.plot(picks=[]) + epochs.plot(**ev, picks=[]) # gh-5906 assert len(epochs_full) == 7 epochs_full.info["bads"] = [epochs_full.ch_names[0]] capsys.readouterr() # test title error handling with pytest.raises(TypeError, match="title must be None or a string, got"): - epochs_full.plot(title=7) + epochs_full.plot(**ev, title=7) # test auto-generated title, and selection mode - epochs_full.plot(group_by="selection", title="") + epochs_full.plot(**ev, group_by="selection", title="") @pytest.mark.parametrize( @@ -77,24 +80,26 @@ def test_plot_epochs_basic(epochs, epochs_full, noise_cov_io, capsys, browser_ba ) def test_plot_epochs_scalings(epochs, scalings, browser_backend): """Test the valid options for scalings.""" - epochs.plot(scalings=scalings) + epochs.plot(**ev, scalings=scalings) def test_plot_epochs_colors(epochs, browser_backend): """Test epoch_colors, for compatibility with autoreject.""" epoch_colors = [["r"] * len(epochs.ch_names) for _ in range(len(epochs.events))] - epochs.plot(epoch_colors=epoch_colors) + epochs.plot(**ev, epoch_colors=epoch_colors) with pytest.raises(ValueError, match="length equal to the number of epo"): - epochs.plot(epoch_colors=[["r"], ["b"]]) # epochs obj has only 1 epoch + # epochs obj has only 1 epoch + epochs.plot(epoch_colors=[["r"], ["b"]]) with pytest.raises(ValueError, match=r"epoch colors for epoch \d+ has"): - epochs.plot(epoch_colors=[["r"]]) # need 1 color for each channel + # need 1 color for each channel + epochs.plot(**ev, epoch_colors=[["r"]]) # also test event_color - epochs.plot(event_color="b") + epochs.plot(**ev, event_color="b") def test_plot_epochs_scale_bar(epochs, browser_backend): """Test scale bar for epochs.""" - fig = epochs.plot() + fig = epochs.plot(**ev) texts = fig._get_scale_bar_texts() # mag & grad in this instance if browser_backend.name == "pyqtgraph": @@ -108,7 +113,7 @@ def test_plot_epochs_scale_bar(epochs, browser_backend): def test_plot_epochs_clicks(epochs, epochs_full, capsys, browser_backend): """Test plot_epochs mouse interaction.""" - fig = epochs.plot(events=epochs.events) + fig = epochs.plot(events=True) x = fig.mne.traces[0].get_xdata()[3] y = fig.mne.traces[0].get_ydata()[3] n_epochs = len(epochs) @@ -126,7 +131,7 @@ def test_plot_epochs_clicks(epochs, epochs_full, capsys, browser_backend): assert n_epochs - 1 == len(epochs) # test marking bad channels # need more than 1 epoch this time - fig = epochs_full.plot(n_epochs=3) + fig = epochs_full.plot(**ev, n_epochs=3) first_ch = fig._get_ticklabels("y")[0] assert first_ch not in fig.mne.info["bads"] fig._click_ch_name(ch_index=0, button=1) # click ch name to mark bad @@ -147,7 +152,7 @@ def test_plot_epochs_clicks(epochs, epochs_full, capsys, browser_backend): fig._close_event() # XXX workaround, MPL Agg doesn't trigger close event assert len(epochs_full) == 6 # test rightclick → image plot - fig = epochs_full.plot() + fig = epochs_full.plot(**ev) fig._click_ch_name(ch_index=0, button=3) # show image plot assert len(fig.mne.child_figs) == 1 # test scroll wheel @@ -159,7 +164,7 @@ def test_plot_epochs_keypresses(epochs_full, browser_backend): """Test plot_epochs keypress interaction.""" # we need more than 1 epoch epochs_full.drop_bad(dict(mag=4e-12)) # for histogram plot coverage - fig = epochs_full.plot(n_epochs=3) + fig = epochs_full.plot(**ev, n_epochs=3) # make sure green vlines are visible first (for coverage) sample_idx = len(epochs_full.times) // 2 # halfway through the first epoch x = fig.mne.traces[0].get_xdata()[sample_idx] @@ -200,20 +205,49 @@ def test_plot_epochs_keypresses(epochs_full, browser_backend): fig._fake_click([x, y], xform="data", button=3) # remove vlines -def test_plot_overlapping_epochs_with_events(browser_backend): +def _get_event_lines_and_texts(fig): + """Get event lines and labels (helper function).""" + lines = fig.mne.event_lines + texts = fig.mne.event_texts + if hasattr(lines, "get_segments"): # matplotlib backend + lines = lines.get_segments() + texts = [t.get_text() for t in texts] + return lines, texts + + +@pytest.mark.parametrize( + "event_id,expected_texts", + [(False, set("123")), (True, set("abc")), (dict(f=1), set("fbc"))], +) +def test_plot_overlapping_epochs_with_events(browser_backend, event_id, expected_texts): """Test drawing of event lines in overlapping epochs.""" data = np.zeros(shape=(3, 2, 100)) # 3 epochs, 2 channels, 100 samples sfreq = 100 info = create_info(ch_names=("a", "b"), ch_types=("misc", "misc"), sfreq=sfreq) # 90% overlap, so all 3 events should appear in all 3 epochs when plotted: - events = np.column_stack(([50, 60, 70], [0, 0, 0], [1, 2, 3])) - epochs = EpochsArray(data, info, tmin=-0.5, events=events) - fig = epochs.plot(events=events, picks="misc") + events = np.column_stack(([40, 50, 60], [0, 0, 0], [1, 2, 3])) + epochs = EpochsArray( + data, info, tmin=-0.4, events=events, event_id=dict(a=1, b=2, c=3) + ) + fig = epochs.plot(events=events, picks="misc", event_id=event_id) + # check that the event lines are there and the labels are correct + lines, texts = _get_event_lines_and_texts(fig) + assert len(lines) == len(epochs) * len(events) + # TODO: Qt browser doesn't show event names, only integers + if browser_backend.name == "matplotlib": + assert set(texts) == expected_texts + # plot one epoch with its defining event plus events at its first & last sample + # (regression test for https://mne.discourse.group/t/6334) + events = np.row_stack(([[0, 0, 4]], events[[0]], [[99, 0, 4]])) + fig = epochs[0].plot(events=events, picks="misc", event_id=event_id) + expected_texts.add("4") + for text in ("2", "3", "b", "c"): + expected_texts.discard(text) + lines, texts = _get_event_lines_and_texts(fig) + assert len(lines) == len(events) + # TODO: Qt browser doesn't show event names, only integers if browser_backend.name == "matplotlib": - n_event_lines = len(fig.mne.event_lines.get_segments()) - else: - n_event_lines = len(fig.mne.event_lines) - assert n_event_lines == 9 + assert set(texts) == expected_texts def test_epochs_plot_sensors(epochs): @@ -227,7 +261,7 @@ def test_plot_epochs_nodata(browser_backend): info = create_info(2, 1000.0, "stim") epochs = EpochsArray(data, info) with pytest.raises(ValueError, match="consider passing picks explicitly"): - epochs.plot() + epochs.plot(**ev) @pytest.mark.slowtest @@ -413,11 +447,11 @@ def test_plot_epochs_ctf(raw_ctf, browser_backend): ) evts = make_fixed_length_events(raw_ctf) epochs = Epochs(raw_ctf, evts, preload=True) - epochs.plot() + epochs.plot(**ev) browser_backend._close_all() # test butterfly - fig = epochs.plot(butterfly=True) + fig = epochs.plot(**ev, butterfly=True) # leave fullscreen testing to Raw / _figure abstraction (too annoying here) keys = ( "b", @@ -476,4 +510,4 @@ def test_plot_epochs_selection_butterfly(raw, browser_backend): events = make_fixed_length_events(raw)[:1] epochs = Epochs(raw, events, tmin=0, tmax=0.5, preload=True, baseline=None) assert len(epochs) == 1 - epochs.plot(group_by="selection", butterfly=True) + epochs.plot(**ev, group_by="selection", butterfly=True) diff --git a/mne/viz/utils.py b/mne/viz/utils.py index e9802e872d2..b62ef71da5d 100644 --- a/mne/viz/utils.py +++ b/mne/viz/utils.py @@ -461,7 +461,7 @@ def _make_event_color_dict(event_color, events=None, event_id=None): new_dict[key] = value return new_dict elif event_color is None: # make a dict from color cycle - uniq_events = set() if events is None else np.unique(events[:, 2]) + uniq_events = set() if events is False else np.unique(events[:, 2]) return _handle_event_colors(event_color, uniq_events, event_id) else: # if event_color is a MPL color-like thing, use it for all events return defaultdict(lambda: event_color) From 52d1e3e016e5caadb040eb3fa30790a8895d8732 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sun, 25 Jun 2023 18:30:30 -0400 Subject: [PATCH 06/26] DEP: Fix post-deprecation warnings (#11752) --- mne/viz/epochs.py | 19 ++++++++++++++----- mne/viz/tests/test_epochs.py | 7 ++++++- tutorials/clinical/20_seeg.py | 2 +- tutorials/epochs/10_epochs_overview.py | 2 +- tutorials/epochs/20_visualize_epochs.py | 5 ++--- tutorials/epochs/30_epochs_metadata.py | 2 +- tutorials/evoked/40_whitened.py | 4 ++-- tutorials/simulation/10_array_objs.py | 2 +- tutorials/simulation/80_dics.py | 2 +- 9 files changed, 29 insertions(+), 16 deletions(-) diff --git a/mne/viz/epochs.py b/mne/viz/epochs.py index 9cf9f785e65..5e4a4d76874 100644 --- a/mne/viz/epochs.py +++ b/mne/viz/epochs.py @@ -921,13 +921,22 @@ def plot_epochs( noise_cov = _check_cov(noise_cov, epochs.info) _check_option("group_by", group_by, ("selection", "position", "original", "type")) # handle event labels - if not event_id: + _validate_type(event_id, (bool, dict, None), "event_id") + if not event_id: # False or None event_id = dict() else: - if not hasattr(event_id, "keys"): - event_id = dict() - # TODO: when min py=3.9, change to `epochs.event_id | event_id` - event_id = dict(**epochs.event_id, **event_id) + # make our own copy of the dict + event_id = dict() if event_id is True else event_id.copy() # to dict + # TODO: when min py=3.9, change to `epochs.event_id | event_id` (maybe). + # Passed-in event_id should take precedence, i.e., not replace existing + # keys *or* repeat existing values. For example, if epochs.event_id has + # a=1 and passed-in event_id has f=1, the second takes precedence. + event_values = set(event_id.values()) + event_id.update( + (k, v) + for k, v in epochs.event_id.items() + if k not in event_id and v not in event_values + ) event_id_rev = {v: k for k, v in event_id.items()} # validate epoch_colors _validate_type(epoch_colors, (list, None), "epoch_colors") diff --git a/mne/viz/tests/test_epochs.py b/mne/viz/tests/test_epochs.py index d911c5ba8c9..aeb2bcd22ce 100644 --- a/mne/viz/tests/test_epochs.py +++ b/mne/viz/tests/test_epochs.py @@ -217,7 +217,12 @@ def _get_event_lines_and_texts(fig): @pytest.mark.parametrize( "event_id,expected_texts", - [(False, set("123")), (True, set("abc")), (dict(f=1), set("fbc"))], + [ + (False, set("123")), + (True, set("abc")), + (dict(f=1), set("fbc")), + (dict(a=1), set("abc")), + ], ) def test_plot_overlapping_epochs_with_events(browser_backend, event_id, expected_texts): """Test drawing of event lines in overlapping epochs.""" diff --git a/tutorials/clinical/20_seeg.py b/tutorials/clinical/20_seeg.py index fcbe16731d6..eeec6fd8f8e 100644 --- a/tutorials/clinical/20_seeg.py +++ b/tutorials/clinical/20_seeg.py @@ -197,7 +197,7 @@ # %% # Next, we'll get the epoch data and plot its amplitude over time. -epochs.plot() +epochs.plot(events=True) # %% # We can visualize this raw data on the ``fsaverage`` brain (in MNI space) as diff --git a/tutorials/epochs/10_epochs_overview.py b/tutorials/epochs/10_epochs_overview.py index f72389cc389..17246906ecb 100644 --- a/tutorials/epochs/10_epochs_overview.py +++ b/tutorials/epochs/10_epochs_overview.py @@ -183,7 +183,7 @@ # The :class:`~mne.Epochs` object can be visualized (and browsed interactively) # using its :meth:`~mne.Epochs.plot` method: -epochs.plot(n_epochs=10) +epochs.plot(n_epochs=10, events=True) # %% # Notice that the individual epochs are sequentially numbered along the bottom diff --git a/tutorials/epochs/20_visualize_epochs.py b/tutorials/epochs/20_visualize_epochs.py index acdbd96076a..a66a606916f 100644 --- a/tutorials/epochs/20_visualize_epochs.py +++ b/tutorials/epochs/20_visualize_epochs.py @@ -118,9 +118,8 @@ # Note that these field maps illustrate aspects of the signal that *have # already been removed* (because projectors in `~mne.io.Raw` data are # applied by default when epoching, and because we called -# `~mne.Epochs.apply_proj` after adding additional ECG projectors from -# file). You can check this by examining the ``'active'`` field of the -# projectors: +# `~mne.Epochs.apply_proj` after adding additional ECG projectors from file). +# You can check this by examining the ``'active'`` field of the projectors: print(all(proj["active"] for proj in epochs.info["projs"])) diff --git a/tutorials/epochs/30_epochs_metadata.py b/tutorials/epochs/30_epochs_metadata.py index e207e0a2059..8754a76e561 100644 --- a/tutorials/epochs/30_epochs_metadata.py +++ b/tutorials/epochs/30_epochs_metadata.py @@ -120,7 +120,7 @@ # plotting: words = ["typhoon", "bungalow", "colossus", "drudgery", "linguist", "solenoid"] -epochs["WORD in {}".format(words)].plot(n_channels=29) +epochs["WORD in {}".format(words)].plot(n_channels=29, events=True) # %% # Notice that in this dataset, each "condition" (A.K.A., each word) occurs only diff --git a/tutorials/evoked/40_whitened.py b/tutorials/evoked/40_whitened.py index 625701e2f7a..3f63b03350e 100644 --- a/tutorials/evoked/40_whitened.py +++ b/tutorials/evoked/40_whitened.py @@ -55,8 +55,8 @@ # %% # Epochs with whitening # --------------------- -epochs.plot() -epochs.plot(noise_cov=noise_cov) +epochs.plot(events=True) +epochs.plot(noise_cov=noise_cov, events=True) # %% # Evoked data with whitening diff --git a/tutorials/simulation/10_array_objs.py b/tutorials/simulation/10_array_objs.py index 420469b6b56..19f887c4417 100644 --- a/tutorials/simulation/10_array_objs.py +++ b/tutorials/simulation/10_array_objs.py @@ -148,7 +148,7 @@ ) simulated_epochs = mne.EpochsArray(data, info) -simulated_epochs.plot(picks="misc", show_scrollbars=False) +simulated_epochs.plot(picks="misc", show_scrollbars=False, events=True) # %% # Since we did not supply an events array, the `~mne.EpochsArray` constructor diff --git a/tutorials/simulation/80_dics.py b/tutorials/simulation/80_dics.py index 2b7a06d5db0..8791cdba514 100644 --- a/tutorials/simulation/80_dics.py +++ b/tutorials/simulation/80_dics.py @@ -216,7 +216,7 @@ def coh_signal_gen(): mne.read_vectorview_selection("Left-frontal"), ordered=False, ) -epochs.plot(picks=picks) +epochs.plot(picks=picks, events=True) # %% # Power mapping From be5f51aa8f5fdaec2b915be0346e163ab7363103 Mon Sep 17 00:00:00 2001 From: Mathieu Scheltienne Date: Tue, 27 Jun 2023 14:22:22 +0200 Subject: [PATCH 07/26] Change color from annotation to red if name changes to "bad_" or "edge_" (#11753) --- mne/viz/_figure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mne/viz/_figure.py b/mne/viz/_figure.py index b103f94c38f..392228785a9 100644 --- a/mne/viz/_figure.py +++ b/mne/viz/_figure.py @@ -163,10 +163,10 @@ def _setup_annotation_colors(self): if color != red and key in labels: next(color_cycle) for idx, key in enumerate(labels): - if key in segment_colors: - continue - elif key.lower().startswith("bad") or key.lower().startswith("edge"): + if key.lower().startswith("bad") or key.lower().startswith("edge"): segment_colors[key] = red + elif key in segment_colors: + continue else: segment_colors[key] = next(color_cycle) self.mne.annotation_segment_colors = segment_colors From 38ed57443a45131f9dee5e7a21ca39c2c1fcd78b Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 27 Jun 2023 12:02:10 -0400 Subject: [PATCH 08/26] BUG: Fix bug with cHPI off (#11754) --- doc/changes/latest.inc | 1 + mne/chpi.py | 2 ++ mne/tests/test_chpi.py | 18 ++++++++++++++---- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 66fac3f4fa4..6ade52f1366 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -41,6 +41,7 @@ Bugs - Fix bug with overlapping text for :meth:`mne.Evoked.plot` (:gh:`11698` by `Alex Rockhill`_) - For :func:`mne.io.read_raw_eyelink`, the default value of the ``gap_description`` parameter is now ``'BAD_ACQ_SKIP'``, following MNE convention (:gh:`11719` by `Scott Huberty`_) - Fix bug with :func:`mne.io.read_raw_fil` where datasets without sensor positions would not import (:gh:`11733` by `George O'Neill`_) +- Fix bug with :func:`mne.chpi.compute_chpi_snr` where cHPI being off for part of the recording led to an error (:gh:`11754` by `Eric Larson`_) - Allow int-like for the argument ``id`` of `~mne.make_fixed_length_events` (:gh:`11748` by `Mathieu Scheltienne`_) - blink :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` now begin with ``'BAD_'``, i.e. ``'BAD_blink'``, because ocular data are missing during blinks. (:gh:`11746` by `Scott Huberty`_) diff --git a/mne/chpi.py b/mne/chpi.py index 4dce5b6b413..6a30c2fdf85 100644 --- a/mne/chpi.py +++ b/mne/chpi.py @@ -1245,6 +1245,8 @@ def _compute_chpi_amp_or_snr( # amps_or_snrs = _fit_chpi_amplitudes(raw, time_sl, hpi, snr) if snr: + if amps_or_snrs is None: + amps_or_snrs = np.full((n_freqs, grad_offset + 3), np.nan) # unpack the SNR estimates. mag & grad are returned in one array # (because of Numba) so take care with which column is which. # note that mean residual is a scalar (same for all HPI freqs) but diff --git a/mne/tests/test_chpi.py b/mne/tests/test_chpi.py index 138fe859e96..c2e94d7087d 100644 --- a/mne/tests/test_chpi.py +++ b/mne/tests/test_chpi.py @@ -383,6 +383,11 @@ def test_calculate_chpi_positions_vv(): def test_calculate_chpi_snr(): """Test cHPI SNR calculation.""" raw = read_raw_fif(chpi_fif_fname, allow_maxshield="yes") + # include handling of NaN (when cHPI was off at the beginning) + raw.load_data() + stop = int(round(raw.info["sfreq"])) * 2 + raw._data[raw.ch_names.index("STI201"), :stop] = 0 + result = compute_chpi_snr(raw) # make sure all the entries are there keys = { @@ -392,10 +397,15 @@ def test_calculate_chpi_snr(): } assert set(result) == keys.union({"times", "freqs"}) # make sure the values are plausible, given the sample data file - assert result["mag_snr"].min() > 1 - assert result["mag_snr"].max() < 40 - assert result["grad_snr"].min() > 1 - assert result["grad_snr"].max() < 40 + n_pts = len(raw.times) // int(round(raw.info["sfreq"] * 0.01)) + # our logic in this test for this length is not perfect + assert_allclose(result["mag_snr"].shape[0], n_pts, atol=5) + n_nan = np.where(result["times"] <= raw.first_time + 2)[0][-1] + assert_allclose(result["mag_snr"][:n_nan], np.nan) + assert result["mag_snr"][n_nan:].min() > 1 + assert result["mag_snr"][n_nan:].max() < 40 + assert result["grad_snr"][n_nan:].min() > 1 + assert result["grad_snr"][n_nan:].max() < 40 @testing.requires_testing_data From 4b0923f8cf491e13da9f602af91c42cd3722261d Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 27 Jun 2023 21:36:51 -0400 Subject: [PATCH 09/26] BUG: Fix bug with SNR calculation (#11755) --- doc/changes/latest.inc | 2 +- mne/chpi.py | 14 ++++++++------ mne/tests/test_chpi.py | 1 + 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 6ade52f1366..1356983a8de 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -41,7 +41,7 @@ Bugs - Fix bug with overlapping text for :meth:`mne.Evoked.plot` (:gh:`11698` by `Alex Rockhill`_) - For :func:`mne.io.read_raw_eyelink`, the default value of the ``gap_description`` parameter is now ``'BAD_ACQ_SKIP'``, following MNE convention (:gh:`11719` by `Scott Huberty`_) - Fix bug with :func:`mne.io.read_raw_fil` where datasets without sensor positions would not import (:gh:`11733` by `George O'Neill`_) -- Fix bug with :func:`mne.chpi.compute_chpi_snr` where cHPI being off for part of the recording led to an error (:gh:`11754` by `Eric Larson`_) +- Fix bug with :func:`mne.chpi.compute_chpi_snr` where cHPI being off for part of the recording or bad channels being defined led to an error or incorrect behavior (:gh:`11754`, :gh:`11755` by `Eric Larson`_) - Allow int-like for the argument ``id`` of `~mne.make_fixed_length_events` (:gh:`11748` by `Mathieu Scheltienne`_) - blink :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` now begin with ``'BAD_'``, i.e. ``'BAD_blink'``, because ocular data are missing during blinks. (:gh:`11746` by `Scott Huberty`_) diff --git a/mne/chpi.py b/mne/chpi.py index 6a30c2fdf85..128faf042e7 100644 --- a/mne/chpi.py +++ b/mne/chpi.py @@ -661,13 +661,15 @@ def _setup_hpi_amplitude_fitting( inv_model_reord = _reorder_inv_model(inv_model, len(hpi_freqs)) proj, proj_op, meg_picks = _setup_ext_proj(info, ext_order) # include mag and grad picks separately, for SNR computations - mag_picks = _picks_to_idx(info, "mag", allow_empty=True) - grad_picks = _picks_to_idx(info, "grad", allow_empty=True) + mag_subpicks = _picks_to_idx(info, "mag", allow_empty=True) + mag_subpicks = np.searchsorted(meg_picks, mag_subpicks) + grad_subpicks = _picks_to_idx(info, "grad", allow_empty=True) + grad_subpicks = np.searchsorted(meg_picks, grad_subpicks) # Set up magnetic dipole fits hpi = dict( meg_picks=meg_picks, - mag_picks=mag_picks, - grad_picks=grad_picks, + mag_subpicks=mag_subpicks, + grad_subpicks=grad_subpicks, hpi_pick=hpi_pick, model=model, inv_model=inv_model, @@ -779,8 +781,8 @@ def _fit_chpi_amplitudes(raw, time_sl, hpi, snr=False): len(hpi["freqs"]), hpi["model"], hpi["inv_model"], - hpi["mag_picks"], - hpi["grad_picks"], + hpi["mag_subpicks"], + hpi["grad_subpicks"], ) return _fast_fit( this_data, diff --git a/mne/tests/test_chpi.py b/mne/tests/test_chpi.py index c2e94d7087d..b77fc88ab8e 100644 --- a/mne/tests/test_chpi.py +++ b/mne/tests/test_chpi.py @@ -385,6 +385,7 @@ def test_calculate_chpi_snr(): raw = read_raw_fif(chpi_fif_fname, allow_maxshield="yes") # include handling of NaN (when cHPI was off at the beginning) raw.load_data() + raw.info["bads"] = ["MEG0342", "MEG1443"] stop = int(round(raw.info["sfreq"])) * 2 raw._data[raw.ch_names.index("STI201"), :stop] = 0 From dca0a2267c02ba10778913bdafebd352d630962e Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 28 Jun 2023 09:14:37 -0400 Subject: [PATCH 10/26] DOC: Fix docs (#11756) --- doc/changes/1.1.inc | 2 +- mne/utils/docs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/changes/1.1.inc b/doc/changes/1.1.inc index 4df2306329f..50ebc8111e8 100644 --- a/doc/changes/1.1.inc +++ b/doc/changes/1.1.inc @@ -41,7 +41,7 @@ Enhancements - Add ``head_source`` argument to :func:`mne.make_field_map` to allow selecting which head source to use (:gh:`10568` by `Eric Larson`_) -- Add support for ``n_jobs=None`` to support :func:`joblib:joblib.parallel_backend` for more precise control over parallelization (:gh:`10567` by `Eric Larson`_) +- Add support for ``n_jobs=None`` to support ``joblib:joblib.parallel_backend`` for more precise control over parallelization (:gh:`10567` by `Eric Larson`_) - It is now possible to compute inverse solutions with restricted source orientations using discrete forward models (:gh:`10464` by `Marijn van Vliet`_) diff --git a/mne/utils/docs.py b/mne/utils/docs.py index 16fd7c78851..0fd8b7de4e5 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -2624,7 +2624,7 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): to the number of CPU cores. Requires the :mod:`joblib` package. ``None`` (default) is a marker for 'unset' that will be interpreted as ``n_jobs=1`` (sequential execution) unless the call is performed under - a :func:`joblib:joblib.parallel_backend` context manager that sets another + a :class:`joblib:joblib.parallel_config` context manager that sets another value for ``n_jobs``. """ From 7def8369d7ca691d807fe916663168c2ee4c10e0 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 28 Jun 2023 11:19:04 -0500 Subject: [PATCH 11/26] add logging message about which EOG channel used for blink detection (#11757) --- mne/preprocessing/eog.py | 7 ++++++- mne/preprocessing/ica.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mne/preprocessing/eog.py b/mne/preprocessing/eog.py index 799cebbe684..ba050606b9c 100644 --- a/mne/preprocessing/eog.py +++ b/mne/preprocessing/eog.py @@ -68,6 +68,7 @@ def find_eog_events( """ # Getting EOG Channel eog_inds = _get_eog_channel_index(ch_name, raw) + eog_names = np.array(raw.ch_names)[eog_inds] # for logging logger.info("EOG channel index for this subject is: %s" % eog_inds) # Reject bad segments. @@ -79,6 +80,7 @@ def find_eog_events( eog_events = _find_eog_events( eog, + ch_names=eog_names, event_id=event_id, l_freq=l_freq, h_freq=h_freq, @@ -97,6 +99,8 @@ def find_eog_events( @verbose def _find_eog_events( eog, + *, + ch_names, event_id, l_freq, h_freq, @@ -137,8 +141,9 @@ def _find_eog_events( ] ) temp = np.sqrt(np.sum(filteog**2, axis=1)) - indexmax = np.argmax(temp) + if ch_names is not None: # it can be None if called from ica_find_eog_events + logger.info(f"Selecting channel {ch_names[indexmax]} for blink detection") # easier to detect peaks with filtering. filteog = filter_data( diff --git a/mne/preprocessing/ica.py b/mne/preprocessing/ica.py index 3938324ee72..a404b3dafb7 100644 --- a/mne/preprocessing/ica.py +++ b/mne/preprocessing/ica.py @@ -2770,6 +2770,7 @@ def ica_find_eog_events( """ eog_events = _find_eog_events( eog_source[np.newaxis], + ch_names=None, event_id=event_id, l_freq=l_freq, h_freq=h_freq, From 3c4a6f6bbc258dcb68f5d80ba5dd9b0d9f258057 Mon Sep 17 00:00:00 2001 From: Clemens Brunner Date: Fri, 30 Jun 2023 15:19:05 +0200 Subject: [PATCH 12/26] Update manual install docs (#11764) --- doc/install/manual_install.rst | 46 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/doc/install/manual_install.rst b/doc/install/manual_install.rst index c4013f84f1f..122bad3a892 100644 --- a/doc/install/manual_install.rst +++ b/doc/install/manual_install.rst @@ -11,14 +11,14 @@ Install via :code:`pip` or :code:`conda` instead. MNE-Python requires Python version |min_python_version| or higher. If you -need to install Python, please see :ref:`install-python`. +need help installing Python, please refer to our :ref:`install-python` guide. Installing MNE-Python with all dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We suggest to install MNE-Python into its own ``conda`` environment. +If you use Anaconda, we suggest installing MNE-Python into its own ``conda`` environment. The dependency stack is large and may take a long time (several tens of -minutes) to resolve on some systems via the default ``conda`` solver. We +minutes) to resolve on some systems via the default ``conda`` command. We therefore highly recommend using `mamba `__ instead, a ``conda`` replacement that is **much** faster. @@ -35,12 +35,12 @@ dependencies into it. If you need to convert structural MRI scans into models of the scalp, inner/outer skull, and cortical surfaces, you will also need -:doc:`FreeSurfer `. +to install :doc:`FreeSurfer `. -Installing a minimal MNE-Python with core functionality only -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you only need MNE-Python's core functionality including 2D plotting (but -**without 3D visualization**), install via :code:`pip`: +Installing MNE-Python with core dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you only need MNE-Python's core functionality, which includes 2D plotting +(but does not support 3D visualization), install via :code:`pip`: .. code-block:: console @@ -55,9 +55,13 @@ or via :code:`conda`: This will create a new ``conda`` environment called ``mne`` (you can adjust this by passing a different name via ``--name``). -Installing a minimal MNE-Python with EEGLAB I/O support -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you plan to use MNE-Python's functions that require **HDF5 I/O** (this +This minimal installation requires only a few dependencies. If you need additional +functionality later on, you can install individual packages as needed. + +Installing MNE-Python with HDF5 support +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you plan to use MNE-Python's functions that require +`HDF5 `__ I/O (this includes :func:`mne.io.read_raw_eeglab`, :meth:`mne.SourceMorph.save`, and others), you should run via :code:`pip`: @@ -65,7 +69,7 @@ others), you should run via :code:`pip`: $ pip install mne[hdf5] -or via :code:`conda` +or via :code:`conda`: .. code-block:: console @@ -74,6 +78,13 @@ or via :code:`conda` This will create a new ``conda`` environment called ``mne`` (you can adjust this by passing a different name via ``--name``). +If you have already installed MNE-Python with core dependencies (e.g. via ``pip install mne``), +you can install these two packages to unlock HDF5 support: + +.. code-block:: console + + $ pip install h5io pymatreader + Installing MNE-Python for other scenarios ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The :ref:`advanced_setup` page has additional @@ -92,7 +103,7 @@ Python development are: - `Visual Studio Code`_ (often shortened to "VS Code" or "vscode") is a development-focused text editor that supports many programming languages in addition to Python, includes an integrated terminal console, and has a rich - ecosystem of packages to extend its capabilities. Installing + extension ecosystem. Installing `Microsoft's Python Extension `__ is enough to get most Python users up and running. VS Code is free and @@ -103,11 +114,11 @@ Python development are: ``spyder`` (or on Windows or macOS, launched from the Anaconda Navigator GUI). It can also be installed with `dedicated installers `_. To avoid dependency conflicts with Spyder, you should install ``mne`` in a - separate environment, like explained in the earlier sections. Then, set + separate environment, as explained in previous sections. Then, instruct Spyder to use the ``mne`` environment as its default interpreter by opening Spyder and navigating to :samp:`Tools > Preferences > Python Interpreter > Use the following interpreter`. - There, paste the output of the following terminal commands + There, paste the output of the following terminal commands: .. code-block:: console @@ -135,6 +146,5 @@ Python development are: environment. - `PyCharm`_ is an IDE specifically for Python development that provides an - all-in-one installation (no extension packages needed). PyCharm comes in a - free "community" edition and a paid "professional" edition, and is - closed-source. + all-in-one solution (no extension packages needed). PyCharm comes in a + free and open-source Community edition as well as a paid Professional edition. From b92c88d70468a0f8174ab527b3e24d7f07af966b Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 30 Jun 2023 10:44:53 -0400 Subject: [PATCH 13/26] MAINT: Work around joblib bug (#11765) --- mne/parallel.py | 6 +++++- mne/tests/test_parallel.py | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mne/parallel.py b/mne/parallel.py index 5e77fcae724..8260e7d3234 100644 --- a/mne/parallel.py +++ b/mne/parallel.py @@ -105,8 +105,12 @@ def parallel_func( max_nbytes = None # disable memmaping kwargs["temp_folder"] = cache_dir kwargs["max_nbytes"] = max_nbytes - parallel = Parallel(n_jobs, **kwargs) + n_jobs_orig = n_jobs + if n_jobs is not None: # https://github.com/joblib/joblib/issues/1473 + kwargs["n_jobs"] = n_jobs + parallel = Parallel(**kwargs) n_jobs = _check_n_jobs(parallel.n_jobs) + logger.debug(f"Got {n_jobs} parallel jobs after requesting {n_jobs_orig}") if max_jobs is not None: n_jobs = min(n_jobs, max(_ensure_int(max_jobs, "max_jobs"), 1)) my_func = delayed(func) diff --git a/mne/tests/test_parallel.py b/mne/tests/test_parallel.py index 2e8e24aae5f..80d60a02f7c 100644 --- a/mne/tests/test_parallel.py +++ b/mne/tests/test_parallel.py @@ -34,7 +34,12 @@ def fun(x): if isinstance(n_jobs, str): backend, n_jobs = n_jobs.split() n_jobs = want_jobs = int(n_jobs) - ctx = joblib.parallel_backend(backend, n_jobs) + try: + func = joblib.parallel_config + except AttributeError: + # joblib < 1.3 + func = joblib.parallel_backend + ctx = func(backend, n_jobs=n_jobs) n_jobs = None else: ctx = nullcontext() @@ -43,5 +48,5 @@ def fun(x): else: want_jobs = 1 with ctx: - parallel, p_fun, got_jobs = parallel_func(fun, n_jobs) + parallel, p_fun, got_jobs = parallel_func(fun, n_jobs, verbose="debug") assert got_jobs == want_jobs From f48398a141ac4ed0ec4ec1f2cef766afd8605696 Mon Sep 17 00:00:00 2001 From: Scott Huberty <52462026+scott-huberty@users.noreply.github.com> Date: Sat, 1 Jul 2023 12:43:04 -0400 Subject: [PATCH 14/26] ENH: Interpolate blinks in eyetrack channels (#11740) Co-authored-by: Britta Westner --- doc/changes/latest.inc | 1 + doc/preprocessing.rst | 1 + mne/preprocessing/eyetracking/__init__.py | 1 + .../eyetracking/_pupillometry.py | 108 ++++++++++++++++++ mne/preprocessing/eyetracking/eyetracking.py | 1 + .../eyetracking/tests/test_pupillometry.py | 69 +++++++++++ .../preprocessing/90_eyetracking_data.py | 84 +++++++++++--- 7 files changed, 250 insertions(+), 15 deletions(-) create mode 100644 mne/preprocessing/eyetracking/_pupillometry.py create mode 100644 mne/preprocessing/eyetracking/tests/test_pupillometry.py diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 1356983a8de..64af9d468d4 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -29,6 +29,7 @@ Enhancements - Add standard montage lookup table for ``easycap-M43`` (:gh:`11744` by :newcontrib:`Diptyajit Das`) - Added :class:`mne.preprocessing.eyetracking.Calibration` to store eye-tracking calibration info, and :func:`mne.preprocessing.eyetracking.read_eyelink_calibration` to read calibration data from EyeLink systems (:gh:`11719` by `Scott Huberty`_) - Ocular :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` are now channel aware. This means if the left eye blinked, the associated annotation will store this in the ``'ch_names'`` key. (:gh:`11746` by `Scott Huberty`_) +- Added :func:`mne.preprocessing.eyetracking.interpolate_blinks` to linear interpolate eyetrack signals during blink periods. (:gh:`11740` by `Scott Huberty`_) Bugs ~~~~ diff --git a/doc/preprocessing.rst b/doc/preprocessing.rst index 7028e7ab307..f24017d5dac 100644 --- a/doc/preprocessing.rst +++ b/doc/preprocessing.rst @@ -156,6 +156,7 @@ Projections: Calibration read_eyelink_calibration set_channel_types_eyetrack + interpolate_blinks EEG referencing: diff --git a/mne/preprocessing/eyetracking/__init__.py b/mne/preprocessing/eyetracking/__init__.py index c232475b2fc..41c30c0bc8d 100644 --- a/mne/preprocessing/eyetracking/__init__.py +++ b/mne/preprocessing/eyetracking/__init__.py @@ -6,3 +6,4 @@ from .eyetracking import set_channel_types_eyetrack from .calibration import Calibration, read_eyelink_calibration +from ._pupillometry import interpolate_blinks diff --git a/mne/preprocessing/eyetracking/_pupillometry.py b/mne/preprocessing/eyetracking/_pupillometry.py new file mode 100644 index 00000000000..9b078bb2399 --- /dev/null +++ b/mne/preprocessing/eyetracking/_pupillometry.py @@ -0,0 +1,108 @@ +# Authors: Scott Huberty +# +# License: BSD-3-Clause + +import numpy as np + +from ...io import BaseRaw +from ...io.constants import FIFF +from ...utils import logger, _check_preload, _validate_type, warn + + +def interpolate_blinks(raw, buffer=0.05, match="BAD_blink", interpolate_gaze=False): + """Interpolate eyetracking signals during blinks. + + This function uses the timing of blink annotations to estimate missing + data. Operates in place. + + Parameters + ---------- + raw : instance of Raw + The raw data with at least one ``'pupil'`` or ``'eyegaze'`` channel. + buffer : float | array-like of float, shape ``(2,))`` + The time in seconds before and after a blink to consider invalid and + include in the segment to be interpolated over. Default is ``0.05`` seconds + (50 ms). If array-like, the first element is the time before the blink and the + second element is the time after the blink to consider invalid, for example, + ``(0.025, .1)``. + match : str | list of str + The description of annotations to interpolate over. If a list, the data within + all annotations that match any of the strings in the list will be interpolated + over. Defaults to ``'BAD_blink'``. + interpolate_gaze : bool + If False, only apply interpolation to ``'pupil channels'``. If True, interpolate + over ``'eyegaze'`` channels as well. Defaults to False, because eye position can + change in unpredictable ways during blinks. + + Returns + ------- + self : instance of Raw + Returns the modified instance. + + Notes + ----- + .. versionadded:: 1.5 + """ + _check_preload(raw, "interpolate_blinks") + _validate_type(raw, BaseRaw, "raw") + _validate_type(buffer, (float, tuple, list, np.ndarray), "buffer") + _validate_type(match, (str, tuple, list, np.ndarray), "match") + + # determine the buffer around blinks to include in the interpolation + buffer = np.array(buffer, dtype=float) + if buffer.size == 1: + buffer = np.array([buffer, buffer]) + + if isinstance(match, str): + match = [match] + + # get the blink annotations + blink_annots = [annot for annot in raw.annotations if annot["description"] in match] + if not blink_annots: + warn("No annotations matching {} found. Aborting.".format(match)) + return raw + _interpolate_blinks(raw, buffer, blink_annots, interpolate_gaze=interpolate_gaze) + + # remove bad from the annotation description + for desc in match: + if desc.startswith("BAD_"): + logger.info("Removing 'BAD_' from {}.".format(desc)) + raw.annotations.rename({desc: desc.replace("BAD_", "")}) + return raw + + +def _interpolate_blinks(raw, buffer, blink_annots, interpolate_gaze): + """Interpolate eyetracking signals during blinks in-place.""" + logger.info("Interpolating missing data during blinks...") + pre_buffer, post_buffer = buffer + # iterate over each eyetrack channel and interpolate the blinks + for ci, ch_info in enumerate(raw.info["chs"]): + if interpolate_gaze: # interpolate over all eyetrack channels + if ch_info["kind"] != FIFF.FIFFV_EYETRACK_CH: + continue + else: # interpolate over pupil channels only + if ch_info["coil_type"] != FIFF.FIFFV_COIL_EYETRACK_PUPIL: + continue + # Create an empty boolean mask + mask = np.zeros_like(raw.times, dtype=bool) + for annot in blink_annots: + if "ch_names" not in annot or not annot["ch_names"]: + msg = f"Blink annotation missing values for 'ch_names' key: {annot}" + raise ValueError(msg) + start = annot["onset"] - pre_buffer + end = annot["onset"] + annot["duration"] + post_buffer + if ch_info["ch_name"] not in annot["ch_names"]: + continue # skip if the channel is not in the blink annotation + # Update the mask for times within the current blink period + mask |= (raw.times >= start) & (raw.times <= end) + blink_indices = np.where(mask)[0] + non_blink_indices = np.where(~mask)[0] + + # Linear interpolation + interpolated_samples = np.interp( + raw.times[blink_indices], + raw.times[non_blink_indices], + raw._data[ci, non_blink_indices], + ) + # Replace the samples at the blink_indices with the interpolated values + raw._data[ci, blink_indices] = interpolated_samples diff --git a/mne/preprocessing/eyetracking/eyetracking.py b/mne/preprocessing/eyetracking/eyetracking.py index 7a6aef7debd..f7f32fcc050 100644 --- a/mne/preprocessing/eyetracking/eyetracking.py +++ b/mne/preprocessing/eyetracking/eyetracking.py @@ -1,4 +1,5 @@ # Authors: Dominik Welke +# Scott Huberty # # License: BSD-3-Clause diff --git a/mne/preprocessing/eyetracking/tests/test_pupillometry.py b/mne/preprocessing/eyetracking/tests/test_pupillometry.py new file mode 100644 index 00000000000..d24ec6de4ad --- /dev/null +++ b/mne/preprocessing/eyetracking/tests/test_pupillometry.py @@ -0,0 +1,69 @@ +# Authors: Scott Huberty +# License: BSD + +import numpy as np +import pytest + +from mne import create_info +from mne.datasets.testing import data_path, requires_testing_data +from mne.io import read_raw_eyelink, RawArray +from mne.preprocessing.eyetracking import interpolate_blinks +from mne.utils import requires_pandas + + +fname = data_path(download=False) / "eyetrack" / "test_eyelink.asc" + + +@requires_testing_data +@requires_pandas +@pytest.mark.parametrize( + "buffer, match, cause_error, interpolate_gaze", + [ + (0.025, "BAD_blink", False, False), + (0.025, "BAD_blink", False, True), + ((0.025, 0.025), ["random_annot"], False, False), + (0.025, "BAD_blink", True, False), + ], +) +def test_interpolate_blinks(buffer, match, cause_error, interpolate_gaze): + """Test interpolating pupil data during blinks.""" + raw = read_raw_eyelink( + fname, preload=True, create_annotations=["blinks"], find_overlaps=True + ) + # Create a dummy stim channel + # this will hit a certain line in the interpolate_blinks function + info = create_info(["STI"], raw.info["sfreq"], ["stim"]) + stim_data = np.zeros((1, len(raw.times))) + stim_raw = RawArray(stim_data, info) + raw.add_channels([stim_raw], force_update_info=True) + + # Get the indices of the first blink + first_blink_start = raw.annotations[0]["onset"] + first_blink_end = raw.annotations[0]["onset"] + raw.annotations[0]["duration"] + if match == ["random_annot"]: + msg = "No annotations matching" + with pytest.warns(RuntimeWarning, match=msg): + interpolate_blinks(raw, buffer=buffer, match=match) + return + + if cause_error: + # Make an annotation without ch_names info + raw.annotations.append(onset=1, duration=1, description="BAD_blink") + with pytest.raises(ValueError): + interpolate_blinks(raw, buffer=buffer, match=match) + return + else: + interpolate_blinks( + raw, buffer=buffer, match=match, interpolate_gaze=interpolate_gaze + ) + + # Now get the data and check that the blinks are interpolated + data, times = raw.get_data(return_times=True) + # Get the indices of the first blink + blink_ind = np.where((times >= first_blink_start) & (times <= first_blink_end))[0] + # pupil data during blinks are zero, check that interpolated data are not zeros + assert not np.any(data[2, blink_ind] == 0) # left eye + assert not np.any(data[5, blink_ind] == 0) # right eye + if interpolate_gaze: + assert not np.isnan(data[0, blink_ind]).any() # left eye + assert not np.isnan(data[1, blink_ind]).any() # right eye diff --git a/tutorials/preprocessing/90_eyetracking_data.py b/tutorials/preprocessing/90_eyetracking_data.py index b394cb72e2c..9e3f03439bd 100644 --- a/tutorials/preprocessing/90_eyetracking_data.py +++ b/tutorials/preprocessing/90_eyetracking_data.py @@ -26,15 +26,14 @@ # ``'eyegaze'``, channels (X/Y), 1 ``'pupil'`` channel, and 1 ``'stim'`` # channel. -from mne import Epochs, find_events -from mne.io import read_raw_eyelink +import mne from mne.datasets.eyelink import data_path from mne.preprocessing.eyetracking import read_eyelink_calibration eyelink_fname = data_path() / "mono_multi-block_multi-DINS.asc" -raw = read_raw_eyelink(eyelink_fname, create_annotations=["blinks", "messages"]) -raw.crop(tmin=0, tmax=146) +raw = mne.io.read_raw_eyelink(eyelink_fname, create_annotations=["blinks", "messages"]) +raw.crop(tmin=0, tmax=130) # for this demonstration, let's take a subset of the data # %% # Ocular annotations @@ -115,7 +114,7 @@ # the screen. We now extract these events to visualize the pupil response. We will use # these later in this tutorial. -events = find_events(raw, "DIN", shortest_event=1, min_duration=0.02, uint_cast=True) +events = mne.find_events(raw, shortest_event=1, min_duration=0.02, uint_cast=True) event_dict = {"flash": 3} @@ -142,25 +141,80 @@ scalings=dict(eyegaze=1e3), ) +# %% +# handling blink artifacts +# ------------------------ +# We also notice that, naturally, there are blinks in our data, and these blink periods +# occur within ``"BAD_blink"`` annotations. During blink periods, ``"eyegaze"`` +# coordinates are not reported, and ``"pupil"`` size data are ``0``. We don't want these +# blink artifacts biasing our analysis, so we have two options: We can either drop the +# blink periods from our data during epoching, or we can interpolate the missing data +# during the blink periods. For this tutorial, let's interpolate the missing samples: + +mne.preprocessing.eyetracking.interpolate_blinks(raw, buffer=0.05) +# Let's plot our data again to see the result of the interpolation: +raw.pick(["pupil_right"]) # Let's pick just the pupil channel +raw.plot(events=events, event_id={"Flash": 3}, event_color="g") + +# %% +# :func:`~mne.preprocessing.eyetracking.interpolate_blinks` performs a simple linear +# interpolation of the pupil size data during blink periods. the ``buffer`` keyword +# argument specifies the amount of time (in seconds) before and after the blinks to +# include in the interpolation. This is helpful because the ``blink`` annotations +# do not always capture the entire blink in the signal. We specified a value of ``.05`` +# seconds (50 ms), which is slightly more than the default value of ``.025``. + +# %% +# Dealing with high frequency noise +# --------------------------------- +# From the plot above, we notice that there is some high frequency noise in the pupil +# signal. We can remove this noise by low-pass filtering the data: + +# Apply a low pass filter to the pupil channel +raw.filter(l_freq=None, h_freq=40, picks=["pupil_right"]) + +# %% +# Rejecting bad spans of data +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Even after filtering the pupil data and interpolating the blink periods, we still see +# some artifacts in the data (the large spikes) that we don't want to include in our +# analysis. Let's epoch our data and then reject any epochs that might contain these +# artifacts. We'll use :class:`mne.Epochs` to epoch our data, and pass in the +# ``events`` array and ``event_dict`` that we created earlier. We'll also pass in the +# ``reject`` keyword argument to reject any epochs that contain data that exceeds a +# peak-to-peak signal amplitude threshold of ``1500`` in the ``"pupil"`` channel. +# Note that this threshold is arbitrary, and should be adjusted based on the data. +# We chose 1500 because eyelink reports pupil size in arbitrary units (AU), which +# typically ranges from 800 to 3000 units. Our epochs already contains large +# signal fluctuations due to the pupil response, so a threshold of 1500 is conservative +# enough to reject epochs only with large artifacts. + +epochs = mne.Epochs( + raw, + events, + tmin=-0.3, + tmax=5, + event_id=event_dict, + preload=True, + reject=dict(pupil=1500), +) +epochs.plot(events=events, event_id=event_dict) + +# %% +# We can clearly see the prominent decrease in pupil size following the +# stimulation. # %% # Plot average pupil response # --------------------------- # -# We now visualize the pupillary light reflex. -# Therefore, we select only the pupil channel and plot the evoked response to -# the light flashes. -# -# As we see, there is a prominent decrease in pupil size following the -# stimulation. The noise starting about 2.5 s after stimulus onset stems from -# eyeblinks and artifacts in some of the 16 trials. +# Finally, let's plot the evoked response to the light flashes to get a sense of the +# average pupillary light response. -epochs = Epochs(raw, events, tmin=-0.3, tmax=5, event_id=event_dict, preload=True) -epochs.pick_types(eyetrack="pupil") epochs.average().plot() # %% -# It is important to note that pupil size data are reported by Eyelink (and +# Again, it is important to note that pupil size data are reported by Eyelink (and # stored internally by MNE) as arbitrary units (AU). While it often can be # preferable to convert pupil size data to millimeters, this requires # information that is not present in the file. MNE does not currently From d78d0f5b0a6dedb4faf49950b07d72bdcc53557f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 3 Jul 2023 14:30:30 -0400 Subject: [PATCH 15/26] MAINT: Update for linkcheck [circle linkcheck] (#11771) --- doc/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index b30d777292f..1c757b12413 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -690,6 +690,9 @@ def append_attr_meth_examples(app, what, name, obj, options, lines): "https://doi.org/10.1167/", # jov.arvojournals.org "https://doi.org/10.1177/", # journals.sagepub.com "https://doi.org/10.1063/", # pubs.aip.org/aip/jap + "https://doi.org/10.1080/", # www.tandfonline.com + "https://doi.org/10.1088/", # www.tandfonline.com + "https://doi.org/10.3109/", # www.tandfonline.com "https://www.researchgate.net/profile/", # 503 Server error "https://hal.archives-ouvertes.fr/hal-01848442", From b4eecda8d66ac083554db8ee1c925b98fdc2a402 Mon Sep 17 00:00:00 2001 From: Mathieu Scheltienne Date: Wed, 5 Jul 2023 11:22:14 +0200 Subject: [PATCH 16/26] [MRG] Use FIFF.FIFF_UNITM_NONE instead of 0 when adding a reference channel (#11774) --- mne/io/reference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/io/reference.py b/mne/io/reference.py index a948313ec62..d6e3f80adb8 100644 --- a/mne/io/reference.py +++ b/mne/io/reference.py @@ -251,7 +251,7 @@ def add_reference_channels(inst, ref_channels, copy=True): "scanno": nchan + 1, "cal": 1, "range": 1.0, - "unit_mul": 0.0, + "unit_mul": FIFF.FIFF_UNITM_NONE, "unit": FIFF.FIFF_UNIT_V, "coord_frame": FIFF.FIFFV_COORD_HEAD, "loc": ref_dig_array, From 06d396d40a6a925473dcf1451758272d68f64185 Mon Sep 17 00:00:00 2001 From: Frostime Date: Wed, 5 Jul 2023 23:53:33 +0800 Subject: [PATCH 17/26] Fix eeglab.py, an error for read annotations in `RawEEGLab` (#11773) Co-authored-by: Daniel McCloy --- mne/io/eeglab/eeglab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/io/eeglab/eeglab.py b/mne/io/eeglab/eeglab.py index 37ffc4e5e85..9bd13797cac 100644 --- a/mne/io/eeglab/eeglab.py +++ b/mne/io/eeglab/eeglab.py @@ -478,7 +478,7 @@ def __init__( ) # create event_ch from annotations - annot = read_annotations(input_fname) + annot = read_annotations(input_fname, uint16_codec=uint16_codec) self.set_annotations(annot) _check_boundary(annot, None) From 5d452c6a091f05d83055c9eef202008b34db2158 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 5 Jul 2023 12:11:01 -0500 Subject: [PATCH 18/26] add missing changelog from 11773 (#11777) --- doc/changes/latest.inc | 1 + doc/changes/names.inc | 2 ++ 2 files changed, 3 insertions(+) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 64af9d468d4..9f0b968aebe 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -33,6 +33,7 @@ Enhancements Bugs ~~~~ +- Fix bug where user-provided codec was not used to read annotations when loading EEGLAB ``.set`` files (:gh:`11773` by :newcontrib:`Yiping Zuo`) - Fix bug that required curv.*h files to create Brain object (:gh:`11704` by :newcontrib:`Aaron Earle-Richardson`) - Extended test to highlight bug in :func:`mne.stats.permutation_t_test` (:gh:`11575` by :newcontrib:`Joshua Calder-Travis`) - Fix bug where :meth:`mne.viz.Brain.add_volume_labels` used an incorrect orientation (:gh:`11730` by `Alex Rockhill`_) diff --git a/doc/changes/names.inc b/doc/changes/names.inc index 357f929309e..3798fefcda6 100644 --- a/doc/changes/names.inc +++ b/doc/changes/names.inc @@ -554,6 +554,8 @@ .. _Yaroslav Halchenko: http://haxbylab.dartmouth.edu/ppl/yarik.html +.. _Yiping Zuo: https://github.com/frostime + .. _Yousra Bekhti: https://www.linkedin.com/pub/yousra-bekhti/56/886/421 .. _Yu-Han Luo: https://github.com/yh-luo From 2eb608a0f111faee842e017993273f950b083012 Mon Sep 17 00:00:00 2001 From: Mathieu Scheltienne Date: Thu, 6 Jul 2023 11:02:42 +0200 Subject: [PATCH 19/26] [MRG] By-pass set_annotations when plotting ICA sources (#11766) --- doc/changes/latest.inc | 1 + mne/viz/ica.py | 2 +- mne/viz/tests/test_ica.py | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 9f0b968aebe..fdbf8e5e30d 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -46,6 +46,7 @@ Bugs - Fix bug with :func:`mne.chpi.compute_chpi_snr` where cHPI being off for part of the recording or bad channels being defined led to an error or incorrect behavior (:gh:`11754`, :gh:`11755` by `Eric Larson`_) - Allow int-like for the argument ``id`` of `~mne.make_fixed_length_events` (:gh:`11748` by `Mathieu Scheltienne`_) - blink :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` now begin with ``'BAD_'``, i.e. ``'BAD_blink'``, because ocular data are missing during blinks. (:gh:`11746` by `Scott Huberty`_) +- Fix display of :class:`~mne.Annotations` in `mne.preprocessing.ICA.plot_sources` when the ``raw`` has ``raw.first_samp != 0`` and doesn't have a measurement date (:gh:`11766` by `Mathieu Scheltienne`_) API changes ~~~~~~~~~~~ diff --git a/mne/viz/ica.py b/mne/viz/ica.py index 6f78512f8b1..92ef98da533 100644 --- a/mne/viz/ica.py +++ b/mne/viz/ica.py @@ -1302,7 +1302,7 @@ def _plot_sources( info["bads"] = [ch_names[x] for x in exclude if x in picks] if is_raw: inst_array = RawArray(data, info, inst.first_samp) - inst_array.set_annotations(inst.annotations) + inst_array._annotations = inst.annotations else: data = data.reshape(-1, len(inst), len(inst.times)).swapaxes(0, 1) inst_array = EpochsArray(data, info) diff --git a/mne/viz/tests/test_ica.py b/mne/viz/tests/test_ica.py index 7eee6b79a44..110f3efcf2b 100644 --- a/mne/viz/tests/test_ica.py +++ b/mne/viz/tests/test_ica.py @@ -9,7 +9,7 @@ import matplotlib.pyplot as plt import numpy as np import pytest -from numpy.testing import assert_equal, assert_array_equal +from numpy.testing import assert_equal, assert_array_equal, assert_allclose from mne import ( read_events, @@ -319,7 +319,7 @@ def test_plot_ica_sources(raw_orig, browser_backend, monkeypatch): assert browser_backend._get_n_figs() == 0 del long_raw - # test with annotations + # test with annotations and a measurement date orig_annot = raw.annotations raw.set_annotations(Annotations([0.2], [0.1], "Test")) fig = ica.plot_sources(raw) @@ -328,6 +328,22 @@ def test_plot_ica_sources(raw_orig, browser_backend, monkeypatch): assert len(fig.mne.ax_hscroll.collections) == 1 else: assert len(fig.mne.regions) == 1 + assert_allclose(fig.mne.regions[0].getRegion(), (0.2, 0.3)) + + # test with annotations and no measurement date + orig_meas_date = raw.info["meas_date"] + raw.set_meas_date(None) + assert raw.first_samp != 0 + raw.set_annotations(Annotations([0.2], [0.1], "Test")) + fig = ica.plot_sources(raw) + if browser_backend.name == "matplotlib": + assert len(fig.mne.ax_main.collections) == 1 + assert len(fig.mne.ax_hscroll.collections) == 1 + else: + assert len(fig.mne.regions) == 1 + assert_allclose(fig.mne.regions[0].getRegion(), (0.2, 0.3)) + + raw.set_meas_date(orig_meas_date) raw.set_annotations(orig_annot) # test error handling From 79647ec0f24de1c2c4d7d637726ef61cdc72a4da Mon Sep 17 00:00:00 2001 From: Scott Huberty <52462026+scott-huberty@users.noreply.github.com> Date: Thu, 6 Jul 2023 12:44:22 -0400 Subject: [PATCH 20/26] DOC: EEG eye-tracking alignment tutorial (#11770) Co-authored-by: Britta Westner Co-authored-by: Daniel McCloy --- doc/changes/latest.inc | 1 + doc/overview/datasets_index.rst | 11 +- mne/datasets/config.py | 6 +- .../preprocessing/90_eyetracking_data.py | 275 +++++++++--------- 4 files changed, 142 insertions(+), 151 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index fdbf8e5e30d..5847cb5f17d 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -30,6 +30,7 @@ Enhancements - Added :class:`mne.preprocessing.eyetracking.Calibration` to store eye-tracking calibration info, and :func:`mne.preprocessing.eyetracking.read_eyelink_calibration` to read calibration data from EyeLink systems (:gh:`11719` by `Scott Huberty`_) - Ocular :class:`mne.Annotations` read in by :func:`mne.io.read_raw_eyelink` are now channel aware. This means if the left eye blinked, the associated annotation will store this in the ``'ch_names'`` key. (:gh:`11746` by `Scott Huberty`_) - Added :func:`mne.preprocessing.eyetracking.interpolate_blinks` to linear interpolate eyetrack signals during blink periods. (:gh:`11740` by `Scott Huberty`_) +- Added a section for combining eye-tracking and EEG data to the preprocessing tutorial "working with eye tracker data in MNE-Python" (:gh:`11770` by `Scott Huberty`_) Bugs ~~~~ diff --git a/doc/overview/datasets_index.rst b/doc/overview/datasets_index.rst index b2d0715e8e9..3946ef64be5 100644 --- a/doc/overview/datasets_index.rst +++ b/doc/overview/datasets_index.rst @@ -475,14 +475,17 @@ standard. * :ref:`tut-ssvep` +.. _eyelink-dataset: + EYELINK ======= :func:`mne.datasets.eyelink.data_path` -A small example dataset in SR research's proprietary .asc format. -1 participant fixated on the screen while short light flashes appeared. -Monocular recording of gaze position and pupil size, 1000 Hz sampling -frequency. +A small example dataset from a pupillary light reflex experiment. Both EEG (EGI) and +eye-tracking (SR Research EyeLink; ASCII format) data were recorded and stored in +separate files. 1 participant fixated on the screen while short light flashes appeared. +Event onsets were recorded by a photodiode attached to the screen and were +sent to both the EEG and eye-tracking systems. .. topic:: Examples diff --git a/mne/datasets/config.py b/mne/datasets/config.py index 7869f97a78e..e1881b4b774 100644 --- a/mne/datasets/config.py +++ b/mne/datasets/config.py @@ -345,9 +345,9 @@ # eyelink dataset MNE_DATASETS["eyelink"] = dict( - archive_name="eyelink_example_data.zip", - hash="md5:081950c05f35267458d9c751e178f161", - url=("https://osf.io/r5ndq/download?version=1"), + archive_name="eeg-eyetrack_data.zip", + hash="md5:c4fc788fe01737e08e9086c90cab642d", + url=("https://osf.io/63fjm/download?version=1"), folder_name="eyelink-example-data", config_key="MNE_DATASETS_EYELINK_PATH", ) diff --git a/tutorials/preprocessing/90_eyetracking_data.py b/tutorials/preprocessing/90_eyetracking_data.py index 9e3f03439bd..4c053806081 100644 --- a/tutorials/preprocessing/90_eyetracking_data.py +++ b/tutorials/preprocessing/90_eyetracking_data.py @@ -6,12 +6,14 @@ Working with eye tracker data in MNE-Python =========================================== -In this tutorial we will load some eye tracker data and plot the average -pupil response to light flashes (i.e. the pupillary light reflex). +In this tutorial we will explore simultaneously recorded eye-tracking and EEG data from +a pupillary light reflex task. We will combine the eye-tracking and EEG data, and plot +the ERP and pupil response to the light flashes (i.e. the pupillary light reflex). """ # noqa: E501 -# Authors: Dominik Welke -# Scott Huberty +# Authors: Scott Huberty +# Dominik Welke +# # # License: BSD-3-Clause @@ -19,75 +21,71 @@ # Data loading # ------------ # -# First we will load an eye tracker recording from SR research's proprietary -# ``'.asc'`` file format. -# -# The info structure tells us we loaded a monocular recording with 2 -# ``'eyegaze'``, channels (X/Y), 1 ``'pupil'`` channel, and 1 ``'stim'`` -# channel. +# As usual we start by importing the modules we need and loading some +# :ref:`example data `: eye-tracking data recorded from SR research's +# ``'.asc'`` file format, and EEG data recorded from EGI's ``'.mff'`` file format. We'll +# pass ``create_annotations=["blinks"]`` to :func:`~mne.io.read_raw_eyelink` so that +# only blinks annotations are created (by default, annotations are created for blinks, +# saccades, fixations, and experiment messages). import mne from mne.datasets.eyelink import data_path from mne.preprocessing.eyetracking import read_eyelink_calibration -eyelink_fname = data_path() / "mono_multi-block_multi-DINS.asc" +et_fpath = data_path() / "sub-01_task-plr_eyetrack.asc" +eeg_fpath = data_path() / "sub-01_task-plr_eeg.mff" -raw = mne.io.read_raw_eyelink(eyelink_fname, create_annotations=["blinks", "messages"]) -raw.crop(tmin=0, tmax=130) # for this demonstration, let's take a subset of the data +raw_et = mne.io.read_raw_eyelink(et_fpath, preload=True, create_annotations=["blinks"]) +raw_eeg = mne.io.read_raw_egi(eeg_fpath, preload=True, verbose="warning") +raw_eeg.filter(1, 30) # %% -# Ocular annotations -# ------------------ -# By default, Eyelink files will output events for ocular events (blinks, -# saccades, fixations), and experiment messages. MNE will store these events -# as `mne.Annotations`. Ocular annotations contain channel information, in the -# ``'ch_names'``` key. This means that we can see which eye an ocular event occurred in: +# .. seealso:: :ref:`tut-importing-eyetracking-data` +# :class: sidebar -print(raw.annotations[0]) # a blink in the right eye +# %% +# The info structure of the eye-tracking data tells us we loaded a monocular recording +# with 2 eyegaze channels (x- and y-coordinate positions), 1 pupil channel, 1 stim +# channel, and 3 channels for the head distance and position (since this data was +# collected using EyeLink's Remote mode). + +raw_et.info # %% -# If we are only interested in certain event types from -# the Eyelink file, we can select for these using the ``'create_annotations'`` -# argument of `mne.io.read_raw_eyelink`. above, we only created annotations -# for blinks, and experiment messages. -# -# Note that ``'blink'`` annotations are read in as ``'BAD_blink'``, and MNE will treat -# these as bad segments of data. This means that blink periods will be dropped during -# epoching by default. +# Ocular annotations +# ------------------ +# By default, EyeLink files will output ocular events (blinks, saccades, and +# fixations), and experiment messages. MNE will store these events +# as `mne.Annotations`. Ocular annotations contain channel information in the +# ``'ch_names'`` key. This means that we can see which eye an ocular event occurred in, +# which can be useful for binocular recordings: + +print(raw_et.annotations[0]["ch_names"]) # a blink in the right eye # %% # Checking the calibration # ------------------------ # -# We can also load the calibrations from the recording and visualize them. -# Checking the quality of the calibration is a useful first step in assessing -# the quality of the eye tracking data. Note that +# EyeLink ``.asc`` files can also include calibration information. +# MNE-Python can load and visualize those eye-tracking calibrations, which +# is a useful first step in assessing the quality of the eye-tracking data. # :func:`~mne.preprocessing.eyetracking.read_eyelink_calibration` # will return a list of :class:`~mne.preprocessing.eyetracking.Calibration` instances, # one for each calibration. We can index that list to access a specific calibration. -cals = read_eyelink_calibration(eyelink_fname) +cals = read_eyelink_calibration(et_fpath) print(f"number of calibrations: {len(cals)}") first_cal = cals[0] # let's access the first (and only in this case) calibration print(first_cal) # %% -# Here we can see that a 5-point calibration was performed at the beginning of -# the recording. Note that you can access the calibration information using -# dictionary style indexing: +# Calibrations have dict-like attribute access; in addition to the attributes shown in +# the output above, additional attributes are ``'positions'`` (the x and y coordinates +# of each calibration point), ``'gaze'`` (the x and y coordinates of the actual gaze +# position to each calibration point), and ``'offsets'`` (the offset in visual degrees +# between the calibration position and the actual gaze position for each calibration +# point). Below is an example of how to access these data: -print(f"Eye calibrated: {first_cal['eye']}") -print(f"Calibration model: {first_cal['model']}") -print(f"Calibration average error: {first_cal['avg_error']}") - -# %% -# The data for individual calibration points are stored as :class:`numpy.ndarray` -# arrays, in the ``'positions'``, ``'gaze'``, and ``'offsets'`` keys. ``'positions'`` -# contains the x and y coordinates of each calibration point. ``'gaze'`` contains the -# x and y coordinates of the actual gaze position for each calibration point. -# ``'offsets'`` contains the offset (in visual degrees) between the calibration position -# and the actual gaze position for each calibration point. Below is an example of -# how to access these data: print(f"offset of the first calibration point: {first_cal['offsets'][0]}") print(f"offset for each calibration point: {first_cal['offsets']}") print(f"x-coordinate for each calibration point: {first_cal['positions'].T[0]}") @@ -98,126 +96,115 @@ # and the offsets (in visual degrees) between the calibration position and the actual # gaze position of each calibration point. -first_cal.plot(show_offsets=True) +first_cal.plot() # %% -# Get stimulus events from DIN channel -# ------------------------------------ +# Plot the raw eye-tracking data +# ------------------------------ # -# Eyelink eye trackers have a DIN port that can be used to feed in stimulus -# or response timings. :func:`mne.io.read_raw_eyelink` loads this data as a -# ``'stim'`` channel. Alternatively, the onset of stimulus events could be sent -# to the eyetracker as ``messages`` - these can be read in as -# `mne.Annotations`. +# Let's plot the raw eye-tracking data. We'll pass a custom `dict` into +# the scalings argument to make the eyegaze channel traces legible when plotting, +# since this file contains pixel position data (as opposed to eye angles, +# which are reported in radians). + +raw_et.plot(scalings=dict(eyegaze=1e3)) + +# %% +# Handling blink artifacts +# ------------------------ # -# In the example data, the DIN channel contains the onset of light flashes on -# the screen. We now extract these events to visualize the pupil response. We will use -# these later in this tutorial. +# Naturally, there are blinks in our data, which occur within ``"BAD_blink"`` +# annotations. During blink periods, eyegaze coordinates are not reported, and pupil +# size data are ``0``. We don't want these blink artifacts biasing our analysis, so we +# have two options: Drop the blink periods from our data during epoching, or interpolate +# the missing data during the blink periods. For this tutorial, let's interpolate the +# blink samples. We'll pass ``(0.05, 0.2)`` to +# :func:`~mne.preprocessing.eyetracking.interpolate_blinks`, expanding the interpolation +# window 50 ms before and 200 ms after the blink, so that the noisy data surrounding +# the blink is also interpolated. -events = mne.find_events(raw, shortest_event=1, min_duration=0.02, uint_cast=True) -event_dict = {"flash": 3} +mne.preprocessing.eyetracking.interpolate_blinks(raw_et, buffer=(0.05, 0.2)) +# %% +# .. important:: By default, :func:`~mne.preprocessing.eyetracking.interpolate_blinks`, +# will only interpolate blinks in pupil channels. Passing +# ``interpolate_gaze=True`` will also interpolate the blink periods of the +# eyegaze channels. Be aware, however, that eye movements can occur +# during blinks which makes the gaze data less suitable for interpolation. # %% -# Plot raw data -# ------------- +# Extract common stimulus events from the data +# -------------------------------------------- # -# As the following plot shows, we now have a raw object with the eye tracker -# data, eyeblink annotations and stimulus events (from the DIN channel). +# In this experiment, a photodiode attached to the display screen was connected to both +# the EEG and eye-tracking systems. The photodiode was triggered by the the light flash +# stimuli, causing a signal to be sent to both systems simultaneously, signifying the +# onset of the flash. The photodiode signal was recorded as a digital input channel in +# the EEG and eye-tracking data. MNE loads these data as a :term:`stim channel`. # -# The plot also shows us that there is some noise in the data (not always -# categorized as blinks). Also, notice that we have passed a custom `dict` into -# the scalings argument of ``raw.plot``. This is necessary to make the eyegaze -# channel traces legible when plotting, since the file contains pixel position -# data (as opposed to eye angles, which are reported in radians). We also could -# have simply passed ``scalings='auto'``. - -raw.plot( - events=events, - event_id={"Flash": 3}, - event_color="g", - start=25, - duration=45, - scalings=dict(eyegaze=1e3), -) +# We'll extract the flash event onsets from both the EEG and eye-tracking data, as they +# are necessary for aligning the data from the two recordings. -# %% -# handling blink artifacts -# ------------------------ -# We also notice that, naturally, there are blinks in our data, and these blink periods -# occur within ``"BAD_blink"`` annotations. During blink periods, ``"eyegaze"`` -# coordinates are not reported, and ``"pupil"`` size data are ``0``. We don't want these -# blink artifacts biasing our analysis, so we have two options: We can either drop the -# blink periods from our data during epoching, or we can interpolate the missing data -# during the blink periods. For this tutorial, let's interpolate the missing samples: - -mne.preprocessing.eyetracking.interpolate_blinks(raw, buffer=0.05) -# Let's plot our data again to see the result of the interpolation: -raw.pick(["pupil_right"]) # Let's pick just the pupil channel -raw.plot(events=events, event_id={"Flash": 3}, event_color="g") +et_events = mne.find_events(raw_et, min_duration=0.01, shortest_event=1, uint_cast=True) +eeg_events = mne.find_events(raw_eeg, stim_channel="DIN3") # %% -# :func:`~mne.preprocessing.eyetracking.interpolate_blinks` performs a simple linear -# interpolation of the pupil size data during blink periods. the ``buffer`` keyword -# argument specifies the amount of time (in seconds) before and after the blinks to -# include in the interpolation. This is helpful because the ``blink`` annotations -# do not always capture the entire blink in the signal. We specified a value of ``.05`` -# seconds (50 ms), which is slightly more than the default value of ``.025``. +# The output above shows us that both the EEG and EyeLink data used event ID ``2`` for +# the flash events, so we'll create a dictionary to use later when plotting to label +# those events. + +event_dict = dict(Flash=2) # %% -# Dealing with high frequency noise -# --------------------------------- -# From the plot above, we notice that there is some high frequency noise in the pupil -# signal. We can remove this noise by low-pass filtering the data: +# Align the eye-tracking data with EEG the data +# --------------------------------------------- +# +# In this dataset, eye-tracking and EEG data were recorded simultaneously, but on +# different systems, so we'll need to align the data before we can analyze them +# together. We can do this using the :func:`~mne.preprocessing.realign_raw` function, +# which will align the data based on the timing of the shared events that are present in +# both :class:`~mne.io.Raw` objects. We'll use the shared photodiode events we extracted +# above, but first we need to convert the event onsets from samples to seconds. Once the +# data have been aligned, we'll add the EEG channels to the eye-tracking raw object. + +# Convert event onsets from samples to seconds +et_flash_times = et_events[:, 0] / raw_et.info["sfreq"] +eeg_flash_times = eeg_events[:, 0] / raw_eeg.info["sfreq"] +# Align the data +mne.preprocessing.realign_raw( + raw_et, raw_eeg, et_flash_times, eeg_flash_times, verbose="error" +) +# Add EEG channels to the eye-tracking raw object +raw_et.add_channels([raw_eeg], force_update_info=True) + +# Define a few channel groups of interest and plot the data +frontal = ["E19", "E11", "E4", "E12", "E5"] +occipital = ["E61", "E62", "E78", "E67", "E72", "E77"] +pupil = ["pupil_right"] +picks_idx = mne.pick_channels( + raw_et.ch_names, frontal + occipital + pupil, ordered=True +) +raw_et.plot(events=et_events, event_id=event_dict, event_color="g", order=picks_idx) -# Apply a low pass filter to the pupil channel -raw.filter(l_freq=None, h_freq=40, picks=["pupil_right"]) # %% -# Rejecting bad spans of data -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# Even after filtering the pupil data and interpolating the blink periods, we still see -# some artifacts in the data (the large spikes) that we don't want to include in our -# analysis. Let's epoch our data and then reject any epochs that might contain these -# artifacts. We'll use :class:`mne.Epochs` to epoch our data, and pass in the -# ``events`` array and ``event_dict`` that we created earlier. We'll also pass in the -# ``reject`` keyword argument to reject any epochs that contain data that exceeds a -# peak-to-peak signal amplitude threshold of ``1500`` in the ``"pupil"`` channel. -# Note that this threshold is arbitrary, and should be adjusted based on the data. -# We chose 1500 because eyelink reports pupil size in arbitrary units (AU), which -# typically ranges from 800 to 3000 units. Our epochs already contains large -# signal fluctuations due to the pupil response, so a threshold of 1500 is conservative -# enough to reject epochs only with large artifacts. +# Showing the pupillary light reflex +# ---------------------------------- +# Now let's extract epochs around our flash events. We should see a clear pupil +# constriction response to the flashes. epochs = mne.Epochs( - raw, - events, - tmin=-0.3, - tmax=5, + raw_et, + events=et_events, event_id=event_dict, + tmin=-0.3, + tmax=3, preload=True, - reject=dict(pupil=1500), ) -epochs.plot(events=events, event_id=event_dict) +epochs[:8].plot(events=et_events, event_id=event_dict, order=picks_idx) # %% -# We can clearly see the prominent decrease in pupil size following the -# stimulation. +# Finally, let's plot the evoked responses to the light flashes to get a sense of the +# average pupillary light response, and the associated ERP in the EEG data. -# %% -# Plot average pupil response -# --------------------------- -# -# Finally, let's plot the evoked response to the light flashes to get a sense of the -# average pupillary light response. - -epochs.average().plot() - -# %% -# Again, it is important to note that pupil size data are reported by Eyelink (and -# stored internally by MNE) as arbitrary units (AU). While it often can be -# preferable to convert pupil size data to millimeters, this requires -# information that is not present in the file. MNE does not currently -# provide methods to convert pupil size data. -# See :ref:`tut-importing-eyetracking-data` for more information on pupil size -# data. +epochs.average().plot(picks=occipital + pupil) From 4cf0d1d5ca692cf8426933ef5e683b648b04203d Mon Sep 17 00:00:00 2001 From: Clemens Brunner Date: Fri, 7 Jul 2023 15:36:56 +0200 Subject: [PATCH 21/26] Refresh eegbci code and docstrings (#11783) --- mne/datasets/eegbci/eegbci.py | 79 +++++++++++++++++------------------ mne/datasets/utils.py | 4 +- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/mne/datasets/eegbci/eegbci.py b/mne/datasets/eegbci/eegbci.py index 4d5b3f9b7d6..76db5ef99ac 100644 --- a/mne/datasets/eegbci/eegbci.py +++ b/mne/datasets/eegbci/eegbci.py @@ -26,49 +26,48 @@ def data_path(url, path=None, force_update=False, update_path=None, *, verbose=None): """Get path to local copy of EEGMMI dataset URL. - This is a low-level function useful for getting a local copy of a - remote EEGBCI dataset :footcite:`SchalkEtAl2004` which is available at PhysioNet :footcite:`GoldbergerEtAl2000`. + This is a low-level function useful for getting a local copy of a remote EEGBCI + dataset :footcite:`SchalkEtAl2004`, which is also available at PhysioNet + :footcite:`GoldbergerEtAl2000`. Parameters ---------- url : str The dataset to use. - path : None | str - Location of where to look for the EEGBCI data storing location. - If None, the environment variable or config parameter - ``MNE_DATASETS_EEGBCI_PATH`` is used. If it doesn't exist, the - "~/mne_data" directory is used. If the EEGBCI dataset - is not found under the given path, the data - will be automatically downloaded to the specified folder. + path : None | path-like + Location of where to look for the EEGBCI data. If ``None``, the environment + variable or config parameter ``MNE_DATASETS_EEGBCI_PATH`` is used. If neither + exists, the ``~/mne_data`` directory is used. If the EEGBCI dataset is not found + under the given path, the data will be automatically downloaded to the specified + folder. force_update : bool Force update of the dataset even if a local copy exists. update_path : bool | None - If True, set the MNE_DATASETS_EEGBCI_PATH in mne-python - config to the given path. If None, the user is prompted. + If ``True``, set ``MNE_DATASETS_EEGBCI_PATH`` in the configuration to the given + path. If ``None``, the user is prompted. %(verbose)s Returns ------- path : list of Path - Local path to the given data file. This path is contained inside a list - of length one, for compatibility. + Local path to the given data file. This path is contained inside a list of + length one for compatibility. Notes ----- For example, one could do: >>> from mne.datasets import eegbci - >>> url = 'http://www.physionet.org/physiobank/database/eegmmidb/' - >>> eegbci.data_path(url, os.getenv('HOME') + '/datasets') # doctest:+SKIP + >>> url = "http://www.physionet.org/physiobank/database/eegmmidb/" + >>> eegbci.data_path(url, "~/datasets") # doctest:+SKIP - This would download the given EEGBCI data file to the 'datasets' folder, - and prompt the user to save the 'datasets' path to the mne-python config, - if it isn't there already. + This would download the given EEGBCI data file to the ``~/datasets`` folder and + prompt the user to store this path in the config (if it does not already exist). References ---------- .. footbibliography:: - """ # noqa: E501 + """ import pooch key = "MNE_DATASETS_EEGBCI_PATH" @@ -78,7 +77,7 @@ def data_path(url, path=None, force_update=False, update_path=None, *, verbose=N destination = _url_to_local_path(url, op.join(path, fname)) destinations = [destination] - # Fetch the file + # fetch the file downloader = pooch.HTTPDownloader(**_downloader_params()) if not op.isfile(destination) or force_update: if op.isfile(destination): @@ -92,7 +91,7 @@ def data_path(url, path=None, force_update=False, update_path=None, *, verbose=N fname=fname, ) - # Offer to update the path + # offer to update the path _do_path_update(path, update_path, key, name) destinations = [Path(dest) for dest in destinations] return destinations @@ -110,27 +109,26 @@ def load_data( ): # noqa: D301 """Get paths to local copies of EEGBCI dataset files. - This will fetch data for the EEGBCI dataset :footcite:`SchalkEtAl2004`, which is also - available at PhysioNet :footcite:`GoldbergerEtAl2000`. + This will fetch data for the EEGBCI dataset :footcite:`SchalkEtAl2004`, which is + also available at PhysioNet :footcite:`GoldbergerEtAl2000`. Parameters ---------- subject : int The subject to use. Can be in the range of 1-109 (inclusive). runs : int | list of int - The runs to use. See Notes for details. - path : None | str - Location of where to look for the EEGBCI data storing location. - If None, the environment variable or config parameter - ``MNE_DATASETS_EEGBCI_PATH`` is used. If it doesn't exist, the - "~/mne_data" directory is used. If the EEGBCI dataset - is not found under the given path, the data - will be automatically downloaded to the specified folder. + The runs to use (see Notes for details). + path : None | path-like + Location of where to look for the EEGBCI data. If ``None``, the environment + variable or config parameter ``MNE_DATASETS_EEGBCI_PATH`` is used. If neither + exists, the ``~/mne_data`` directory is used. If the EEGBCI dataset is not found + under the given path, the data will be automatically downloaded to the specified + folder. force_update : bool Force update of the dataset even if a local copy exists. update_path : bool | None - If True, set the MNE_DATASETS_EEGBCI_PATH in mne-python - config to the given path. If None, the user is prompted. + If ``True``, set ``MNE_DATASETS_EEGBCI_PATH`` in the configuration to the given + path. If ``None``, the user is prompted. base_url : str The URL root for the data. %(verbose)s @@ -158,17 +156,16 @@ def load_data( For example, one could do:: >>> from mne.datasets import eegbci - >>> eegbci.load_data(1, [4, 10, 14], os.getenv('HOME') + '/datasets') # doctest:+SKIP + >>> eegbci.load_data(1, [6, 10, 14], "~/datasets") # doctest:+SKIP - This would download runs 4, 10, and 14 (hand/foot motor imagery) runs from - subject 1 in the EEGBCI dataset to the 'datasets' folder, and prompt the - user to save the 'datasets' path to the mne-python config, if it isn't - there already. + This would download runs 6, 10, and 14 (hand/foot motor imagery) runs from subject 1 + in the EEGBCI dataset to "~/datasets" and prompt the user to store this path in the + config (if it does not already exist). References ---------- .. footbibliography:: - """ # noqa: E501 + """ import pooch t0 = time.time() @@ -196,8 +193,8 @@ def load_data( fetcher = pooch.create( path=base_path, base_url=base_url, - version=None, # Data versioning is decoupled from MNE-Python version. - registry=None, # Registry is loaded from file, below. + version=None, # data versioning is decoupled from MNE-Python version + registry=None, # registry is loaded from file (below) retry_if_failed=2, # 2 retries = 3 total attempts ) diff --git a/mne/datasets/utils.py b/mne/datasets/utils.py index 32ff152cd5e..041baf32812 100644 --- a/mne/datasets/utils.py +++ b/mne/datasets/utils.py @@ -103,7 +103,7 @@ def _get_path(path, key, name): # 1. Input _validate_type(path, ("path-like", None), path) if path is not None: - return path + return Path(path).expanduser() # 2. get_config(key) — unless key is None or "" (special get_config values) # 3. get_config('MNE_DATA') path = get_config(key or "MNE_DATA", get_config("MNE_DATA")) @@ -133,7 +133,7 @@ def _get_path(path, key, name): "write permissions, for ex:data_path" "('/home/xyz/me2/')" % (path) ) - return Path(path) + return Path(path).expanduser() def _do_path_update(path, update_path, key, name): From 30800ad71edcd1ae569c39083cd8d7dcc01c92ee Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 7 Jul 2023 10:45:37 -0400 Subject: [PATCH 22/26] DOC: Fix incorrect docs for annotate_movement [ci skip] (#11779) --- mne/preprocessing/artifact_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/preprocessing/artifact_detection.py b/mne/preprocessing/artifact_detection.py index 1a783dc5d62..54b106e2f37 100644 --- a/mne/preprocessing/artifact_detection.py +++ b/mne/preprocessing/artifact_detection.py @@ -166,9 +166,9 @@ def annotate_movement( The position and quaternion parameters from cHPI fitting. Obtained with `mne.chpi` functions. rotation_velocity_limit : float - Head rotation velocity limit in radians per second. + Head rotation velocity limit in degrees per second. translation_velocity_limit : float - Head translation velocity limit in radians per second. + Head translation velocity limit in meters per second. mean_distance_limit : float Head position limit from mean recording in meters. use_dev_head_trans : 'average' (default) | 'info' From 413e4a38601ce36e47018e9e4144d62a9d1c7a8e Mon Sep 17 00:00:00 2001 From: Samuel Louviot <44468686+Sam54000@users.noreply.github.com> Date: Sat, 8 Jul 2023 04:41:51 -0400 Subject: [PATCH 23/26] Keeping event names when combining channel on epochs objects (#11786) --- mne/channels/channels.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mne/channels/channels.py b/mne/channels/channels.py index 0f306b7b85e..1e60fc0295a 100644 --- a/mne/channels/channels.py +++ b/mne/channels/channels.py @@ -2300,6 +2300,7 @@ def combine_channels( new_data, info, events=inst.events, + event_id=inst.event_id, tmin=inst.times[0], baseline=inst.baseline, ) From 2a0b111349380429dcd5cab8abf1a211fd5eb408 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 8 Jul 2023 09:56:47 -0400 Subject: [PATCH 24/26] MAINT: Ignore warning in example (#11781) --- doc/conf.py | 8 ++- .../time_frequency/time_frequency_erds.py | 1 - mne/conftest.py | 4 ++ mne/io/tag.py | 44 +++++++------- mne/viz/tests/test_topomap.py | 13 ++++- mne/viz/topomap.py | 57 ++++++++++++------- tools/azure_dependencies.sh | 18 +++--- tools/github_actions_dependencies.sh | 14 ++--- tools/github_actions_test.sh | 6 +- 9 files changed, 97 insertions(+), 68 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 1c757b12413..2f441a0641a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1293,7 +1293,13 @@ def reset_warnings(gallery_conf, fname): ) warnings.filterwarnings( "ignore", - message=("Matplotlib is currently using agg, which is a non-GUI backend.*"), + message="Matplotlib is currently using agg, which is a non-GUI backend.*", + ) + # seaborn + warnings.filterwarnings( + "ignore", + message="The figure layout has changed to tight", + category=UserWarning, ) # matplotlib 3.6 in nilearn and pyvista warnings.filterwarnings("ignore", message=".*cmap function will be deprecated.*") diff --git a/examples/time_frequency/time_frequency_erds.py b/examples/time_frequency/time_frequency_erds.py index 72b5f36d172..7aae3e93bf2 100644 --- a/examples/time_frequency/time_frequency_erds.py +++ b/examples/time_frequency/time_frequency_erds.py @@ -45,7 +45,6 @@ from mne.time_frequency import tfr_multitaper from mne.stats import permutation_cluster_1samp_test as pcluster_test - # %% # First, we load and preprocess the data. We use runs 6, 10, and 14 from # subject 1 (these runs contains hand and feet motor imagery). diff --git a/mne/conftest.py b/mne/conftest.py index 616356396ce..acdda9a8170 100644 --- a/mne/conftest.py +++ b/mne/conftest.py @@ -156,6 +156,10 @@ def pytest_configure(config): ignore:`product` is deprecated as of NumPy.*:DeprecationWarning # pandas ignore:.*np\.find_common_type is deprecated.*:DeprecationWarning + # https://github.com/joblib/joblib/issues/1454 + ignore:.*`byte_bounds` is dep.*:DeprecationWarning + # TODO: we should fix this one + ignore:The provided callable.*:FutureWarning """ # noqa: E501 for warning_line in warning_lines.split("\n"): warning_line = warning_line.strip() diff --git a/mne/io/tag.py b/mne/io/tag.py index 291c02c59e0..9fa9dca5c16 100644 --- a/mne/io/tag.py +++ b/mne/io/tag.py @@ -51,17 +51,13 @@ def __init__(self, kind, type_, size, next, pos=None): # noqa: D102 self.data = None def __repr__(self): # noqa: D105 - out = "" def __eq__(self, tag): # noqa: D105 return int( @@ -96,7 +92,8 @@ def _frombuffer_rows(fid, tag_size, dtype=None, shape=None, rlims=None): have_shape = tag_size // item_size if want_shape != have_shape: raise ValueError( - "Wrong shape specified, requested %s have %s" % (want_shape, have_shape) + f"Wrong shape specified, requested {want_shape} but got " + f"{have_shape}" ) if not len(rlims) == 2: raise ValueError("rlims must have two elements") @@ -189,8 +186,8 @@ def _read_matrix(fid, tag, shape, rlims, matrix_coding): # This should be easy to implement (see _frombuffer_rows) # if we need it, but for now, it's not... - if shape is not None: - raise ValueError("Row reading not implemented for matrices " "yet") + if shape is not None or rlims is not None: + raise ValueError("Row reading not implemented for matrices yet") # Matrices if matrix_coding == _matrix_coding_dense: @@ -207,14 +204,16 @@ def _read_matrix(fid, tag, shape, rlims, matrix_coding): if ndim > 3: raise Exception( - "Only 2 or 3-dimensional matrices are " "supported at this time" + "Only 2 or 3-dimensional matrices are supported at this time" ) matrix_type = _data_type & tag.type try: bit, dtype = _matrix_bit_dtype[matrix_type] except KeyError: - raise RuntimeError("Cannot handle matrix of type %d yet" % matrix_type) + raise RuntimeError( + f"Cannot handle matrix of type {matrix_type} yet" + ) from None data = fid.read(int(bit * dims.prod())) data = np.frombuffer(data, dtype=dtype) # Note: we need the non-conjugate transpose here @@ -231,16 +230,17 @@ def _read_matrix(fid, tag, shape, rlims, matrix_coding): fid.seek(-(ndim + 2) * 4, 1) dims = np.frombuffer(fid.read(4 * (ndim + 1)), dtype=">i4") if ndim != 2: - raise Exception( - "Only two-dimensional matrices are " "supported at this time" - ) + raise Exception("Only two-dimensional matrices are supported at this time") # Back to where the data start fid.seek(pos, 0) nnz = int(dims[0]) nrow = int(dims[1]) ncol = int(dims[2]) - data = np.frombuffer(fid.read(4 * nnz), dtype=">f4") + # We need to make a copy so that we can own the data, otherwise we get: + # _sparsetools.csr_sort_indices(len(self.indptr) - 1, self.indptr, + # E ValueError: WRITEBACKIFCOPY base is read-only + data = np.frombuffer(fid.read(4 * nnz), dtype=">f4").copy() shape = (dims[1], dims[2]) if matrix_coding == _matrix_coding_CCS: # CCS @@ -277,7 +277,7 @@ def _read_matrix(fid, tag, shape, rlims, matrix_coding): indptr = np.frombuffer(tmp_ptr, dtype=" 0: - tp = cont.collections[0] - visible = tp.get_visible() - patch_ = tp.get_clip_path() - color = tp.get_edgecolors() - lw = tp.get_linewidth() - for tp in cont.collections: - tp.remove() + cont_collections = _cont_collections(cont) + for col in cont_collections: + col.remove() + col = cont_collections[0] + lw = col.get_linewidth() + visible = col.get_visible() + patch_ = col.get_clip_path() + color = col.get_edgecolors() cont = ax.contour( interp.Xi, interp.Yi, Zi, params["contours"], colors=color, linewidths=lw ) - for tp in cont.collections: - tp.set_visible(visible) - tp.set_clip_path(patch_) + cont_collections = _cont_collections(cont) + for col in cont_collections: + col.set_visible(visible) + col.set_clip_path(patch_) new_contours.append(cont) params["contours_"] = new_contours @@ -1280,7 +1293,7 @@ def _plot_topomap( if patch_ is not None: im.set_clip_path(patch_) if cont is not None: - for col in cont.collections: + for col in _cont_collections(cont): col.set_clip_path(patch_) pos_x, pos_y = pos.T @@ -2315,7 +2328,7 @@ def plot_evoked_topomap( plot_update_proj_callback=_plot_update_evoked_topomap, merge_channels=merge_channels, scale=scaling, - axes=axes, + axes=axes[: len(axes) - bool(interactive)], contours=contours, interp=interp, extrapolate=extrapolate, @@ -2984,14 +2997,15 @@ def _init_anim( params["text"] = text items.append(im) items.append(text) - for col in cont.collections: + cont_collections = _cont_collections(cont) + for col in cont_collections: col.set_clip_path(patch_) outlines_ = _draw_outlines(ax, outlines) params.update({"patch": patch_, "outlines": outlines_}) tight_layout(fig=ax.figure) - return tuple(items) + tuple(cont.collections) + return tuple(items) + cont_collections def _animate(frame, ax, ax_line, params): @@ -3042,7 +3056,8 @@ def _animate(frame, ax, ax_line, params): cont = ax.contour(Xi, Yi, Zi, levels=cont_lims, colors="k", linewidths=1) im.set_clip_path(patch) - for col in cont.collections: + cont_collections = _cont_collections(cont) + for col in cont_collections: col.set_clip_path(patch) items = [im, text] @@ -3055,7 +3070,7 @@ def _animate(frame, ax, ax_line, params): ax_line.set_ylim(ylim) items.append(params["line"]) params["frame"] = frame - return tuple(items) + tuple(cont.collections) + return tuple(items) + cont_collections def _pause_anim(event, params): diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh index 50bf4d902e6..380113d1127 100755 --- a/tools/azure_dependencies.sh +++ b/tools/azure_dependencies.sh @@ -5,18 +5,16 @@ if [ "${TEST_MODE}" == "pip" ]; then python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --only-binary="numba,llvmlite,numpy,scipy,vtk" -r requirements.txt elif [ "${TEST_MODE}" == "pip-pre" ]; then - python -m pip install --progress-bar off --upgrade pip setuptools wheel - python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" python-dateutil pytz joblib threadpoolctl six cycler kiwisolver pyparsing patsy - # Broken as of 2022/09/20 - # python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --no-deps --extra-index-url https://www.riverbankcomputing.com/pypi/simple PyQt6 PyQt6-sip PyQt6-Qt6 - python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --no-deps PyQt6 PyQt6-sip PyQt6-Qt6 - python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --no-deps -i "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy scipy statsmodels pandas scikit-learn dipy matplotlib - python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --no-deps -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" h5py - python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --no-deps -i "https://test.pypi.org/simple" openmeeg - python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --no-deps -i "https://wheels.vtk.org" vtk + python -m pip install --progress-bar off --upgrade pip setuptools wheel packaging + python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://www.riverbankcomputing.com/pypi/simple" PyQt6 PyQt6-sip PyQt6-Qt6 + python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" numpy scipy statsmodels pandas scikit-learn matplotlib + python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://pypi.anaconda.org/scipy-wheels-nightly/simple" dipy + python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" h5py + python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://wheels.vtk.org" vtk + python -m pip install --progress-bar off --upgrade --pre --only-binary ":all:" --extra-index-url "https://test.pypi.org/simple" openmeeg python -m pip install --progress-bar off git+https://github.com/pyvista/pyvista python -m pip install --progress-bar off git+https://github.com/pyvista/pyvistaqt - python -m pip install --progress-bar off --upgrade --pre imageio-ffmpeg xlrd mffpy python-picard patsy pillow + python -m pip install --progress-bar off --upgrade --pre imageio-ffmpeg xlrd mffpy python-picard pillow EXTRA_ARGS="--pre" ./tools/check_qt_import.sh PyQt6 else diff --git a/tools/github_actions_dependencies.sh b/tools/github_actions_dependencies.sh index 0391ef59df0..cec75a75e1a 100755 --- a/tools/github_actions_dependencies.sh +++ b/tools/github_actions_dependencies.sh @@ -11,23 +11,19 @@ elif [ ! -z "$CONDA_DEPENDENCIES" ]; then else echo "Install pip-pre dependencies" test "${MNE_CI_KIND}" == "pip-pre" - python -m pip install $STD_ARGS pip setuptools wheel + python -m pip install $STD_ARGS pip setuptools wheel packaging echo "Numpy" pip uninstall -yq numpy - echo "Date utils" - # https://pip.pypa.io/en/latest/user_guide/#possible-ways-to-reduce-backtracking-occurring - pip install $STD_ARGS --pre --only-binary ":all:" python-dateutil pytz joblib threadpoolctl six echo "PyQt6" - # Broken as of 2022/09/20 - # pip install $STD_ARGS --pre --only-binary ":all:" --no-deps --extra-index-url https://www.riverbankcomputing.com/pypi/simple PyQt6 - pip install $STD_ARGS --pre --only-binary ":all:" PyQt6 + pip install $STD_ARGS --pre --only-binary ":all:" --default-timeout=60 --extra-index-url https://www.riverbankcomputing.com/pypi/simple PyQt6 echo "NumPy/SciPy/pandas etc." - pip install $STD_ARGS --pre --only-binary ":all:" --default-timeout=60 --extra-index-url "https://pypi.anaconda.org/scipy-wheels-nightly/simple" numpy scipy scikit-learn dipy pandas matplotlib pillow statsmodels + pip install $STD_ARGS --pre --only-binary ":all:" --default-timeout=60 --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" numpy scipy scikit-learn pandas matplotlib pillow statsmodels + pip install $STD_ARGS --pre --only-binary ":all:" --default-timeout=60 --extra-index-url "https://pypi.anaconda.org/scipy-wheels-nightly/simple" dipy pip install $STD_ARGS --pre --only-binary ":all:" -f "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com" h5py + pip install $STD_ARGS --pre --only-binary ":all:" --extra-index-url "https://test.pypi.org/simple" openmeeg # No Numba because it forces an old NumPy version echo "nilearn and openmeeg" pip install $STD_ARGS --pre git+https://github.com/nilearn/nilearn - pip install $STD_ARGS --pre --only-binary ":all:" --extra-index-url "https://test.pypi.org/simple" openmeeg echo "VTK" pip install $STD_ARGS --pre --only-binary ":all:" --extra-index-url "https://wheels.vtk.org" vtk python -c "import vtk" diff --git a/tools/github_actions_test.sh b/tools/github_actions_test.sh index 512af0f4047..d4fa197dad4 100755 --- a/tools/github_actions_test.sh +++ b/tools/github_actions_test.sh @@ -3,7 +3,11 @@ set -eo pipefail if [[ "${CI_OS_NAME}" != "macos"* ]]; then - CONDITION="not (ultraslowtest or pgtest)" + if [[ "${MNE_CI_KIND}" == "pip-pre" ]]; then + CONDITION="not (slowtest or pgtest)" + else + CONDITION="not (ultraslowtest or pgtest)" + fi else CONDITION="not (slowtest or pgtest)" fi From 5608da261cc18f4813c1165baa8a6d5f64a1ea8c Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Sat, 8 Jul 2023 19:40:00 +0200 Subject: [PATCH 25/26] udpate doc for 11786 (#11787) --- doc/changes/latest.inc | 1 + doc/changes/names.inc | 2 ++ 2 files changed, 3 insertions(+) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index 5847cb5f17d..545af247f0c 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -34,6 +34,7 @@ Enhancements Bugs ~~~~ +- Fix bug where epochs ``event_id`` was not kept by :func:`mne.channels.combine_channels` (:gh:`11786` by :newcontrib:`Samuel Louviot`) - Fix bug where user-provided codec was not used to read annotations when loading EEGLAB ``.set`` files (:gh:`11773` by :newcontrib:`Yiping Zuo`) - Fix bug that required curv.*h files to create Brain object (:gh:`11704` by :newcontrib:`Aaron Earle-Richardson`) - Extended test to highlight bug in :func:`mne.stats.permutation_t_test` (:gh:`11575` by :newcontrib:`Joshua Calder-Travis`) diff --git a/doc/changes/names.inc b/doc/changes/names.inc index 3798fefcda6..2f3848cf2b6 100644 --- a/doc/changes/names.inc +++ b/doc/changes/names.inc @@ -458,6 +458,8 @@ .. _Samuel Deslauriers-Gauthier: https://github.com/sdeslauriers +.. _Samuel Louviot: https://github.com/Sam54000 + .. _Samuel Powell: https://github.com/samuelpowell .. _Santeri Ruuskanen: https://github.com/ruuskas From e28943603444353511bea8f22f5646153b7ec903 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 10 Jul 2023 13:05:06 -0400 Subject: [PATCH 26/26] MAINT: Better fix for deprecation (#11789) --- mne/conftest.py | 2 -- mne/time_frequency/tests/test_spectrum.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mne/conftest.py b/mne/conftest.py index acdda9a8170..1bfc74bfe89 100644 --- a/mne/conftest.py +++ b/mne/conftest.py @@ -158,8 +158,6 @@ def pytest_configure(config): ignore:.*np\.find_common_type is deprecated.*:DeprecationWarning # https://github.com/joblib/joblib/issues/1454 ignore:.*`byte_bounds` is dep.*:DeprecationWarning - # TODO: we should fix this one - ignore:The provided callable.*:FutureWarning """ # noqa: E501 for warning_line in warning_lines.split("\n"): warning_line = warning_line.strip() diff --git a/mne/time_frequency/tests/test_spectrum.py b/mne/time_frequency/tests/test_spectrum.py index 842fc3460eb..21a38198c83 100644 --- a/mne/time_frequency/tests/test_spectrum.py +++ b/mne/time_frequency/tests/test_spectrum.py @@ -250,9 +250,9 @@ def test_unaggregated_spectrum_to_data_frame(raw, long_format, method, output): def _fun(x): return np.nanmean(np.abs(x)) + agg_df = gb.agg(_fun) else: - _fun = np.nanmean - agg_df = gb.aggregate(_fun) + agg_df = gb.mean() # excludes missing values itself else: gb = gb[df.columns] # https://github.com/pandas-dev/pandas/pull/52477 agg_df = gb.apply(_agg_helper, spectrum._mt_weights, grouping_cols)