From 945c1431537784263364648734f8c7612b8a7518 Mon Sep 17 00:00:00 2001 From: Robert Luke <748691+rob-luke@users.noreply.github.com> Date: Fri, 12 Jun 2020 13:51:53 +1000 Subject: [PATCH 1/8] Add meas_date and age to nirx io --- mne/io/nirx/nirx.py | 52 ++++++++++++++++++++++++++++------ mne/io/nirx/tests/test_nirx.py | 37 ++++++++++++++++++++---- 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index fdd78ac236b..293e4504ba6 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -6,6 +6,8 @@ import glob as glob import re as re import os.path as op +import datetime as dt +import locale import numpy as np @@ -15,8 +17,7 @@ from ..meas_info import create_info, _format_dig_points from ...annotations import Annotations from ...transforms import apply_trans, _get_trans -from ...utils import logger, verbose, fill_doc -from ...utils import warn +from ...utils import logger, verbose, fill_doc, warn @fill_doc @@ -118,22 +119,51 @@ def __init__(self, fname, preload=False, verbose=None): # Parse required header fields + # Extract measurement date and time + datetime_str = hdr['GeneralInfo']['Date'] + hdr['GeneralInfo']['Time'] + meas_date = None + # Several formats have been observed so we try each in turn + try: + dt_code = '"%a, %b %d, %Y""%H:%M:%S.%f"' + meas_date = dt.datetime.strptime(datetime_str, dt_code) + meas_date = meas_date.replace(tzinfo=dt.timezone.utc) + except ValueError: + pass + try: + dt_code = '"%a, %d %b %Y""%H:%M:%S.%f"' + meas_date = dt.datetime.strptime(datetime_str, dt_code) + meas_date = meas_date.replace(tzinfo=dt.timezone.utc) + except ValueError: + pass + if meas_date is None: + warn("Extraction of measurement date from NIRX file failed. " + "This can be caused by files saved in certain locales. " + "Please report this as a github issue. " + "The date is being set to January 1st, 2000, " + "instead of {}".format(datetime_str)) + meas_date = dt.datetime(2000, 1, 1, 0, 0, 0, + tzinfo=dt.timezone.utc) + # Extract frequencies of light used by machine fnirs_wavelengths = [int(s) for s in re.findall(r'(\d+)', - hdr['ImagingParameters']['Wavelengths'])] + hdr['ImagingParameters'][ + 'Wavelengths'])] # Extract source-detectors sources = np.asarray([int(s) for s in re.findall(r'(\d+)-\d+:\d+', - hdr['DataStructure']['S-D-Key'])], int) + hdr['DataStructure'][ + 'S-D-Key'])], int) detectors = np.asarray([int(s) for s in re.findall(r'\d+-(\d+):\d+', - hdr['DataStructure']['S-D-Key'])], int) + hdr['DataStructure'] + ['S-D-Key'])], + int) # Determine if short channels are present and on which detectors if 'shortbundles' in hdr['ImagingParameters']: short_det = [int(s) for s in re.findall(r'(\d+)', - hdr['ImagingParameters']['ShortDetIndex'])] + hdr['ImagingParameters']['ShortDetIndex'])] short_det = np.array(short_det, int) else: short_det = [] @@ -150,6 +180,7 @@ def __init__(self, fname, preload=False, verbose=None): # Note: NIRX also records "Study Type", "Experiment History", # "Additional Notes", "Contact Information" and this information # is currently discarded + # NIRStar does not record an id, or handedness by default subject_info = {} names = inf['name'].split() if len(names) > 0: @@ -161,7 +192,6 @@ def __init__(self, fname, preload=False, verbose=None): if len(names) > 2: subject_info['middle_name'] = \ inf['name'].split()[-2].replace("\"", "") - # subject_info['birthday'] = inf['age'] # TODO: not formatted properly subject_info['sex'] = inf['gender'].replace("\"", "") # Recode values if subject_info['sex'] in {'M', 'Male', '1'}: @@ -170,7 +200,9 @@ def __init__(self, fname, preload=False, verbose=None): subject_info['sex'] = FIFF.FIFFV_SUBJ_SEX_FEMALE else: subject_info['sex'] = FIFF.FIFFV_SUBJ_SEX_UNKNOWN - # NIRStar does not record an id, or handedness by default + subject_info['birthday'] = (meas_date.year - int(inf['age']), + meas_date.month, + meas_date.day) # Read information about probe/montage/optodes # A word on terminology used here: @@ -222,7 +254,8 @@ def __init__(self, fname, preload=False, verbose=None): def prepend(list, str): str += '{0}' list = [str.format(i) for i in list] - return(list) + return (list) + snames = prepend(sources[req_ind], 'S') dnames = prepend(detectors[req_ind], '_D') sdnames = [m + str(n) for m, n in zip(snames, dnames)] @@ -235,6 +268,7 @@ def prepend(list, str): samplingrate, ch_types='fnirs_cw_amplitude') info.update(subject_info=subject_info, dig=dig) + info['meas_date'] = meas_date # Store channel, source, and detector locations # The channel location is stored in the first 3 entries of loc. diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index 366dbee49da..e399b9eb89b 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -79,6 +79,12 @@ def test_nirx_15_2_short(): # Test data import assert raw._data.shape == (26, 145) assert raw.info['sfreq'] == 12.5 + assert raw.info['meas_date'].month == 8 + assert raw.info['meas_date'].day == 23 + assert raw.info['meas_date'].year == 2019 + assert raw.info['meas_date'].hour == 7 + assert raw.info['meas_date'].minute == 37 + assert raw.info['meas_date'].second == 4 # Test channel naming assert raw.info['ch_names'][:4] == ["S1_D1 760", "S1_D1 850", @@ -92,7 +98,8 @@ def test_nirx_15_2_short(): # Test info import assert raw.info['subject_info'] == dict(sex=1, first_name="MNE", middle_name="Test", - last_name="Recording") + last_name="Recording", + birthday=(2014, 8, 23)) # Test distance between optodes matches values from # nirsite https://github.com/mne-tools/mne-testing-data/pull/51 @@ -179,8 +186,10 @@ def test_nirx_15_3_short(): assert raw.info['chs'][1]['loc'][9] == 850 # Test info import - assert raw.info['subject_info'] == dict( - sex=0, first_name="testMontage\\0ATestMontage") + assert raw.info['subject_info'] == dict(birthday=(2020, 8, 18), + sex=0, + first_name="testMontage\\0A" + "TestMontage") # Test distance between optodes matches values from # https://github.com/mne-tools/mne-testing-data/pull/72 @@ -257,7 +266,8 @@ def test_encoding(tmpdir): for line in hdr: fid.write(line) # smoke test - read_raw_nirx(fname) + with pytest.raises(RuntimeWarning, match='Extraction of measurement date'): + read_raw_nirx(fname) @requires_testing_data @@ -268,16 +278,24 @@ def test_nirx_15_2(): # Test data import assert raw._data.shape == (64, 67) assert raw.info['sfreq'] == 3.90625 + assert raw.info['meas_date'].month == 10 + assert raw.info['meas_date'].day == 2 + assert raw.info['meas_date'].year == 2019 + assert raw.info['meas_date'].hour == 9 + assert raw.info['meas_date'].minute == 8 + assert raw.info['meas_date'].second == 47 # Test channel naming assert raw.info['ch_names'][:4] == ["S1_D1 760", "S1_D1 850", "S1_D10 760", "S1_D10 850"] # Test info import - assert raw.info['subject_info'] == dict(sex=1, first_name="TestRecording") + assert raw.info['subject_info'] == dict(sex=1, first_name="TestRecording", + birthday=(1989, 10, 2)) # Test trigger events assert_array_equal(raw.annotations.description, ['4.0', '6.0', '2.0']) + print(raw.annotations.onset) # Test location of detectors allowed_dist_error = 0.0002 @@ -312,6 +330,12 @@ def test_nirx_15_0(): # Test data import assert raw._data.shape == (20, 92) assert raw.info['sfreq'] == 6.25 + assert raw.info['meas_date'].month == 10 + assert raw.info['meas_date'].day == 27 + assert raw.info['meas_date'].year == 2019 + assert raw.info['meas_date'].hour == 13 + assert raw.info['meas_date'].minute == 53 + assert raw.info['meas_date'].second == 34 # Test channel naming assert raw.info['ch_names'][:12] == ["S1_D1 760", "S1_D1 850", @@ -322,7 +346,8 @@ def test_nirx_15_0(): "S6_D6 760", "S6_D6 850"] # Test info import - assert raw.info['subject_info'] == {'first_name': 'NIRX', + assert raw.info['subject_info'] == {'birthday': (2004, 10, 27), + 'first_name': 'NIRX', 'last_name': 'Test', 'sex': FIFF.FIFFV_SUBJ_SEX_UNKNOWN} From e1757b80116d91edfbfdbf305d22853c0e29f7ea Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Tue, 8 Sep 2020 11:04:57 +1000 Subject: [PATCH 2/8] Remove unused import --- mne/io/nirx/nirx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 293e4504ba6..f025fde1380 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -7,7 +7,6 @@ import re as re import os.path as op import datetime as dt -import locale import numpy as np From 43243751aab6cb35eb3cb48bf797249a899be824 Mon Sep 17 00:00:00 2001 From: Robert Luke <748691+rob-luke@users.noreply.github.com> Date: Wed, 9 Sep 2020 09:33:27 +1000 Subject: [PATCH 3/8] Update mne/io/nirx/nirx.py Co-authored-by: Alexandre Gramfort --- mne/io/nirx/nirx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index f025fde1380..a017cf0ad81 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -253,7 +253,7 @@ def __init__(self, fname, preload=False, verbose=None): def prepend(list, str): str += '{0}' list = [str.format(i) for i in list] - return (list) + return list snames = prepend(sources[req_ind], 'S') dnames = prepend(detectors[req_ind], '_D') From 49ecb4f7aae4d0eaeda7eb479a2d59326c51f1ac Mon Sep 17 00:00:00 2001 From: Robert Luke <748691+rob-luke@users.noreply.github.com> Date: Wed, 9 Sep 2020 09:34:02 +1000 Subject: [PATCH 4/8] Update mne/io/nirx/nirx.py Co-authored-by: Alexandre Gramfort --- mne/io/nirx/nirx.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index a017cf0ad81..930a5d98ae1 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -123,15 +123,10 @@ def __init__(self, fname, preload=False, verbose=None): meas_date = None # Several formats have been observed so we try each in turn try: - dt_code = '"%a, %b %d, %Y""%H:%M:%S.%f"' - meas_date = dt.datetime.strptime(datetime_str, dt_code) - meas_date = meas_date.replace(tzinfo=dt.timezone.utc) - except ValueError: - pass - try: - dt_code = '"%a, %d %b %Y""%H:%M:%S.%f"' - meas_date = dt.datetime.strptime(datetime_str, dt_code) - meas_date = meas_date.replace(tzinfo=dt.timezone.utc) + for dt_code in ['"%a, %b %d, %Y""%H:%M:%S.%f"', '"%a, %d %b %Y""%H:%M:%S.%f"']: + meas_date = dt.datetime.strptime(datetime_str, dt_code) + meas_date = meas_date.replace(tzinfo=dt.timezone.utc) + break except ValueError: pass if meas_date is None: From 42f60f569232dcc38513169c48313827024e89a4 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 9 Sep 2020 09:44:26 +1000 Subject: [PATCH 5/8] Get date for loop to work --- mne/io/nirx/nirx.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 930a5d98ae1..8256d1d0721 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -122,13 +122,14 @@ def __init__(self, fname, preload=False, verbose=None): datetime_str = hdr['GeneralInfo']['Date'] + hdr['GeneralInfo']['Time'] meas_date = None # Several formats have been observed so we try each in turn - try: - for dt_code in ['"%a, %b %d, %Y""%H:%M:%S.%f"', '"%a, %d %b %Y""%H:%M:%S.%f"']: + for dt_code in ['"%a, %b %d, %Y""%H:%M:%S.%f"', + '"%a, %d %b %Y""%H:%M:%S.%f"']: + try: meas_date = dt.datetime.strptime(datetime_str, dt_code) meas_date = meas_date.replace(tzinfo=dt.timezone.utc) break - except ValueError: - pass + except ValueError: + pass if meas_date is None: warn("Extraction of measurement date from NIRX file failed. " "This can be caused by files saved in certain locales. " From 13f7e7b646b7ef693bcaca3d82bdd6cf0fcfa7e1 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 9 Sep 2020 09:47:38 +1000 Subject: [PATCH 6/8] Shorten tests --- mne/io/nirx/tests/test_nirx.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/mne/io/nirx/tests/test_nirx.py b/mne/io/nirx/tests/test_nirx.py index e399b9eb89b..109ad57bbbc 100644 --- a/mne/io/nirx/tests/test_nirx.py +++ b/mne/io/nirx/tests/test_nirx.py @@ -6,6 +6,7 @@ import os.path as op import shutil import os +import datetime as dt import pytest from numpy.testing import assert_allclose, assert_array_equal @@ -79,12 +80,8 @@ def test_nirx_15_2_short(): # Test data import assert raw._data.shape == (26, 145) assert raw.info['sfreq'] == 12.5 - assert raw.info['meas_date'].month == 8 - assert raw.info['meas_date'].day == 23 - assert raw.info['meas_date'].year == 2019 - assert raw.info['meas_date'].hour == 7 - assert raw.info['meas_date'].minute == 37 - assert raw.info['meas_date'].second == 4 + assert raw.info['meas_date'] == dt.datetime(2019, 8, 23, 7, 37, 4, 540000, + tzinfo=dt.timezone.utc) # Test channel naming assert raw.info['ch_names'][:4] == ["S1_D1 760", "S1_D1 850", @@ -278,12 +275,8 @@ def test_nirx_15_2(): # Test data import assert raw._data.shape == (64, 67) assert raw.info['sfreq'] == 3.90625 - assert raw.info['meas_date'].month == 10 - assert raw.info['meas_date'].day == 2 - assert raw.info['meas_date'].year == 2019 - assert raw.info['meas_date'].hour == 9 - assert raw.info['meas_date'].minute == 8 - assert raw.info['meas_date'].second == 47 + assert raw.info['meas_date'] == dt.datetime(2019, 10, 2, 9, 8, 47, 511000, + tzinfo=dt.timezone.utc) # Test channel naming assert raw.info['ch_names'][:4] == ["S1_D1 760", "S1_D1 850", @@ -330,12 +323,9 @@ def test_nirx_15_0(): # Test data import assert raw._data.shape == (20, 92) assert raw.info['sfreq'] == 6.25 - assert raw.info['meas_date'].month == 10 - assert raw.info['meas_date'].day == 27 - assert raw.info['meas_date'].year == 2019 - assert raw.info['meas_date'].hour == 13 - assert raw.info['meas_date'].minute == 53 - assert raw.info['meas_date'].second == 34 + assert raw.info['meas_date'] == dt.datetime(2019, 10, 27, 13, 53, 34, + 209000, + tzinfo=dt.timezone.utc) # Test channel naming assert raw.info['ch_names'][:12] == ["S1_D1 760", "S1_D1 850", From 36bdd1e2d46794f7f2e94708f78a5febc2bd3ab6 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 9 Sep 2020 09:48:51 +1000 Subject: [PATCH 7/8] Dont use the word list --- mne/io/nirx/nirx.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index 8256d1d0721..ae7c8ab1c53 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -246,10 +246,10 @@ def __init__(self, fname, preload=False, verbose=None): req_ind = req_ind.astype(int) # Generate meaningful channel names - def prepend(list, str): + def prepend(l, str): str += '{0}' - list = [str.format(i) for i in list] - return list + l = [str.format(i) for i in l] + return l snames = prepend(sources[req_ind], 'S') dnames = prepend(detectors[req_ind], '_D') From b3b66402162d2fe34d0871292541fbf11d583022 Mon Sep 17 00:00:00 2001 From: Robert Luke Date: Wed, 9 Sep 2020 10:58:22 +1000 Subject: [PATCH 8/8] Flake fix --- mne/io/nirx/nirx.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mne/io/nirx/nirx.py b/mne/io/nirx/nirx.py index ae7c8ab1c53..c569ea95d00 100644 --- a/mne/io/nirx/nirx.py +++ b/mne/io/nirx/nirx.py @@ -246,10 +246,10 @@ def __init__(self, fname, preload=False, verbose=None): req_ind = req_ind.astype(int) # Generate meaningful channel names - def prepend(l, str): + def prepend(li, str): str += '{0}' - l = [str.format(i) for i in l] - return l + li = [str.format(i) for i in li] + return li snames = prepend(sources[req_ind], 'S') dnames = prepend(detectors[req_ind], '_D')