[ENH] Upgrade custom entities

.github/workflows/coverage.yml

@@ -8,10 +8,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.10.11
+    - name: Set up Python 3.10
       uses: actions/setup-python@v1
       with:
-        python-version: 3.10.11
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/publish_doc.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: "3.10.11"
+          python-version: "3.10"
       - name: Upgrade pip
         run: |
           # install pip=>20.1 to use "pip cache dir"

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ jobs:
           - ubuntu-latest  # ubuntu-18.04
           - macos-latest  # macOS-10.14
           - windows-latest  # windows-2019
-        python-version: [3.8, 3.9, 3.10.11]
+        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v1

dcm2bids/acquisition.py

@@ -17,7 +17,7 @@ class Acquisition(object):
         dataType (str): A functional group of MRI data (ex: func, anat ...)
         modalityLabel (str): The modality of the acquisition
                 (ex: T1w, T2w, bold ...)
-        customLabels (str): Optional labels (ex: task-rest)
+        customEntities (str): Optional entities (ex: task-rest)
         srcSidecar (Sidecar): Optional sidecar object
     """
 
@@ -26,7 +26,7 @@ def __init__(
         participant,
         dataType,
         modalityLabel,
-        customLabels="",
+        customEntities="",
         id=None,
         srcSidecar=None,
         sidecarChanges=None,
@@ -37,14 +37,14 @@ def __init__(
         self.logger = logging.getLogger(__name__)
 
         self._modalityLabel = ""
-        self._customLabels = ""
+        self._customEntities = ""
         self._id = ""
         self._intendedFor = None
 
         self.participant = participant
         self.dataType = dataType
         self.modalityLabel = modalityLabel
-        self.customLabels = customLabels
+        self.customEntities = customEntities
         self.srcSidecar = srcSidecar
 
         if sidecarChanges is None:
@@ -97,29 +97,32 @@ def id(self, value):
         self._id = value
 
     @property
-    def customLabels(self):
+    def customEntities(self):
         """
         Returns:
-            A string '_<customLabels>'
+            A string '_<customEntities>'
         """
-        return self._customLabels
+        return self._customEntities
 
-    @customLabels.setter
-    def customLabels(self, customLabels):
+    @customEntities.setter
+    def customEntities(self, customEntities):
         """ Prepend '_' if necessary"""
-        self._customLabels = self.prepend(customLabels)
+        if isinstance(customEntities, list):
+            self._customEntities = self.prepend('_'.join(customEntities))
+        else:
+            self._customEntities = self.prepend(customEntities)
 
     @property
     def suffix(self):
         """ The suffix to build filenames
 
         Returns:
-            A string '_<modalityLabel>' or '_<customLabels>_<modalityLabel>'
+            A string '_<modalityLabel>' or '_<customEntities>_<modalityLabel>'
         """
-        if self.customLabels.strip() == "":
+        if self.customEntities.strip() == "":
             return self.modalityLabel
         else:
-            return self.customLabels + self.modalityLabel
+            return self.customEntities + self.modalityLabel
 
     @property
     def srcRoot(self):
@@ -214,6 +217,10 @@ def dstSidecarData(self, intendedForList):
         data = self.srcSidecar.origData
         data["Dcm2bidsVersion"] = __version__
 
+        # TaskName
+        if 'TaskName' in self.srcSidecar.data:
+            data["TaskName"] = self.srcSidecar.data["TaskName"]
+
         # intendedFor key
         if self.intendedFor != [None]:
             intendedValue = []

dcm2bids/cli/dcm2bids.py

@@ -38,6 +38,11 @@ def _build_arg_parser():
                    default=DEFAULT.cliOutputDir,
                    help="Output BIDS directory. [%(default)s]")
 
+    p.add_argument("--auto_extract_entities",
+                   action='store_true',
+                   help="If set, it will automatically try to extract entity information [task, dir, echo]"
+                        " depending on the suffix and dataType. [%(default)s]")
+
     p.add_argument("--bids_validate",
                    action='store_true',
                    help="If set, once your conversion is done it"

dcm2bids/dcm2bids_gen.py

@@ -44,6 +44,7 @@ def __init__(
         config,
         output_dir=DEFAULT.outputDir,
         bids_validate=DEFAULT.bids_validate,
+        auto_extract_entities=DEFAULT.auto_extract_entities,
         session=DEFAULT.session,
         clobber=DEFAULT.clobber,
         forceDcm2niix=DEFAULT.forceDcm2niix,
@@ -58,6 +59,7 @@ def __init__(
         self.participant = Participant(participant, session)
         self.clobber = clobber
         self.bids_validate = bids_validate
+        self.auto_extract_entities = auto_extract_entities
         self.forceDcm2niix = forceDcm2niix
         self.logLevel = log_level
 
@@ -73,6 +75,7 @@ def __init__(
         self.logger.info("session: %s", self.participant.session)
         self.logger.info("config: %s", os.path.realpath(config))
         self.logger.info("BIDS directory: %s", os.path.realpath(output_dir))
+        self.logger.info("Auto extract entities: %s", self.auto_extract_entities)
         self.logger.info("Validate BIDS: %s", self.bids_validate)
 
     @property
@@ -123,6 +126,8 @@ def run(self):
         parser = SidecarPairing(
             sidecars,
             self.config["descriptions"],
+            self.config.get("extractors", DEFAULT.extractors),
+            self.auto_extract_entities,
             self.config.get("searchMethod", DEFAULT.searchMethod),
             self.config.get("caseSensitive", DEFAULT.caseSensitive)
         )
@@ -134,7 +139,6 @@ def run(self):
 
         intendedForList = {}
         for acq in parser.acquisitions:
-            acq.setDstFile()
             intendedForList = self.move(acq, intendedForList)
 
         if self.bids_validate:
@@ -178,7 +182,6 @@ def move(self, acquisition, intendedForList):
                 else:
                     intendedForList[acquisition.id] = [acquisition.dstIntendedFor + "".join(ext)]
 
-            # it's an anat nifti file and the user using a deface script
             if (self.config.get("defaceTpl") and acquisition.dataType == "anat" and ".nii" in ext):
                 try:
                     os.remove(dstFile)

dcm2bids/sidecar.py

@@ -10,8 +10,8 @@
 from fnmatch import fnmatch
 
 from dcm2bids.acquisition import Acquisition
-from dcm2bids.utils.utils import DEFAULT, splitext_
 from dcm2bids.utils.io import load_json
+from dcm2bids.utils.utils import DEFAULT, convert_dir, splitext_
 
 
 class Sidecar(object):
@@ -90,14 +90,17 @@ class SidecarPairing(object):
         descriptions (list): List of dictionaries describing acquisitions
     """
 
-    def __init__(self, sidecars, descriptions, searchMethod=DEFAULT.searchMethod,
-                 caseSensitive=DEFAULT.caseSensitive):
+    def __init__(self, sidecars, descriptions, extractors=DEFAULT.extractors,
+                 auto_extractor=DEFAULT.auto_extract_entities,
+                 searchMethod=DEFAULT.searchMethod, caseSensitive=DEFAULT.caseSensitive):
         self.logger = logging.getLogger(__name__)
 
         self._searchMethod = ""
         self.graph = OrderedDict()
         self.acquisitions = []
 
+        self.extractors = extractors
+        self.auto_extract_entities = auto_extractor
         self.sidecars = sidecars
         self.descriptions = descriptions
         self.searchMethod = searchMethod
@@ -224,6 +227,8 @@ def build_acquisitions(self, participant):
             # only one description for the sidecar
             if len(valid_descriptions) == 1:
                 desc = valid_descriptions[0]
+                desc, sidecar = self.searchDcmTagEntity(sidecar, desc)
+
                 acq = Acquisition(participant,
                                   srcSidecar=sidecar, **desc)
                 acq.setDstFile()
@@ -233,7 +238,7 @@ def build_acquisitions(self, participant):
                 else:
                     acquisitions.append(acq)
 
-                self.logger.info("%s  <-  %s", acq.suffix, sidecarName)
+                self.logger.info("%s  <-  %s", acq.dstFile.replace(acq.participant.prefix + "-", ""), sidecarName)
 
             # sidecar with no link
             elif len(valid_descriptions) == 0:
@@ -251,10 +256,78 @@ def build_acquisitions(self, participant):
 
         return self.acquisitions
 
+    def searchDcmTagEntity(self, sidecar, desc):
+        """
+        Add DCM Tag to customEntities
+        """
+        descWithTask = desc.copy()
+        concatenated_matches = {}
+        entities = []
+
+        if "customEntities" in desc.keys() or self.auto_extract_entities:
+            if 'customEntities' in desc.keys():
+                if isinstance(descWithTask["customEntities"], str):
+                    descWithTask["customEntities"] = [descWithTask["customEntities"]]
+            else:
+                descWithTask["customEntities"] = []
+
+            if self.auto_extract_entities:
+                self.extractors.update(DEFAULT.auto_extractors)
+
+            for dcmTag in self.extractors:
+                if dcmTag in sidecar.data.keys():
+                    dcmInfo = sidecar.data.get(dcmTag)
+                    for regex in self.extractors[dcmTag]:
+                        compile_regex = re.compile(regex)
+                        if not isinstance(dcmInfo, list):
+                            if compile_regex.search(str(dcmInfo)) is not None:
+                                concatenated_matches.update(compile_regex.search(str(dcmInfo)).groupdict())
+                        else:
+                            for curr_dcmInfo in dcmInfo:
+                                if compile_regex.search(curr_dcmInfo) is not None:
+                                    concatenated_matches.update(compile_regex.search(curr_dcmInfo).groupdict())
+                                    break
+
+            if "customEntities" in desc.keys():
+                entities = set(concatenated_matches.keys()).union(set(descWithTask["customEntities"]))
+                    #entities_left = set(concatenated_matches.keys()).symmetric_difference(set(descWithTask["customEntities"]))
+
+            if self.auto_extract_entities:
+                auto_acq = '_'.join([descWithTask['dataType'], descWithTask["modalityLabel"]])
+                if auto_acq in DEFAULT.auto_entities:
+                    # Check if these auto entities have been found before merging
+                    auto_entities = set(concatenated_matches.keys()).intersection(set(DEFAULT.auto_entities[auto_acq]))
+                    left_auto_entities = auto_entities.symmetric_difference(set(DEFAULT.auto_entities[auto_acq]))
+                    if left_auto_entities:
+                        self.logger.warning(f"{left_auto_entities} have not been found for dataType '{descWithTask['dataType']}' "
+                                            f"and suffix '{descWithTask['modalityLabel']}'.")
+                    else:
+                        entities = list(entities) + DEFAULT.auto_entities[auto_acq]
+                        entities = list(set(entities))
+                        descWithTask["customEntities"] = entities
+
+            for curr_entity in entities:
+                if curr_entity in concatenated_matches.keys():
+                    if curr_entity == 'dir':
+                        descWithTask["customEntities"] = list(map(lambda x: x.replace(curr_entity, '-'.join([curr_entity, convert_dir(concatenated_matches[curr_entity])])), descWithTask["customEntities"]))
+                    elif curr_entity == 'task':
+                        sidecar.data['TaskName'] = concatenated_matches[curr_entity]
+                        descWithTask["customEntities"] = list(map(lambda x: x.replace(curr_entity, '-'.join([curr_entity, concatenated_matches[curr_entity]])), descWithTask["customEntities"]))
+                    else:
+                        descWithTask["customEntities"] = list(map(lambda x: x.replace(curr_entity, '-'.join([curr_entity, concatenated_matches[curr_entity]])), descWithTask["customEntities"]))
+
+            # Remove entities without -
+            for curr_entity in descWithTask["customEntities"]:
+                if '-' not in curr_entity:
+                    self.logger.info(f"Removing entity '{curr_entity}' since it does not fit the basic BIDS specification (Entity-Value)")
+                    descWithTask["customEntities"].remove(curr_entity)
+
+        return descWithTask, sidecar
+
     def find_runs(self):
         """
         Check if there is duplicate destination roots in the acquisitions
-        and add '_run-' to the customLabels of the acquisition
+        and add '_run-' to the customEntities of the acquisition
         """
 
         def duplicates(seq):
@@ -282,4 +355,5 @@ def duplicates(seq):
             self.logger.info("Adding 'run' information to the acquisition")
             for runNum, acqInd in enumerate(dup):
                 runStr = DEFAULT.runTpl.format(runNum + 1)
-                self.acquisitions[acqInd].customLabels += runStr
+                self.acquisitions[acqInd].customEntities += runStr
+                self.acquisitions[acqInd].setDstFile()

dcm2bids/utils/io.py

@@ -34,28 +34,6 @@ def write_txt(filename, lines):
         f.write(f"{lines}\n")
 
 
-def get_scaffold_dir():
-    """
-    Return SCAFFOLD data directory in dcm2bids repository
-
-    Returns
-    -------
-    scaffold_dir: string
-        SCAFFOLD path
-    """
-
-    module_path = os.path.dirname(os.path.dirname(inspect.getfile(dcm2bids)))
-    # module_path = inspect.getfile(dcm2bids)
-    scaffold_dir = opj(module_path, 'data', 'scaffold')
-    # scaffold_dir = pkg_resources.resource_filename(__name__,  os.path.join("data", "scaffold"))
-    # print(module_path)
-    # scaffold_dir = os.path.join(os.path.dirname(
-    # os.path.dirname(module_path)), "data", "scaffold")
-
-    print(scaffold_dir)
-    return scaffold_dir
-
-
 def valid_path(in_path, type="folder"):
     """Assert that file exists.