Update to BIDS 1.9

Dockerfile

@@ -0,0 +1,57 @@
+# Conda image for installing FSL tools
+FROM continuumio/miniconda3 AS build
+
+# Install FSL tools with conda
+COPY environment.yml /tmp/environment.yml
+RUN conda env create -f /tmp/environment.yml
+
+# Install and use conda-pack
+RUN conda install -c conda-forge conda-pack
+RUN conda-pack -n fsl -o /tmp/env.tar && \
+  mkdir /venv && cd /venv && tar xf /tmp/env.tar && \
+  rm /tmp/env.tar
+RUN /venv/bin/conda-unpack
+
+# Runtime image for executing FSL tools
+FROM debian:stable AS runtime
+
+# Copy the conda env from previous stage
+COPY --from=build /venv /venv
+
+# Point to conda executables
+ENV PATH="/venv/bin:$PATH"
+
+# Set FSL variables
+ENV FSLDIR="/venv" 
+ENV FSLCONFDIR="${FSLDIR}/config"
+ENV FSLOUTPUTTYPE="NIFTI"
+ENV FSLMULTIFILEQUIT="TRUE"
+ENV FSLTCLSH="${FSLDIR}/bin/fsltclsh"
+ENV FSLWISH="${FSLDIR}/bin/fslwish"
+ENV FSLGECUDAQ="cuda.q"
+
+# Update and install some utils
+RUN apt-get -y update && apt-get -y install dc wget npm unzip
+
+# Fetch data
+RUN wget -P ${FSLDIR}/data https://git.fmrib.ox.ac.uk/fsl/data_standard/-/raw/master/MNI152_T1_1mm_brain.nii.gz
+
+# Install bids-validator
+RUN npm install -g bids-validator
+
+# Install dcm2niix
+WORKDIR /
+RUN wget https://github.com/rordenlab/dcm2niix/releases/download/v1.0.20240202/dcm2niix_lnx.zip
+RUN unzip dcm2niix_lnx.zip
+RUN mv dcm2niix /usr/bin/
+
+# Install dcm2bids
+
+WORKDIR /
+ADD . /dcm2bids
+WORKDIR /dcm2bids
+RUN pip install -e .
+
+RUN pip install pydeface
+
+ENTRYPOINT ["dcm2bids"]

dcm2bids/acquisition.py

@@ -156,8 +156,8 @@ def setExtraDstFile(self, new_entities):
         """
         Return:
             The destination filename formatted following
-            the v1.8.0 BIDS entity key table
-            https://bids-specification.readthedocs.io/en/v1.8.0/99-appendices/04-entity-table.html
+            the v1.9.0 BIDS entity key table
+            https://bids-specification.readthedocs.io/en/v1.9.0/99-appendices/04-entity-table.html
         """
 
         if self.custom_entities.strip() == "":
@@ -210,8 +210,8 @@ def setDstFile(self):
         """
         Return:
             The destination filename formatted following
-            the v1.8.0 BIDS entity key table
-            https://bids-specification.readthedocs.io/en/v1.8.0/99-appendices/04-entity-table.html
+            the v1.9.0 BIDS entity key table
+            https://bids-specification.readthedocs.io/en/v1.9.0/99-appendices/04-entity-table.html
         """
         current_name = self.participant.prefix + self.build_suffix
         new_name = ''

dcm2bids/dcm2bids_gen.py

@@ -123,10 +123,12 @@ def run(self):
 
         if self.bids_validate:
             try:
-                self.logger.info(f"Validate if {self.output_dir} is BIDS valid.")
-                self.logger.info("Use bids-validator version: ")
-                run_shell_command(['bids-validator', '-v'])
-                run_shell_command(['bids-validator', self.bids_dir])
+                self.logger.info("BIDS VALIDATION")
+                bids_version = run_shell_command(['bids-validator', '-v'], False)
+                self.logger.info(f"Use bids-validator version: {bids_version.decode()[:-1]}")
+                bids_report = run_shell_command(['bids-validator', self.bids_dir])
+                self.logger.info("Report from bids-validator")
+                self.logger.info(bids_report.decode())
             except Exception:
                 self.logger.error("The bids-validator does not seem to work properly. "
                                   "The bids-validator may not be installed on your "
@@ -186,8 +188,18 @@ def move(self, acq, idList, post_op):
                             else:
                                 cmd = cmd.replace('dst_file', str(dstFile))
 
-                            run_shell_command(cmd.split())
-                            continue
+                            try:
+                                std_out = run_shell_command(cmd.split())
+                                self.logger.debug(f"Log from: {cmd}")
+                                self.logger.debug(std_out.decode())
+                                self.logger.info("")
+                                continue
+                            except Exception:
+                                self.logger.error(
+                                  f"The command post_op: \"{cmd}\" "
+                                  "does not seem to work properly. "
+                                  "Check if it is installed on your "
+                                  "computer.\n")
 
             if ".json" in ext:
                 data = acq.dstSidecarData(idList)

dcm2bids/dcm2niix_gen.py

@@ -142,8 +142,16 @@ def execute(self):
                     shutil.rmtree(self.rm_tmp_dir)
                     self.logger.info("Temporary dicom directory removed.")
 
-                self.logger.debug(f"\n{output}")
-                self.logger.info("Check log file for dcm2niix output\n")
+                if "Warning" in output or "Error" in output:
+                    self.logger.info("Log from dcm2niix execution")
+                    if "Warning" in output:
+                        self.logger.warning(f"{output}")
+                    else:
+                        self.logger.error(f"{output}")
+                else:
+                    self.logger.debug(f"\n{output}")
+                    self.logger.info("Check log file for dcm2niix output\n")
+
 
         else:
             for dicomDir in self.dicom_dirs:

dcm2bids/sidecar.py

@@ -427,6 +427,7 @@ def searchDcmTagEntity(self, sidecar, desc):
         """
         descWithTask = desc.copy()
         concatenated_matches = {}
+        keys_custom_entities = []
         entities = []
         if "custom_entities" in desc.keys() or self.auto_extract_entities:
             if 'custom_entities' in desc.keys():
@@ -435,10 +436,12 @@ def searchDcmTagEntity(self, sidecar, desc):
             else:
                 descWithTask["custom_entities"] = []
 
+            keys_custom_entities = [curr_entity.split('-')[0] for curr_entity in descWithTask["custom_entities"]]
+
             if self.auto_extract_entities:
                 self.extractors = combine_dict_extractors(self.extractors, DEFAULT.auto_extractors)
 
-
+            # Loop to check if we find self.extractor
             for dcmTag in self.extractors:
                 if dcmTag in sidecar.data.keys():
                     dcmInfo = sidecar.data.get(dcmTag)
@@ -447,12 +450,12 @@ def searchDcmTagEntity(self, sidecar, desc):
                         if not isinstance(dcmInfo, list):
                             if compile_regex.search(str(dcmInfo)) is not None:
                                 concatenated_matches.update(
-                                  compile_regex.search(str(dcmInfo)).groupdict())
+                                    compile_regex.search(str(dcmInfo)).groupdict())
                         else:
                             for curr_dcmInfo in dcmInfo:
                                 if compile_regex.search(curr_dcmInfo) is not None:
                                     concatenated_matches.update(
-                                      compile_regex.search(curr_dcmInfo).groupdict())
+                                        compile_regex.search(curr_dcmInfo).groupdict())
                                     break
 
             # Keep entities asked in custom_entities
@@ -463,22 +466,24 @@ def searchDcmTagEntity(self, sidecar, desc):
                 # custom_entities not a key for extractor or auto_extract_entities
                 complete_entities = [ent for ent in descWithTask["custom_entities"] if '-' in ent]
                 entities = entities.union(set(complete_entities))
-
             if self.auto_extract_entities:
                 auto_acq = '_'.join([descWithTask['datatype'], descWithTask["suffix"]])
                 if auto_acq in DEFAULT.auto_entities:
                     # Check if these auto entities have been found before merging
                     auto_entities = set(concatenated_matches.keys()).intersection(set(DEFAULT.auto_entities[auto_acq]))
+
                     left_auto_entities = auto_entities.symmetric_difference(set(DEFAULT.auto_entities[auto_acq]))
+                    left_auto_entities = left_auto_entities.difference(keys_custom_entities)
+
                     if left_auto_entities:
-                        self.logger.warning(f"{left_auto_entities} have not been found for datatype '{descWithTask['datatype']}' "
+                        self.logger.warning(f"Entities {left_auto_entities} have not been found "
+                                            f"for datatype '{descWithTask['datatype']}' "
                                             f"and suffix '{descWithTask['suffix']}'.")
-                    
+
                     entities = list(entities) + list(auto_entities)
                     entities = list(set(entities))
                     descWithTask["custom_entities"] = entities
 
-
             for curr_entity in entities:
                 if curr_entity in concatenated_matches.keys():
                     if curr_entity == 'dir':

dcm2bids/utils/logger.py

@@ -19,7 +19,7 @@ def setup_logging(log_level, log_file=None):
 
     sh = logging.StreamHandler(sys.stdout)
     sh.setLevel(log_level)
-    sh_fmt = logging.Formatter(fmt="%(levelname)-8s| %(message)s")
+    sh_fmt = CustomFormatter(fmt="%(levelname)-8s| %(message)s")
     sh.setFormatter(sh_fmt)
 
     # default formatting is kept for the log file"
@@ -30,3 +30,30 @@ def setup_logging(log_level, log_file=None):
         datefmt="%Y-%m-%d %H:%M:%S",
         handlers=[fh, sh]
     )
+
+
+class CustomFormatter(logging.Formatter):
+    """Logging colored formatter, adapted from https://stackoverflow.com/a/56944256/3638629"""
+
+    grey = '\x1b[38;21m'
+    blue = '\x1b[38;5;39m'
+    yellow = '\x1b[38;5;226m'
+    red = '\x1b[38;5;196m'
+    bold_red = '\x1b[31;1m'
+    reset = '\x1b[0m'
+
+    def __init__(self, fmt):
+        super().__init__()
+        self.fmt = fmt
+        self.FORMATS = {
+            logging.DEBUG: self.grey + self.fmt + self.reset,
+            logging.INFO: self.blue + self.fmt + self.reset,
+            logging.WARNING: self.yellow + self.fmt + self.reset,
+            logging.ERROR: self.red + self.fmt + self.reset,
+            logging.CRITICAL: self.bold_red + self.fmt + self.reset
+        }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)

dcm2bids/utils/utils.py

@@ -5,7 +5,7 @@
 import logging
 import os
 from pathlib import Path
-from subprocess import check_output
+from subprocess import Popen, PIPE
 
 
 class DEFAULT(object):
@@ -53,13 +53,28 @@ class DEFAULT(object):
 
     extractors = {}
 
-    auto_entities = {"anat_MEGRE": ["echo"],
+    auto_entities = {"anat_IRT1": ["inv"],
+                     "anat_MEGRE": ["echo"],
                      "anat_MESE": ["echo"],
+                     "anat_MP2RAGE": ["inv"],
+                     "anat_MPM": ["flip", "mt"],
+                     "anat_MTS": ["flip", "mt"],
+                     "anat_MTR": ["mt"],
+                     "anat_VFA": ["flip"],
                      "func_cbv": ["task"],
                      "func_bold": ["task"],
                      "func_sbref": ["task"],
-                     "fmap_epi": ["dir"]}
-
+                     "func_event": ["task"],
+                     "func_stim": ["task"],
+                     "func_phase": ["task"],
+                     "fmap_epi": ["dir"],
+                     "fmap_m0scan": ["dir"],
+                     "fmap_TB1DAM": ["flip"],
+                     "fmap_TB1EPI": ["echo", "flip"],
+                     "fmap_TB1SRGE": ["echo", "inv"],
+                     "perf_physio": ["task"],
+                     "perf_stim": ["task"]}
+
     compKeys = ["AcquisitionTime", "SeriesNumber", "SidecarFilename"]
     search_methodChoices = ["fnmatch", "re"]
     search_method = "fnmatch"
@@ -70,10 +85,12 @@ class DEFAULT(object):
     case_sensitive = True
 
     # Entity table:
-    # https://bids-specification.readthedocs.io/en/v1.7.0/99-appendices/04-entity-table.html
-    entityTableKeys = ["sub", "ses", "task", "acq", "ce", "rec", "dir",
-                       "run", "mod", "echo", "flip", "inv", "mt", "part",
-                       "recording"]
+    # https://bids-specification.readthedocs.io/en/v1.9.0/99-appendices/04-entity-table.html
+    entityTableKeys = ["sub", "ses", "sample", "task", "tracksys",
+                       "acq", "ce", "trc", "stain", "rec", "dir",
+                       "run", "mod", "echo", "flip", "inv", "mt",
+                       "part", "proc", "hemi", "space", "split", "recording",
+                       "chunk", "seg", "res", "den", "label", "desc"]
 
     keyWithPathsidecar_changes = ['IntendedFor', 'Sources']
 
@@ -82,7 +99,7 @@ class DEFAULT(object):
     helper_dir = "helper"
 
     # BIDS version
-    bids_version = "v1.8.0"
+    bids_version = "v1.9.0"
 
 
 def write_participants(filename, participants):
@@ -128,7 +145,11 @@ def run_shell_command(commandLine, log=True):
     if log:
         logger = logging.getLogger(__name__)
         logger.info("Running: %s", " ".join(str(item) for item in commandLine))
-    return check_output(commandLine)
+
+    pipes = Popen(commandLine, stdout=PIPE, stderr=PIPE)
+    std_out, std_err = pipes.communicate()
+
+    return std_out
 
 
 def convert_dir(dir):
@@ -143,15 +164,15 @@ def convert_dir(dir):
 
 
 def combine_dict_extractors(d1, d2):
-    """ combine dict 
+    """ combine dict
     Args:
         d1 (dic): dictionary
         d2 (dic): dictionary
-    
+
     Returns:
         dict: dictionary with combined information
               if d1 d2 use the same keys, return dict will return a list of items.
-    """    
+    """
     return {
             k: [d[k][0] for d in (d1, d2) if k in d]
             for k in set(d1.keys()) | set(d2.keys())

docs/tutorial/parallel.md

@@ -58,7 +58,7 @@ First thing first, let's make sure our software are usable.
     ```sh
     (dcm2bids) sam:~$ dcm2bids -v
     dcm2bids version:       3.1.0
-    Based on BIDS version:  v1.8.0
+    Based on BIDS version:  v1.9.0
     (dcm2bids) sam:~$ parallel --version
     GNU parallel 20230722
     Copyright (C) 2007-2023 Ole Tange, http://ole.tange.dk and Free Software

environment.yml

@@ -0,0 +1,7 @@
+name: fsl
+channels:
+  - https://fsl.fmrib.ox.ac.uk/fsldownloads/fslconda/public/
+  - conda-forge
+dependencies:
+  - fsl-bet2
+  - fsl-flirt