Skip to content

Commit

Permalink
- Use a true list (instead of a semi-colon separated string) for bids…
Browse files Browse the repository at this point in the history
…ignore for better consistency

- Add descriptions and other tweaks to the bidsmap schema
  • Loading branch information
marcelzwiers committed Oct 25, 2023
1 parent 36a2943 commit af9dd5c
Show file tree
Hide file tree
Showing 11 changed files with 124 additions and 134 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
- name: Install dependencies
run: |
pip install --upgrade pip
pip install .[all] jsonschema
pip install .[all] jsonschema pytest
- name: Install dcm2niix posix
if: runner.os != 'Windows'
Expand Down
2 changes: 1 addition & 1 deletion bidscoin/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
configfile.write_text(f"[bidscoin]\n"
f"bidsmap_template = '{templatefolder}/bidsmap_dccn.yaml' # The default template bidsmap (change to use a different default)\n"
f"trackusage = 'yes' # Upload anonymous usage data if 'yes' (maximally 1 upload every {tracking['sleep']} hour) (see `bidscoin --tracking show`)\n")
for template in (bidscoinfolder/'heuristics').glob('*.yaml'):
for template in list((bidscoinfolder/'heuristics').glob('*.yaml')) + [bidscoinfolder/'heuristics'/'schema.json']:
if not (templatefolder/template.name).is_file():
print(f"-> {templatefolder/template.name}")
shutil.copyfile(template, templatefolder/template.name)
Expand Down
16 changes: 8 additions & 8 deletions bidscoin/bids.py
Original file line number Diff line number Diff line change
Expand Up @@ -927,7 +927,7 @@ def load_bidsmap(yamlfile: Path, folder: Path=Path(), plugins:Union[tuple,list]=
subprefix = bidsmap['Options']['bidscoin'].get('subprefix','')
sesprefix = bidsmap['Options']['bidscoin'].get('sesprefix','')
for dataformat in bidsmap:
if dataformat == 'Options': continue
if dataformat in ('$schema', 'Options'): continue
for datatype in bidsmap[dataformat]:
if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject', 'session' and empty datatypes
for index, run in enumerate(bidsmap[dataformat][datatype]):
Expand Down Expand Up @@ -986,7 +986,7 @@ def save_bidsmap(filename: Path, bidsmap: dict) -> None:
# Remove the added DataSource object
bidsmap = copy.deepcopy(bidsmap)
for dataformat in bidsmap:
if dataformat == 'Options': continue
if dataformat in ('$schema', 'Options'): continue
if not bidsmap[dataformat]: continue
for datatype in bidsmap[dataformat]:
if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject' and 'session'
Expand Down Expand Up @@ -1028,7 +1028,7 @@ def validate_bidsmap(bidsmap: dict, level: int=1) -> bool:
# Test all the runs in the bidsmap
LOGGER.info(f"bids-validator {bids_validator.__version__} test results (* = in .bidsignore):")
for dataformat in bidsmap:
if dataformat == 'Options': continue
if dataformat in ('$schema', 'Options'): continue
if not bidsmap[dataformat]: continue
for datatype in bidsmap[dataformat]:
if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject' and 'session'
Expand Down Expand Up @@ -1072,7 +1072,7 @@ def check_bidsmap(bidsmap: dict, checks: Tuple[bool, bool, bool]=(True, True, Tr
# Check all the runs in the bidsmap
LOGGER.info('Checking the bidsmap run-items:')
for dataformat in bidsmap:
if dataformat == 'Options': continue # TODO: Check Options
if dataformat in ('$schema', 'Options'): continue # TODO: Check Options
if not bidsmap[dataformat]: continue
for datatype in bidsmap[dataformat]:
if not isinstance(bidsmap[dataformat][datatype], list): continue # E.g. 'subject' and 'session'
Expand Down Expand Up @@ -1115,7 +1115,7 @@ def check_template(bidsmap: dict) -> bool:
# Check all the datatypes in the bidsmap
LOGGER.info('Checking the bidsmap datatypes:')
for dataformat in bidsmap:
if dataformat == 'Options': continue
if dataformat in ('$schema', 'Options'): continue
for datatype in bidsmap[dataformat]:
if not isinstance(bidsmap[dataformat][datatype], list): continue # Skip datatype = 'subject'/'session'
if not (datatype in bidsdatatypesdef or datatype in ignoretypes or check_ignore(datatype, bidsignore)):
Expand All @@ -1132,7 +1132,7 @@ def check_template(bidsmap: dict) -> bool:
LOGGER.warning(f"Invalid regex pattern in the {key} value '{val}' in: bidsmap[{dataformat}][{datatype}] -> {run['provenance']}\nThis may cause run-matching errors unless '{val}' is a literal attribute value")
for typegroup in datatyperules.get(datatype, {}):
for suffix in datatyperules[datatype][typegroup]['suffixes']:
if not (suffix in datatypesuffixes or suffix in bidsignore or
if not (suffix in datatypesuffixes or suffix in str(bidsignore) or
'[DEPRECATED]' in suffixes[suffix]['description'] or
'**Change:** Removed from' in suffixes[suffix]['description'] or
'**Change:** Replaced by' in suffixes[suffix]['description']):
Expand Down Expand Up @@ -1250,7 +1250,7 @@ def check_ignore(entry: str, bidsignore: Union[str,list], datatype: str= 'dir')
:return: True if the entry should be ignored, else False
"""

# Parse bidsignore to be a list
# Parse bidsignore to be a list (legacy bidsmaps)
if isinstance(bidsignore, str):
bidsignore = bidsignore.split(';')

Expand Down Expand Up @@ -1435,7 +1435,7 @@ def find_run(bidsmap: dict, provenance: str, dataformat: str='', datatype: str='
if dataformat:
dataformats = (dataformat,)
else:
dataformats = [item for item in bidsmap if item not in ('Options','PlugIns') and bidsmap[item]]
dataformats = [item for item in bidsmap if item not in ('$schema','Options') and bidsmap[item]]
for dataformat in dataformats:
if datatype:
datatypes = (datatype,)
Expand Down
4 changes: 2 additions & 2 deletions bidscoin/bidscoiner.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def bidscoiner(rawfolder: str, bidsfolder: str, subjects: list=(), force: bool=F

# Get the bidsmap heuristics from the bidsmap YAML-file
bidsmap, _ = bids.load_bidsmap(bidsmapfile, bidsfolder/'code'/'bidscoin')
dataformats = [dataformat for dataformat in bidsmap if dataformat and dataformat != 'Options']
dataformats = [dataformat for dataformat in bidsmap if dataformat and dataformat not in ('$schema','Options')]
if not bidsmap:
LOGGER.error(f"No bidsmap file found in {bidsfolder}. Please run the bidsmapper first and/or use the correct bidsfolder")
return
Expand All @@ -105,7 +105,7 @@ def bidscoiner(rawfolder: str, bidsfolder: str, subjects: list=(), force: bool=F
return

# Append options to the .bidsignore file
bidsignore_items = [item.strip() for item in bidsmap['Options']['bidscoin']['bidsignore'].split(';')]
bidsignore_items = bidsmap['Options']['bidscoin']['bidsignore']
bidsignore_file = bidsfolder/'.bidsignore'
if bidsignore_items:
LOGGER.verbose(f"Writing {bidsignore_items} entries to {bidsignore_file}")
Expand Down
10 changes: 5 additions & 5 deletions bidscoin/bidseditor.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@

TOOLTIP_BIDSCOIN = """BIDScoin
version: Used to check for version conflicts
bidsignore: Semicolon-separated list of data types that are added to the .bidsignore file,
bidsignore: List of data types that are added to the .bidsignore file,
e.g. extra_data/;myfile.txt;yourfile.csv
subprefix: The subject prefix used in the source data folders (e.g. "Pt" is the subprefix if subject folders are named "Pt018", "Pt019", ...)
sesprefix: The session prefix used in the source data folders (e.g. "M_" is the subprefix if session folders are named "M_pre", "M_post", ...)
Expand Down Expand Up @@ -103,10 +103,10 @@ def __init__(self, bidsfolder: Path, input_bidsmap: dict, template_bidsmap: dict
self.output_bidsmap = copy.deepcopy(input_bidsmap) # The edited bidsmap
self.template_bidsmap = template_bidsmap # The bidsmap from which new datatype run-items are taken
self.datasaved = datasaved # True if data has been saved on disk
self.dataformats = [dataformat for dataformat in input_bidsmap if dataformat and dataformat not in ('Options', 'PlugIns') and bids.dir_bidsmap(input_bidsmap, dataformat)]
self.dataformats = [dataformat for dataformat in input_bidsmap if dataformat and dataformat not in ('$schema','Options') and bids.dir_bidsmap(input_bidsmap, dataformat)]
self.unknowndatatypes = input_bidsmap['Options']['bidscoin'].get('unknowntypes',[])
self.ignoredatatypes = input_bidsmap['Options']['bidscoin'].get('ignoretypes',[])
self.bidsignore = input_bidsmap['Options']['bidscoin'].get('bidsignore','')
self.bidsignore = input_bidsmap['Options']['bidscoin'].get('bidsignore',[])

# Set up the tabs, add the tables and put the bidsmap data in them
tabwidget = self.tabwidget = QtWidgets.QTabWidget()
Expand Down Expand Up @@ -641,7 +641,7 @@ def options2bidsmap(self, rowindex: int, colindex: int):
self.output_bidsmap['Options']['bidscoin'] = newoptions
self.unknowndatatypes = newoptions.get('unknowntypes', [])
self.ignoredatatypes = newoptions.get('ignoretypes', [])
self.bidsignore = newoptions.get('bidsignore', '')
self.bidsignore = newoptions.get('bidsignore', [])
for dataformat in self.dataformats:
self.update_subses_samples(self.output_bidsmap, dataformat)
else:
Expand Down Expand Up @@ -877,7 +877,7 @@ def __init__(self, run, bidsmap: dict, template_bidsmap: dict):
self.unknowndatatypes = [datatype for datatype in bidsmap['Options']['bidscoin'].get('unknowntypes',[]) if datatype in template_bidsmap[self.dataformat]]
self.ignoredatatypes = [datatype for datatype in bidsmap['Options']['bidscoin'].get('ignoretypes', []) if datatype in template_bidsmap[self.dataformat]]
self.bidsdatatypes = [datatype for datatype in template_bidsmap[self.dataformat] if datatype not in self.unknowndatatypes + self.ignoredatatypes + ['subject', 'session']]
self.bidsignore = bidsmap['Options']['bidscoin'].get('bidsignore','')
self.bidsignore = bidsmap['Options']['bidscoin'].get('bidsignore',[])
self.source_bidsmap = bidsmap # The bidsmap at the start of the edit = output_bidsmap in the MainWindow
self.target_bidsmap = copy.deepcopy(bidsmap) # The edited bidsmap -> will be returned as output_bidsmap in the MainWindow
self.template_bidsmap = template_bidsmap # The bidsmap from which new datatype run-items are taken
Expand Down
4 changes: 2 additions & 2 deletions bidscoin/bidsmapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def bidsmapper(rawfolder: str, bidsfolder: str, bidsmapfile: str, templatefile:
else:
unzip = bidsmap_new['Options']['bidscoin'].get('unzip','')
for dataformat in bidsmap_new:
if dataformat == 'Options': continue
if dataformat in ('$schema', 'Options'): continue
for datatype in bidsmap_new[dataformat]:
if datatype not in ('subject', 'session'):
bidsmap_new[dataformat][datatype] = []
Expand Down Expand Up @@ -203,7 +203,7 @@ def setprefix(bidsmap: dict, subprefix: str, sesprefix: str, rawfolder: Path, up
# Update the bidsmap dataformat sections
reprefix = lambda prefix: '' if prefix=='*' else re.escape(prefix).replace(r'\-','-')
for dataformat in bidsmap:
if not bidsmap[dataformat] or dataformat=='Options': continue
if not bidsmap[dataformat] or dataformat in ('$schema','Options'): continue

# Update the run-DataSources
for datatype in bidsmap[dataformat]:
Expand Down
28 changes: 7 additions & 21 deletions bidscoin/heuristics/bidsmap_dccn.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@
# For more information, see: https://bidscoin.readthedocs.io
# --------------------------------------------------------------------------------

$schema: schema.json

Options:
# --------------------------------------------------------------------------------
# General options and plugins
# General BIDScoin and plugin options
# --------------------------------------------------------------------------------
bidscoin:
version: 4.2.0 # BIDScoin version (should correspond with the version in pyproject.toml)
bidsignore: mrs/;extra_data/;sub-*_ct.* # Semicolon-separated list of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. extra_data/;pet/;myfile.txt;yourfile.csv
bidsignore: [mrs/, extra_data/, sub-*_ct.*] # list of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
subprefix: sub- # The subject prefix of the source data
sesprefix: ses- # The session prefix of the source data
unknowntypes: [mrs, extra_data] # A list of datatypes that are converted to BIDS-like datatype folders
Expand Down Expand Up @@ -62,7 +63,7 @@ DICOM:
filename: # File name, e.g. ".*fmap.*" or ".*(fmap|field.?map|B0.?map).*"
filesize: # File size, e.g. "2[4-6]\d MB" for matching files between 240-269 MB
nrfiles: # Number of files in the folder
attributes: &anat_dicomattr # An empty / non-matching reference dictionary that can be derefenced in other run-items of this data type
attributes: &anat_dicomattr # An empty / non-matching reference dictionary that can be dereferenced in other run-items of this data type
Modality:
ProtocolName:
SeriesDescription:
Expand All @@ -87,7 +88,7 @@ DICOM:
run: <<>> # This will be updated dynamically during bidscoiner runtime (as it depends on the already existing files)
part: ['', mag, phase, real, imag, 0] # # This BIDS value list will be shown as a dropdown menu in the bidseditor with the first (empty) item selected (as indicated by the last item, i.e. 0)
suffix: T1w
meta: # This is an optional entry for meta-data that will be appended to the json sidecar files produced by dcm2niix
meta: {} # This is an optional entry for meta-data that will be appended to the json sidecar files produced by the plugin
- provenance:
attributes:
<<: *anat_dicomattr
Expand Down Expand Up @@ -398,7 +399,7 @@ DICOM:
part: ['', mag, phase, real, imag, 0]
suffix: MTR
- provenance: # ----------- Experimental CT support (see BEP024) -------------
attributes: &ct_dicomattr # An empty / non-matching reference dictionary that can be derefenced in other run-items of this data type
attributes: &ct_dicomattr # An empty / non-matching reference dictionary that can be dereferenced in other run-items of this data type
Modality: CT
ProtocolName:
SeriesDescription:
Expand All @@ -413,7 +414,6 @@ DICOM:
rec:
run: <<>> # This will be updated dynamically during bidscoiner runtime (as it depends on the already existing files)
suffix: ct
meta: # This is an optional entry for meta-data that will be appended to the json sidecar files produced by dcm2niix

func: # ----------------------- All functional runs --------------------
- provenance: # The first item with empty attributes will not match anything but is used by default when changing datatype in the bidseditor GUI -> suffix = bold
Expand Down Expand Up @@ -1825,19 +1825,16 @@ SPAR:
type:
run: <<>> # This will be updated during bidscoiner runtime (as it depends on the already existing files)
suffix: press
meta: # This is an optional entry for meta-data dictionary that will be appended to the json sidecar files produced by dcm2niix

extra_data: # ----------------------- All extra data -------------------------
- provenance:
attributes: *mrs_sparattr
bids: *mrs_sparent
meta:

exclude: # ----------------------- Data that will be left out -------------
- provenance:
attributes: *mrs_sparattr
bids: *mrs_sparent
meta:


Twix:
Expand Down Expand Up @@ -1875,27 +1872,23 @@ Twix:
type:
run: <<>> # This will be updated during bidscoiner runtime (as it depends on the already existing files)
suffix: mrs
meta: # This is an optional entry for meta-data dictionary that will be appended to the json sidecar files produced by dcm2niix
- provenance:
attributes:
<<: *mrs_twixattr
SequenceName: .*
bids:
<<: *mrs_twixent
suffix: press
meta:

extra_data: # ----------------------- All extra data -------------------------
- provenance:
attributes: *mrs_twixattr
bids: *mrs_twixent
meta:

exclude: # ----------------------- Data that will be left out -------------
- provenance:
attributes: *mrs_twixattr
bids: *mrs_twixent
meta:


Pfile:
Expand Down Expand Up @@ -1942,27 +1935,23 @@ Pfile:
type:
run: <<>> # This will be updated during bidscoiner runtime (as it depends on the already existing files)
suffix: mrs
meta: # This is an optional entry for meta-data dictionary that will be appended to the json sidecar files produced by dcm2niix
- provenance: # The fullpath name of the source file from which the attributes are read. Serves also as a look-up key to find a run in the bidsmap
attributes:
<<: *mrs_p7attr # The matching (regex) criteria for spec2nii2bids go in here
rhs_se_desc: .*PRESS.*
bids:
<<: *mrs_p7fileent # See: https://docs.google.com/document/d/1pWCb02YNv5W-UZZja24fZrdXLm4X7knXMiZI7E2z7mY/
suffix: press
meta: # This is an optional entry for meta-data dictionary that will be appended to the json sidecar files produced by dcm2niix

extra_data: # ----------------------- All extra data -------------------------
- provenance:
attributes: *mrs_p7attr
bids: *mrs_p7fileent
meta:

exclude: # ----------------------- Data that will be left out -------------
- provenance:
attributes: *mrs_p7attr
bids: *mrs_p7fileent
meta:


#Physio:
Expand Down Expand Up @@ -2004,7 +1993,6 @@ Pfile:
# - provenance:
# attributes: *attributes_physio
# bids: *bids_physio
# meta:
#
# exclude: # ----------------------- Data that will be left out -------------
# - provenance:
Expand All @@ -2016,7 +2004,6 @@ Pfile:
# units:
# trigger_idx:
# bids: *bids_physio
# meta:


Nibabel:
Expand All @@ -2029,7 +2016,7 @@ Nibabel:
anat: # ----------------------- All anatomical runs --------------------
- provenance: # The fullpath name of the DICOM file from which the attributes are read. Serves also as a look-up key to find a run in the bidsmap
properties: *fileprop
attributes: &nibattr # An empty / non-matching reference dictionary that can be derefenced in other run-items of this data type
attributes: &nibattr # An empty / non-matching reference dictionary that can be dereferenced in other run-items of this data type
dim:
dim_info:
pixdim:
Expand All @@ -2056,7 +2043,6 @@ Nibabel:
run: <<>> # This will be updated during bidscoiner runtime (as it depends on the already existing files)
part: ['', mag, phase, real, imag, 0]
suffix: T1w
meta: # This is an optional entry for meta-data that will be appended to the json sidecar files produced by dcm2niix
- provenance:
properties:
<<: *fileprop
Expand Down
Loading

0 comments on commit af9dd5c

Please sign in to comment.