add funloc dataset (#1045)

Co-authored-by: Eric Larson <[email protected]>

.circleci/config.yml

@@ -297,6 +297,25 @@ jobs:
           paths:
             - ~/mne_data/eeg_matchingpennies
 
+  cache_MNE-funloc-data:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      - restore_cache:
+          keys:
+            - data-cache-MNE-funloc-data-5
+      - bash_env
+      - run:
+          name: Get MNE-funloc-data
+          command: |
+            $DOWNLOAD_DATA MNE-funloc-data
+      - codecov/upload
+      - save_cache:
+          key: data-cache-MNE-funloc-data-5
+          paths:
+            - ~/mne_data/MNE-funloc-data
+
   cache_MNE-phantom-KIT-data:
     <<: *imageconfig
     steps:
@@ -784,6 +803,33 @@ jobs:
           paths:
             - mne_data/derivatives/mne-bids-pipeline/eeg_matchingpennies/*/*/*.html
 
+  test_MNE-funloc-data:
+    <<: *imageconfig
+    resource_class: large
+    steps:
+      - attach_workspace:
+          at: ~/
+      - bash_env
+      - restore_cache:
+          keys:
+            - data-cache-MNE-funloc-data-5
+      - run:
+          name: test MNE-funloc-data
+          command: $RUN_TESTS MNE-funloc-data
+      - codecov/upload
+      - store_test_results:
+          path: ./test-results
+      - store_artifacts:
+          path: ./test-results
+          destination: test-results
+      - store_artifacts:
+          path: /home/circleci/reports/MNE-funloc-data
+          destination: reports/MNE-funloc-data
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - mne_data/derivatives/mne-bids-pipeline/MNE-funloc-data/*/*/*.html
+
   test_MNE-phantom-KIT-data:
     <<: *imageconfig
     steps:
@@ -1244,6 +1290,15 @@ workflows:
             - cache_eeg_matchingpennies
           <<: *filter_tags
 
+      - cache_MNE-funloc-data:
+          requires:
+            - setup_env
+          <<: *filter_tags
+      - test_MNE-funloc-data:
+          requires:
+            - cache_MNE-funloc-data
+          <<: *filter_tags
+
       - cache_MNE-phantom-KIT-data:
           requires:
             - setup_env
@@ -1304,6 +1359,7 @@ workflows:
             - test_ds003392
             - test_ds004229
             - test_eeg_matchingpennies
+            - test_MNE-funloc-data
             - test_MNE-phantom-KIT-data
             - test_ERP_CORE_N400
             - test_ERP_CORE_ERN

docs/mkdocs.yml

@@ -121,6 +121,7 @@ nav:
         - examples/ds000248_no_mri.md
         - examples/ds003104.md
         - examples/eeg_matchingpennies.md
+        - examples/MNE-funloc-data.md
         - examples/MNE-phantom-KIT-data.md
         - examples/ds001810.md
         - examples/ds000117.md

docs/source/dev.md.inc

@@ -38,3 +38,4 @@
 - Enable strict type checking via mypy (#995, #1013, #1016 by @larsoner)
 - Improve logging messages in maxwell filtering steps. (#893 by @drammock)
 - Validate extra config params passed in during testing. (#1044 by @drammock)
+- New testing/example dataset "funloc" added. (#1045 by @drammock)

mne_bids_pipeline/_download.py

@@ -2,6 +2,7 @@
 
 import argparse
 from pathlib import Path
+from warnings import filterwarnings
 
 import mne
 
@@ -10,6 +11,19 @@
 DEFAULT_DATA_DIR = Path("~/mne_data").expanduser()
 
 
+# TODO this can be removed when https://github.com/fatiando/pooch/pull/458 is merged and
+# we pin to a version of pooch that includes that commit
+filterwarnings(
+    action="ignore",
+    message=(
+        "Python 3.14 will, by default, filter extracted tar archives and reject files "
+        "or modify their metadata. Use the filter argument to control this behavior."
+    ),
+    category=DeprecationWarning,
+    module="tarfile",
+)
+
+
 def _download_via_openneuro(*, ds_name: str, ds_path: Path) -> None:
     import openneuro
 
@@ -26,7 +40,7 @@ def _download_via_openneuro(*, ds_name: str, ds_path: Path) -> None:
 
 
 def _download_from_web(*, ds_name: str, ds_path: Path) -> None:
-    """Retrieve Zip archives from a web URL."""
+    """Retrieve `.zip` or `.tar.gz` archives from a web URL."""
     import pooch
 
     options = DATASET_OPTIONS[ds_name]
@@ -43,16 +57,18 @@ def _download_from_web(*, ds_name: str, ds_path: Path) -> None:
 
     ds_path.mkdir(parents=True, exist_ok=True)
     path = ds_path.parent.resolve(strict=True)
-    fname = f"{ds_name}.zip"
+    ext = "tar.gz" if options.get("processor") == "untar" else "zip"
+    processor = pooch.Untar if options.get("processor") == "untar" else pooch.Unzip
+    fname = f"{ds_name}.{ext}"
     pooch.retrieve(
         url=url,
         path=path,
         fname=fname,
-        processor=pooch.Unzip(extract_dir="."),  # relative to path
+        processor=processor(extract_dir="."),  # relative to path
         progressbar=True,
         known_hash=known_hash,
     )
-    (path / f"{ds_name}.zip").unlink()
+    (path / f"{ds_name}.{ext}").unlink()
 
 
 def _download_via_mne(*, ds_name: str, ds_path: Path) -> None:

mne_bids_pipeline/tests/configs/README.md

@@ -0,0 +1,9 @@
+# Config files for test datasets
+
+!!! warning
+
+    The documentation build scripts (`docs/source/examples/gen_examples.py`)
+    assume a config file name of `config_{name_of_dataset}.py` or
+    `config_{name-of-dataset}.py`. If you want a dataset to be shown in the docs as an
+    example dataset, you **must** name the config file accordingly (and also, add  the
+    dataset to `docs/mkdocs.yml` in the list of `Examples`).

mne_bids_pipeline/tests/configs/config_MNE_funloc_data.py

@@ -0,0 +1,39 @@
+"""Funloc data."""
+
+from pathlib import Path
+
+data_root = Path("~/mne_data").expanduser().resolve()
+bids_root = data_root / "MNE-funloc-data"
+deriv_root = data_root / "derivatives" / "mne-bids-pipeline" / "MNE-funloc-data"
+subjects_dir = bids_root / "derivatives" / "freesurfer" / "subjects"
+task = "funloc"
+ch_types = ["meg", "eeg"]
+data_type = "meg"
+
+# filter
+l_freq = None
+h_freq = 50.0
+# maxfilter
+use_maxwell_filter: bool = True
+crop_runs = (40, 190)
+mf_st_duration = 60.0
+# SSP
+n_proj_eog = dict(n_mag=1, n_grad=1, n_eeg=2)
+n_proj_ecg = dict(n_mag=1, n_grad=1, n_eeg=0)
+
+# Epochs
+epochs_tmin = -0.2
+epochs_tmax = 0.5
+epochs_decim = 5  # 1000 -> 200 Hz
+baseline = (None, 0)
+conditions = [
+    "auditory/standard",
+    # "auditory/deviant",
+    "visual/standard",
+    # "visual/deviant",
+]
+decode = False
+decoding_time_generalization = False
+
+# contrasts
+# contrasts = [("auditory", "visual")]

mne_bids_pipeline/tests/datasets.py

@@ -15,6 +15,7 @@ class DATASET_OPTIONS_T(TypedDict, total=False):
     include: list[str]  # []
     exclude: list[str]  # []
     hash: str  # ""
+    processor: str  # ""
 
 
 DATASET_OPTIONS: dict[str, DATASET_OPTIONS_T] = {
@@ -118,4 +119,9 @@ class DATASET_OPTIONS_T(TypedDict, total=False):
     "MNE-phantom-KIT-data": {
         "mne": "phantom_kit",
     },
+    "MNE-funloc-data": {
+        "web": "https://osf.io/upj3h/download?version=1",
+        "hash": "sha256:67dbd38f7207db5c93c540d9c7c92ec2ac09ee1bd1b5d5e5cdd8866c08ec4858",  # noqa: E501
+        "processor": "untar",
+    },
 }

mne_bids_pipeline/tests/test_run.py

@@ -135,6 +135,10 @@ class _TestOptionsT(TypedDict, total=False):
     "MNE-phantom-KIT-data": {
         "config": "config_MNE_phantom_KIT_data.py",
     },
+    "MNE-funloc-data": {
+        "config": "config_MNE_funloc_data.py",
+        "steps": ["init", "preprocessing", "sensor", "source"],
+    },
 }