[ENH] render table from TSV

mkdocs.yml

@@ -98,13 +98,15 @@ extra:
 extra_javascript:
   - js/jquery-3.6.0.min.js
 markdown_extensions:
+  - tables
   - toc:
       anchorlink: true
   - pymdownx.superfences
   - admonition
   - pymdownx.details
 plugins:
   - search
+  - table-reader
   - branchcustomization:
       update_config:
         - branch: /(?!^master$)/

pdf_build_src/process_markdowns.py

@@ -13,9 +13,11 @@
 import re
 import subprocess
 import sys
+from pathlib import Path
 from datetime import datetime
 
 import numpy as np
+import pandas as pd
 
 from remove_admonitions import remove_admonitions
 
@@ -299,6 +301,41 @@ def modify_changelog():
         file.writelines(data)
 
 
+def read_table_from_tsv(input_file: Path, output_file : Path | None = None):
+    """Inject table in markdown document from a tsv file.
+
+    :param input_file: path to input markdown file
+
+    :param output_file: path to output markdown file
+                        defaults to writing to the input file if None is passed.
+    """
+
+    if output_file is None:
+        output_file = input_file
+
+    with open(input_file, "r", encoding="utf8") as f:
+        content = f.readlines()
+
+    with open(output_file, "w", encoding="utf8") as f:
+
+        for line in content:
+
+            if not line.startswith("{{ read_table('"):
+                f.write(line)
+                continue
+
+            table = line.split("(")[1].split(",")[0].replace("'", '').replace('"', '')
+            table_to_read = input_file.parent / table
+            assert table_to_read.exists(), f"This file does not exist\n:{table_to_read}"
+
+            df = pd.read_csv(table_to_read, sep="\t")
+            df_as_md = df.to_markdown(index=False)
+            f.write(df_as_md)
+            f.write("\n")
+
+            continue
+
+
 def correct_table(table, offset=[0.0, 0.0], debug=False):
     """Create the corrected table.
 
@@ -636,6 +673,9 @@ def process_macros(duplicated_src_dir_path):
                 continue
 
             filename = os.path.join(root, name)
+
+            read_table_from_tsv(Path(filename))
+
             with open(filename, "r") as fo:
                 contents = fo.read()
 

pdf_build_src/test_pdf_build.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+
+from remove_admonitions import remove_admonitions
+from process_markdowns import read_table_from_tsv
+
+
+def test_remove_admonitions(tmp_path):
+    input_folder = Path(__file__).parent / "tests" / "data" / "admonitions" / "input"
+    expected_folder = Path(__file__).parent / "tests" / "data" / "admonitions" / "expected"
+
+    remove_admonitions(input_folder, tmp_path)
+
+    generated_files = list(tmp_path.glob("**/*.md"))
+
+    for file in generated_files:
+
+        expected = expected_folder / file.relative_to(tmp_path)
+
+        with open(expected, "r", encoding="utf8") as f:
+            expected_content = f.readlines()
+
+        with open(file, "r", encoding="utf8") as f:
+            generated_content = f.readlines()
+
+        for expected_line, generated_line in zip(expected_content, generated_content):
+            assert generated_line == expected_line
+
+
+def test_read_table_from_tsv(tmp_path):
+    data_path = Path(__file__).parent / "tests" / "data" / "read_from_tsv"
+    input_file = data_path / "input.md"
+    output_file = tmp_path / "output.md"
+    expected_file = data_path / "expected.md"
+
+    read_table_from_tsv(input_file, output_file)
+
+    with open(expected_file, "r", encoding="utf8") as f:
+        expected_content = f.readlines()
+
+    with open(output_file, "r", encoding="utf8") as f:
+        generated_content = f.readlines()
+
+    for expected_line, generated_line in zip(expected_content, generated_content):
+        assert generated_line == expected_line

pdf_build_src/tests/data/read_from_tsv/expected.md

@@ -0,0 +1,10 @@
+# foo
+
+lorem ipsum
+
+|   onset |   duration |   response_time | trial_type   | trial_extra   |
+|--------:|-----------:|----------------:|:-------------|:--------------|
+|     200 |         20 |          15.8   | word         | 中国人        |
+|     240 |          5 |           1.734 | visual       | nan           |
+
+TSV files MUST be in UTF-8 encoding.

pdf_build_src/tests/data/read_from_tsv/input.md

@@ -0,0 +1,7 @@
+# foo
+
+lorem ipsum
+
+{{ read_table('../../../../src/tables/principles_tabular_example.tsv', sep = '\t') }}
+
+TSV files MUST be in UTF-8 encoding.

requirements.txt

@@ -4,5 +4,7 @@ pymdown-extensions>=7.0.0
 mkdocs-branchcustomization-plugin~=0.1.3
 mkdocs-macros-plugin
 mkdocs-redirects
+mkdocs-table-reader-plugin
 numpy
+pandas
 tools/schemacode/[render]

src/appendices/arterial-spin-labeling.md

@@ -26,14 +26,7 @@ and the exact volume_type series should be specified in the `*_aslcontext.tsv`.
 
 Example of `*_aslcontext.tsv`:
 
-```Text
-volume_type
-control
-label
-control
-label
-m0scan
-```
+{{ read_table('../tables/asl_context_case_1.tsv', sep = '\t') }}
 
 ### Case 2: `*_asl.nii[.gz]` consists of volume_types `deltam` (scanner does not export `control` or `label` volumes)
 
@@ -44,11 +37,7 @@ In this case, the `deltam` should be included in the `*_asl.nii[.gz]` and specif
 
 Example of `*_aslcontext.tsv`:
 
-```Text
-volume_type
-deltam
-m0scan
-```
+{{ read_table('../tables/asl_context_case_2.tsv', sep = '\t') }}
 
 ### Case 3: `*_asl.nii[.gz]` consists of volume_type `cbf` (scanner does not export `control`, `label`, or `deltaM` volumes)
 
@@ -58,11 +47,7 @@ the `cbf` should be included in the `*_asl.nii[.gz]` and specified in the `*_asl
 
 Example of `*_aslcontext.tsv`:
 
-```Text
-volume_type
-cbf
-m0scan
-```
+{{ read_table('../tables/asl_context_case_3.tsv', sep = '\t') }}
 
 ## Summary Image of the most common ASL sequences
 

src/appendices/hed.md

@@ -44,11 +44,7 @@ meanings in associated JSON sidecar files (`events.json`).
 (`trial_type`, `response_time`, and `stim_file`) in addition to
 the required `onset` and `duration` columns.
 
-```Text
-onset  duration  trial_type  response_time stim_file
-1.2    0.6       go          1.435         images/red_square.jpg
-5.6    0.6       stop        n/a           images/blue_square.jpg
-```
+{{ read_table('../tables/hed_events.tsv', sep = '\t') }}
 
 The `trial_type` column in the above example contains a limited number of distinct
 values (`go` and `stop`).

src/appendices/qmri.md

@@ -326,7 +326,7 @@ A guide for using macros can be found at
 
 `dataset_description.json`:
 
-```text
+```json
 {
   "Name": "qMRLab Outputs",
   "BIDSVersion": "1.5.0",
@@ -395,7 +395,7 @@ sub-01_T1map.json
 
 sub-01_T1map.json:
 
-```text
+```json
 {
 
 <<Parameter injected by the software/pipeline>>

src/common-principles.md

@@ -466,11 +466,7 @@ TSV files MUST be in UTF-8 encoding.
 
 Example:
 
-```Text
-onset   duration    response_time   trial_type        trial_extra
-200     20.0        15.8            word              中国人
-240     5.0         17.34e-1        visual            n/a
-```
+{{ read_table('tables/principles_tabular_example.tsv', sep = '\t') }}
 
 !!! warning "Attention"
 

src/derivatives/imaging.md

@@ -524,22 +524,12 @@ and a guide for using macros can be found at
 
 An example, custom `dseg.tsv` that defines three labels:
 
-```Text
-index   name            abbreviation    color       mapping
-100     Gray Matter     GM              #ff53bb     1
-101     White Matter    WM              #2f8bbe     2
-102     Brainstem       BS              #36de72     11
-```
+{{ read_table('../tables/derivatives_imaging_dseg_1.tsv', sep = '\t') }}
 
 The following example `dseg.tsv` defines regions that are not part of the
 standard BIDS labels:
 
-```Text
-index   name                abbreviation
-137     pars opercularis    IFGop
-138     pars triangularis   IFGtr
-139     pars orbitalis      IFGor
-```
+{{ read_table('../tables/derivatives_imaging_dseg_2.tsv', sep = '\t') }}
 
 <!-- Link Definitions -->
 

src/longitudinal-and-multi-site-studies.md

@@ -65,11 +65,7 @@ A guide for using macros can be found at
 
 `sub-control01_sessions.tsv` content:
 
-```Text
-session_id	acq_time	systolic_blood_pressure
-ses-predrug	2009-06-15T13:45:30	120
-ses-postdrug	2009-06-16T13:45:30	100
-```
+{{ read_table('tables/longitudinal_sessions.tsv', sep = '\t') }}
 
 See this [example dataset](https://github.com/bids-standard/bids-examples/tree/master/7t_trt)
 that has been formatted

src/modality-agnostic-files.md

@@ -246,12 +246,8 @@ available").
 
 `participants.tsv` example:
 
-```Text
-participant_id age sex handedness group
-sub-01 34 M right read
-sub-02 12 F right write
-sub-03 33 F n/a read
-```
+{{ read_table('tables/agnostic_participants.tsv', sep = '\t') }}
+
 
 It is RECOMMENDED to accompany each `participants.tsv` file with a sidecar
 `participants.json` file to describe the TSV column names and properties of their values (see also
@@ -320,14 +316,8 @@ and a guide for using macros can be found at
 
 `samples.tsv` example:
 
-```Text
-sample_id participant_id sample_type derived_from
-sample-01 sub-01 tissue n/a
-sample-02 sub-01 tissue sample-01
-sample-03 sub-01 tissue sample-01
-sample-04 sub-02 tissue n/a
-sample-05 sub-02 tissue n/a
-```
+{{ read_table('tables/agnostic_samples.tsv', sep = '\t') }}
+
 
 It is RECOMMENDED to accompany each `samples.tsv` file with a sidecar
 `samples.json` file to describe the TSV column names and properties of their values
@@ -486,13 +476,7 @@ All such included additional fields SHOULD be documented in an accompanying
 
 Example `_scans.tsv`:
 
-```Text
-filename	acq_time
-func/sub-control01_task-nback_bold.nii.gz	1877-06-15T13:45:30
-func/sub-control01_task-motor_bold.nii.gz	1877-06-15T13:55:33
-meg/sub-control01_task-rest_split-01_meg.nii.gz	1877-06-15T12:15:27
-meg/sub-control01_task-rest_split-02_meg.nii.gz	1877-06-15T12:15:27
-```
+{{ read_table('tables/agnostic_scans.tsv', sep = '\t') }}
 
 ## Sessions file
 
@@ -522,12 +506,7 @@ and a guide for using macros can be found at
 
 `_sessions.tsv` example:
 
-```Text
-session_id	acq_time	systolic_blood_pressure
-ses-predrug	2009-06-15T13:45:30	120
-ses-postdrug	2009-06-16T13:45:30	100
-ses-followup	2009-06-17T13:45:30	110
-```
+{{ read_table('tables/agnostic_sessions.tsv', sep = '\t') }}
 
 ## Code
 

src/modality-specific-files/behavioral-experiments.md

@@ -59,11 +59,7 @@ A guide for using macros can be found at
 
 ## Example `_beh.tsv`
 
-```Text
-trial	response	response_time	stim_file
-congruent	red	1.435	images/word-red_color-red.jpg
-incongruent	red	1.739	images/word-red_color-blue.jpg
-```
+{{ read_table('../tables/behavioral_beh.tsv', sep = '\t') }}
 
 In the accompanying JSON sidecar, the `trial` column might be documented as follows: