From d2a54946cb605a7500ed118b1dc445f2f1d5983e Mon Sep 17 00:00:00 2001 From: lochhh Date: Tue, 2 Jul 2024 14:14:52 +0100 Subject: [PATCH 1/5] Automate api_index.rst generation --- .github/workflows/docs_build_and_deploy.yml | 45 +++++++++++++++++++-- .gitignore | 1 + CONTRIBUTING.md | 32 ++++----------- docs/Makefile | 13 +++++- docs/make.bat | 3 ++ docs/make_api_index.py | 44 ++++++++++++++++++++ docs/source/_templates/api_index_head.rst | 13 ++++++ docs/source/api_index.rst | 25 ------------ 8 files changed, 122 insertions(+), 54 deletions(-) create mode 100644 docs/make_api_index.py create mode 100644 docs/source/_templates/api_index_head.rst delete mode 100644 docs/source/api_index.rst diff --git a/.github/workflows/docs_build_and_deploy.yml b/.github/workflows/docs_build_and_deploy.yml index fe65f42fc..682ea8f28 100644 --- a/.github/workflows/docs_build_and_deploy.yml +++ b/.github/workflows/docs_build_and_deploy.yml @@ -27,9 +27,48 @@ jobs: name: Build Sphinx Docs runs-on: ubuntu-latest steps: - - uses: neuroinformatics-unit/actions/build_sphinx_docs@main - with: - python-version: 3.11 + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Upgrade pip + shell: bash + run: | + # install pip=>20.1 to use "pip cache dir" + python3 -m pip install --upgrade pip + + - name: Get pip cache dir + shell: bash + id: pip-cache + run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + shell: bash + run: python3 -m pip install -r ./docs/requirements.txt + + - name: Check links and build documentation + shell: bash + run: | + cd docs + make linkcheck + make html + + - name: Upload the content for deployment + uses: actions/upload-artifact@v4 + with: + name: docs-build + path: ./docs/build/ deploy_sphinx_docs: name: Deploy Sphinx Docs diff --git a/.gitignore b/.gitignore index 19c88937c..c46b1d71a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ instance/ docs/build/ docs/source/examples/ docs/source/api/ +docs/source/api_index.rst sg_execution_times.rst # MkDocs documentation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec3ee0e74..890c88ad1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -209,32 +209,14 @@ If it is not yet defined and you have multiple external links pointing to the sa ### Updating the API reference -If your PR introduces new public modules, or renames existing ones, -make sure to add them to the `docs/source/api_index.rst` page, so that they are included in the [API reference](target-api), e.g.: - -```rst -API Reference -============= - -Information on specific functions, classes, and methods. - -.. rubric:: Modules - -.. autosummary:: - :toctree: api - :recursive: - :nosignatures: - - movement.move_accessor - movement.io.load_poses - movement.io.save_poses - movement.your_new_module -``` - -The API reference is auto-generated by the `sphinx-autodoc` and `sphinx-autosummary` plugins, based on docstrings. +The API reference is auto-generated by the `docs/make_api_index.py` script, and the `sphinx-autodoc` and `sphinx-autosummary` plugins. +The script generates the `docs/source/api_index.rst` file containing the list of modules to be included in the [API reference](target-api). +The plugins then generate the API reference pages for each module listed in `api_index.rst`, based on the docstrings in the source code. So make sure that all your public functions/classes/methods have valid docstrings following the [numpydoc](https://numpydoc.readthedocs.io/en/latest/format.html) style. Our `pre-commit` hooks include some checks (`ruff` rules) that ensure the docstrings are formatted consistently. +If your PR introduces new modules that should not be documented in the [API reference](target-api), or if there are changes to existing modules that necessitate their removal from the documentation, make sure to update the `exclude_modules` list within the `docs/make_api_index.py` script to reflect these exclusions. + ### Updating the examples We use [sphinx-gallery](sphinx-gallery:) to create the [examples](target-examples). @@ -259,14 +241,14 @@ pip install -r docs/requirements.txt Then, from the root of the repository, run: ```sh -sphinx-build docs/source docs/build +python docs/make_api_index.py && sphinx-build docs/source docs/build ``` You can view the local build by opening `docs/build/index.html` in a browser. To refresh the documentation, after making changes, remove the `docs/build` folder and re-run the above command: ```sh -rm -rf docs/build && sphinx-build docs/source docs/build +rm -rf docs/build && python docs/make_api_index.py && sphinx-build docs/source docs/build ``` To check that external links are correctly resolved, run: diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf10..623d2906a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -14,7 +14,18 @@ help: .PHONY: help Makefile +# Generate the API index file +api_index.rst: + python make_api_index.py + +# Remove all generated files +clean: + rm -rf ./build + rm -f ./source/api_index.rst + rm -rf ./source/api + rm -rf ./source/examples + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile +%: Makefile api_index.rst @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat index dc1312ab0..79a8b01a6 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -25,6 +25,9 @@ if errorlevel 9009 ( if "%1" == "" goto help +echo "Generating API index..." +python make_api_index.py + %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end diff --git a/docs/make_api_index.py b/docs/make_api_index.py new file mode 100644 index 000000000..372231129 --- /dev/null +++ b/docs/make_api_index.py @@ -0,0 +1,44 @@ +"""Generate the API index page for all ``movement`` modules.""" + +import os + +# Modules to exclude from the API index +exclude_modules = ["cli_entrypoint"] + +# Set the current working directory to the directory of this script +script_dir = os.path.dirname(os.path.abspath(__file__)) +os.chdir(script_dir) + + +def make_api_index(): + """Create a doctree of all ``movement`` modules.""" + doctree = "\n" + + for root, _, files in os.walk("../movement"): + # Remove leading "../" + root = root[3:] + for file in sorted(files): + if file.endswith(".py") and not file.startswith("_"): + # Convert file path to module name + module_name = os.path.join(root, file) + module_name = module_name[:-3].replace(os.sep, ".") + # Check if the module should be excluded + if not any( + file.startswith(exclude_module) + for exclude_module in exclude_modules + ): + doctree += f" {module_name}\n" + + # Get the header + with open("./source/_templates/api_index_head.rst") as f: + api_head = f.read() + # Write api_index.rst with header + doctree + with open("./source/api_index.rst", "w") as f: + f.write("..\n This file is auto-generated.\n\n") + f.write(api_head) + f.write(doctree) + print(os.path.abspath("./source/api_index.rst")) + + +if __name__ == "__main__": + make_api_index() diff --git a/docs/source/_templates/api_index_head.rst b/docs/source/_templates/api_index_head.rst new file mode 100644 index 000000000..f1df7eb61 --- /dev/null +++ b/docs/source/_templates/api_index_head.rst @@ -0,0 +1,13 @@ +.. _target-api: + +API Reference +============= + +Information on specific functions, classes, and methods. + +.. rubric:: Modules + +.. autosummary:: + :toctree: api + :recursive: + :nosignatures: diff --git a/docs/source/api_index.rst b/docs/source/api_index.rst deleted file mode 100644 index 3960acf12..000000000 --- a/docs/source/api_index.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _target-api: - -API Reference -============= - -Information on specific functions, classes, and methods. - -.. rubric:: Modules - -.. autosummary:: - :toctree: api - :recursive: - :nosignatures: - - movement.move_accessor - movement.io.load_poses - movement.io.save_poses - movement.filtering - movement.analysis.kinematics - movement.utils.vector - movement.utils.logging - movement.utils.reports - movement.sample_data - movement.validators.files - movement.validators.datasets From 2282a2844990e3daa3072d470afa4d4a039628d4 Mon Sep 17 00:00:00 2001 From: lochhh Date: Tue, 30 Jul 2024 11:07:17 +0100 Subject: [PATCH 2/5] Revert to using (updated) NIU build and publish docs actions --- .github/workflows/docs_build_and_deploy.yml | 47 +++------------------ 1 file changed, 5 insertions(+), 42 deletions(-) diff --git a/.github/workflows/docs_build_and_deploy.yml b/.github/workflows/docs_build_and_deploy.yml index 682ea8f28..f9e3a5c89 100644 --- a/.github/workflows/docs_build_and_deploy.yml +++ b/.github/workflows/docs_build_and_deploy.yml @@ -27,48 +27,10 @@ jobs: name: Build Sphinx Docs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - - name: Upgrade pip - shell: bash - run: | - # install pip=>20.1 to use "pip cache dir" - python3 -m pip install --upgrade pip - - - name: Get pip cache dir - shell: bash - id: pip-cache - run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install dependencies - shell: bash - run: python3 -m pip install -r ./docs/requirements.txt - - - name: Check links and build documentation - shell: bash - run: | - cd docs - make linkcheck - make html - - - name: Upload the content for deployment - uses: actions/upload-artifact@v4 - with: - name: docs-build - path: ./docs/build/ + - uses: neuroinformatics-unit/actions/build_sphinx_docs@main + with: + python-version: 3.11 + use-make: true deploy_sphinx_docs: name: Deploy Sphinx Docs @@ -81,3 +43,4 @@ jobs: - uses: neuroinformatics-unit/actions/deploy_sphinx_docs@main with: secret_input: ${{ secrets.GITHUB_TOKEN }} + use-make: true From 6d783fccd838032222e4538c842474abbb314b93 Mon Sep 17 00:00:00 2001 From: lochhh Date: Tue, 30 Jul 2024 14:18:33 +0100 Subject: [PATCH 3/5] Include `make` commands in fancy tabs --- CONTRIBUTING.md | 56 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 890c88ad1..04865557d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -215,7 +215,7 @@ The plugins then generate the API reference pages for each module listed in `api So make sure that all your public functions/classes/methods have valid docstrings following the [numpydoc](https://numpydoc.readthedocs.io/en/latest/format.html) style. Our `pre-commit` hooks include some checks (`ruff` rules) that ensure the docstrings are formatted consistently. -If your PR introduces new modules that should not be documented in the [API reference](target-api), or if there are changes to existing modules that necessitate their removal from the documentation, make sure to update the `exclude_modules` list within the `docs/make_api_index.py` script to reflect these exclusions. +If your PR introduces new modules that should *not* be documented in the [API reference](target-api), or if there are changes to existing modules that necessitate their removal from the documentation, make sure to update the `exclude_modules` list within the `docs/make_api_index.py` script to reflect these exclusions. ### Updating the examples We use [sphinx-gallery](sphinx-gallery:) @@ -234,28 +234,67 @@ examples are run. See the relevant section of the ### Building the documentation locally We recommend that you build and view the documentation website locally, before you push it. -To do so, first install the requirements for building the documentation: +To do so, first navigate to `docs/`. +All subsequent commands should be run from within this directory. ```sh -pip install -r docs/requirements.txt +cd docs +``` +Install the requirements for building the documentation: +```sh +pip install -r requirements.txt ``` -Then, from the root of the repository, run: +Build the documentation: + +::::{tab-set} +:::{tab-item} All platforms ```sh -python docs/make_api_index.py && sphinx-build docs/source docs/build +python make_api_index.py && sphinx-build source build ``` +::: + +:::{tab-item} Unix platforms with `make` +```sh +make html +``` +::: +:::: You can view the local build by opening `docs/build/index.html` in a browser. -To refresh the documentation, after making changes, remove the `docs/build` folder and re-run the above command: +To refresh the documentation after making changes, remove all generated files in `docs/`, +including the auto-generated API index `source/api_index.rst`, and those in `build/`, `source/api/`, and `source/examples/`. +Then, re-run the above command to rebuild the documentation. +::::{tab-set} +:::{tab-item} All platforms ```sh -rm -rf docs/build && python docs/make_api_index.py && sphinx-build docs/source docs/build +rm -f source/api_index.rst && rm -rf build && rm -rf source/api && rm -rf source/examples +python make_api_index.py && sphinx-build source build ``` +::: + +:::{tab-item} Unix platforms with `make` +```sh +make clean html +``` +::: +:::: To check that external links are correctly resolved, run: +::::{tab-set} +:::{tab-item} All platforms +```sh +sphinx-build source build -b linkcheck +``` +::: + +:::{tab-item} Unix platforms with `make` ```sh -sphinx-build docs/source docs/build -b linkcheck +make linkcheck ``` +::: +:::: If the linkcheck step incorrectly marks links with valid anchors as broken, you can skip checking the anchors in specific links by adding the URLs to `linkcheck_anchors_ignore_for_url` in `docs/source/conf.py`, e.g.: @@ -314,7 +353,6 @@ To add a new file, you will need to: 5. Add your new files to the `poses`, `bboxes`, `videos` and/or `frames` folders as appropriate. Follow the existing file naming conventions as closely as possible. 6. Determine the sha256 checksum hash of each new file. You can do this in a terminal by running: ::::{tab-set} - :::{tab-item} Ubuntu ```bash sha256sum From 84f0f0af4315840a8f6e360bb1bf36f4f89f90fd Mon Sep 17 00:00:00 2001 From: lochhh Date: Tue, 30 Jul 2024 15:37:26 +0100 Subject: [PATCH 4/5] Swap tab order --- CONTRIBUTING.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 04865557d..e1a42004f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -247,35 +247,36 @@ pip install -r requirements.txt Build the documentation: ::::{tab-set} -:::{tab-item} All platforms +:::{tab-item} Unix platforms with `make` ```sh -python make_api_index.py && sphinx-build source build +make html ``` ::: -:::{tab-item} Unix platforms with `make` +:::{tab-item} All platforms ```sh -make html +python make_api_index.py && sphinx-build source build ``` ::: :::: -You can view the local build by opening `docs/build/index.html` in a browser. +If using `make`, the local build can be viewed by opening `docs/build/html/index.html` in a browser. Otherwise, the file will be located at `docs/build/index.html`. + To refresh the documentation after making changes, remove all generated files in `docs/`, including the auto-generated API index `source/api_index.rst`, and those in `build/`, `source/api/`, and `source/examples/`. Then, re-run the above command to rebuild the documentation. ::::{tab-set} -:::{tab-item} All platforms +:::{tab-item} Unix platforms with `make` ```sh -rm -f source/api_index.rst && rm -rf build && rm -rf source/api && rm -rf source/examples -python make_api_index.py && sphinx-build source build +make clean html ``` ::: -:::{tab-item} Unix platforms with `make` +:::{tab-item} All platforms ```sh -make clean html +rm -f source/api_index.rst && rm -rf build && rm -rf source/api && rm -rf source/examples +python make_api_index.py && sphinx-build source build ``` ::: :::: @@ -283,15 +284,15 @@ make clean html To check that external links are correctly resolved, run: ::::{tab-set} -:::{tab-item} All platforms +:::{tab-item} Unix platforms with `make` ```sh -sphinx-build source build -b linkcheck +make linkcheck ``` ::: -:::{tab-item} Unix platforms with `make` +:::{tab-item} All platforms ```sh -make linkcheck +sphinx-build source build -b linkcheck ``` ::: :::: From 57b8282d64c655b4226992b672cd21e359c6c8eb Mon Sep 17 00:00:00 2001 From: lochhh Date: Tue, 30 Jul 2024 15:43:45 +0100 Subject: [PATCH 5/5] Mention index paths in the corresponding tabs --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1a42004f..e39412a65 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -251,17 +251,17 @@ Build the documentation: ```sh make html ``` +The local build can be viewed by opening `docs/build/html/index.html` in a browser. ::: :::{tab-item} All platforms ```sh python make_api_index.py && sphinx-build source build ``` +The local build can be viewed by opening `docs/build/index.html` in a browser. ::: :::: -If using `make`, the local build can be viewed by opening `docs/build/html/index.html` in a browser. Otherwise, the file will be located at `docs/build/index.html`. - To refresh the documentation after making changes, remove all generated files in `docs/`, including the auto-generated API index `source/api_index.rst`, and those in `build/`, `source/api/`, and `source/examples/`. Then, re-run the above command to rebuild the documentation.