diff --git a/CHANGES.rst b/CHANGES.rst index d47cc43f228..f9490d957fd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -114,6 +114,9 @@ Features added * #13354: Insert abbreviation nodes (hover text) for positional- and keyword-only separators in Python signatures. Patch by Adam Turner. +* #13333: Add the :mod:`sphinx.ext.autodoc` extension, + to automate API documentation generation from Python modules. + Patch by Chris Sewell and Adam Turner. Bugs fixed ---------- diff --git a/doc/usage/extensions/apidoc.rst b/doc/usage/extensions/apidoc.rst new file mode 100644 index 00000000000..4dc8ca941a2 --- /dev/null +++ b/doc/usage/extensions/apidoc.rst @@ -0,0 +1,172 @@ +.. _ext-apidoc: + +:mod:`sphinx.ext.apidoc` -- Generate API documentation from Python packages +=========================================================================== + +.. py:module:: sphinx.ext.apidoc + :synopsis: Generate API documentation from Python modules + +.. index:: pair: automatic; documentation +.. index:: pair: generation; documentation +.. index:: pair: generate; documentation + +.. versionadded:: 8.2 + +.. role:: code-py(code) + :language: Python + +:mod:`sphinx.ext.apidoc` is a tool for automatic generation +of Sphinx sources from Python packages. +It provides the :program:`sphinx-apidoc` command-line tool as an extension, +allowing it to be run during the Sphinx build process. + +The extension writes generated source files to a provided directory, +which are then read by Sphinx using the :mod:`sphinx.ext.autodoc` extension. + +.. warning:: + + :mod:`sphinx.ext.apidoc` generates source files that + use :mod:`sphinx.ext.autodoc` to document all found modules. + If any modules have side effects on import, + these will be executed by ``autodoc`` when :program:`sphinx-build` is run. + + If you document scripts (as opposed to library modules), + make sure their main routine is protected by + an ``if __name__ == '__main__'`` condition. + + +Configuration +------------- + +The apidoc extension uses the following configuration values: + +.. confval:: apidoc_modules + :type: :code-py:`Sequence[dict[str, str | int | bool | Sequence[str] | Set[str]]` + :default: :code-py:`()` + + A list or sequence of dictionaries describing modules to document. + If a value is left unspecified in any dictionary, + the general configuration value is used as the default. + + For example: + + .. code-block:: python + + apidoc_modules = [ + {'path': 'path/to/module', 'destination': 'source/'}, + { + 'path': 'path/to/another_module', + 'destination': 'source/', + 'exclude_patterns': ['**/test*'], + 'max_depth': 4, + 'follow_links': False, + 'separate_modules': False, + 'include_private': False, + 'no_headings': False, + 'module_first': False, + 'implicit_namespaces': False, + 'automodule_options': { + 'members', 'show-inheritance', 'undoc-members' + }, + }, + ] + + + Valid keys are: + + :code-py:`'path'` + The path to the module to document (**required**). + This must be absolute or relative to the configuration directory. + + :code-py:`'destination'` + The output directory for generated files (**required**). + This must be relative to the source directory, + and will be created if it does not exist. + + :code-py:`'exclude_patterns'` + See :confval:`apidoc_exclude_patterns`. + + :code-py:`'max_depth'` + See :confval:`apidoc_max_depth`. + + :code-py:`'follow_links'` + See :confval:`apidoc_follow_links`. + + :code-py:`'separate_modules'` + See :confval:`apidoc_separate_modules`. + + :code-py:`'include_private'` + See :confval:`apidoc_include_private`. + + :code-py:`'no_headings'` + See :confval:`apidoc_no_headings`. + + :code-py:`'module_first'` + See :confval:`apidoc_module_first`. + + :code-py:`'implicit_namespaces'` + See :confval:`apidoc_implicit_namespaces`. + + :code-py:`'automodule_options'` + See :confval:`apidoc_automodule_options`. + +.. confval:: apidoc_exclude_patterns + :type: :code-py:`Sequence[str]` + :default: :code-py:`()` + + A sequence of patterns to exclude from generation. + These may be literal paths or :py:mod:`fnmatch`-style patterns. + +.. confval:: apidoc_max_depth + :type: :code-py:`int` + :default: :code-py:`4` + + The maximum depth of submodules to show in the generated table of contents. + +.. confval:: apidoc_follow_links + :type: :code-py:`bool` + :default: :code-py:`False` + + Follow symbolic links. + +.. confval:: apidoc_separate_modules + :type: :code-py:`bool` + :default: :code-py:`False` + + Put documentation for each module on an individual page. + +.. confval:: apidoc_include_private + :type: :code-py:`bool` + :default: :code-py:`False` + + Generate documentation for '_private' modules with leading underscores. + +.. confval:: apidoc_no_headings + :type: :code-py:`bool` + :default: :code-py:`False` + + Do not create headings for the modules/packages. + Useful when source docstrings already contain headings. + +.. confval:: apidoc_module_first + :type: :code-py:`bool` + :default: :code-py:`False` + + Place module documentation before submodule documentation. + +.. confval:: apidoc_implicit_namespaces + :type: :code-py:`bool` + :default: :code-py:`False` + + By default sphinx-apidoc processes sys.path searching for modules only. + Python 3.3 introduced :pep:`420` implicit namespaces that allow module path + structures such as ``foo/bar/module.py`` or ``foo/bar/baz/__init__.py`` + (notice that ``bar`` and ``foo`` are namespaces, not modules). + + Interpret module paths using :pep:`420` implicit namespaces. + +.. confval:: apidoc_automodule_options + :type: :code-py:`Set[str]` + :default: :code-py:`{'members', 'show-inheritance', 'undoc-members'}` + + Options to pass to generated :rst:dir:`automodule` directives. diff --git a/doc/usage/extensions/index.rst b/doc/usage/extensions/index.rst index 4be426c3fe2..7e53fea6d91 100644 --- a/doc/usage/extensions/index.rst +++ b/doc/usage/extensions/index.rst @@ -21,6 +21,7 @@ These extensions are built in and can be activated by respective entries in the .. toctree:: :maxdepth: 1 + apidoc autodoc autosectionlabel autosummary diff --git a/sphinx/ext/apidoc/__init__.py b/sphinx/ext/apidoc/__init__.py index b574b9bda4a..be52485771c 100644 --- a/sphinx/ext/apidoc/__init__.py +++ b/sphinx/ext/apidoc/__init__.py @@ -13,9 +13,54 @@ from typing import TYPE_CHECKING +import sphinx from sphinx.ext.apidoc._cli import main if TYPE_CHECKING: from collections.abc import Sequence -__all__: Sequence[str] = ('main',) + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + +__all__: Sequence[str] = 'main', 'setup' + + +def setup(app: Sphinx) -> ExtensionMetadata: + from sphinx.ext.apidoc._extension import run_apidoc + + # Require autodoc + app.setup_extension('sphinx.ext.autodoc') + + # Configuration values + app.add_config_value( + 'apidoc_exclude_patterns', (), 'env', types=frozenset({list, tuple}) + ) + app.add_config_value('apidoc_max_depth', 4, 'env', types=frozenset({int})) + app.add_config_value('apidoc_follow_links', False, 'env', types=frozenset({bool})) + app.add_config_value( + 'apidoc_separate_modules', False, 'env', types=frozenset({bool}) + ) + app.add_config_value( + 'apidoc_include_private', False, 'env', types=frozenset({bool}) + ) + app.add_config_value('apidoc_no_headings', False, 'env', types=frozenset({bool})) + app.add_config_value('apidoc_module_first', False, 'env', types=frozenset({bool})) + app.add_config_value( + 'apidoc_implicit_namespaces', False, 'env', types=frozenset({bool}) + ) + app.add_config_value( + 'apidoc_automodule_options', + frozenset(('members', 'undoc-members', 'show-inheritance')), + 'env', + types=frozenset({frozenset, list, set, tuple}), + ) + app.add_config_value('apidoc_modules', (), 'env', types=frozenset({list, tuple})) + + # Entry point to run apidoc + app.connect('builder-inited', run_apidoc) + + return { + 'version': sphinx.__display_version__, + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/sphinx/ext/apidoc/_extension.py b/sphinx/ext/apidoc/_extension.py new file mode 100644 index 00000000000..cb40cc4fd78 --- /dev/null +++ b/sphinx/ext/apidoc/_extension.py @@ -0,0 +1,262 @@ +"""Sphinx extension for auto-generating API documentation.""" + +from __future__ import annotations + +import fnmatch +import re +from pathlib import Path +from typing import TYPE_CHECKING + +from sphinx._cli.util.colour import bold +from sphinx.ext.apidoc._generate import create_modules_toc_file, recurse_tree +from sphinx.ext.apidoc._shared import ( + LOGGER, + ApidocDefaults, + ApidocOptions, + _remove_old_files, +) +from sphinx.locale import __ + +if TYPE_CHECKING: + from collections.abc import Collection, Sequence + from typing import Any + + from sphinx.application import Sphinx + +_BOOL_KEYS = frozenset({ + 'follow_links', + 'separate_modules', + 'include_private', + 'no_headings', + 'module_first', + 'implicit_namespaces', +}) +_ALLOWED_KEYS = _BOOL_KEYS | frozenset({ + 'path', + 'destination', + 'exclude_patterns', + 'automodule_options', + 'max_depth', +}) + + +def run_apidoc(app: Sphinx) -> None: + """Run the apidoc extension.""" + defaults = ApidocDefaults.from_config(app.config) + apidoc_modules: Sequence[dict[str, Any]] = app.config.apidoc_modules + srcdir: Path = app.srcdir + confdir: Path = app.confdir + + LOGGER.info(bold(__('Running apidoc'))) + + module_options: dict[str, Any] + for i, module_options in enumerate(apidoc_modules): + _run_apidoc_module( + i, + options=module_options, + defaults=defaults, + srcdir=srcdir, + confdir=confdir, + ) + + +def _run_apidoc_module( + i: int, + *, + options: dict[str, Any], + defaults: ApidocDefaults, + srcdir: Path, + confdir: Path, +) -> None: + """Run apidoc for a single module.""" + args = _parse_module_options( + i, options=options, defaults=defaults, srcdir=srcdir, confdir=confdir + ) + if args is None: + return + + exclude_patterns_compiled: list[re.Pattern[str]] = [ + re.compile(fnmatch.translate(exclude)) for exclude in args.exclude_pattern + ] + + written_files, modules = recurse_tree( + args.module_path, exclude_patterns_compiled, args, args.template_dir + ) + if args.toc_file: + written_files.append( + create_modules_toc_file(modules, args, args.toc_file, args.template_dir) + ) + if args.remove_old: + _remove_old_files(written_files, args.dest_dir, args.suffix) + + +def _parse_module_options( + i: int, + *, + options: dict[str, Any], + defaults: ApidocDefaults, + srcdir: Path, + confdir: Path, +) -> ApidocOptions | None: + if not isinstance(options, dict): + LOGGER.warning(__('apidoc_modules item %i must be a dict'), i, type='apidoc') + return None + + # module path should be absolute or relative to the conf directory + try: + path = Path(options['path']) + except KeyError: + LOGGER.warning( + __("apidoc_modules item %i must have a 'path' key"), i, type='apidoc' + ) + return None + except TypeError: + LOGGER.warning( + __("apidoc_modules item %i 'path' must be a string"), i, type='apidoc' + ) + return None + module_path = confdir / path + if not module_path.is_dir(): + LOGGER.warning( + __("apidoc_modules item %i 'path' is not an existing folder: %s"), + i, + module_path, + type='apidoc', + ) + return None + + # destination path should be relative to the source directory + try: + destination = Path(options['destination']) + except KeyError: + LOGGER.warning( + __("apidoc_modules item %i must have a 'destination' key"), + i, + type='apidoc', + ) + return None + except TypeError: + LOGGER.warning( + __("apidoc_modules item %i 'destination' must be a string"), + i, + type='apidoc', + ) + return None + if destination.is_absolute(): + LOGGER.warning( + __("apidoc_modules item %i 'destination' should be a relative path"), + i, + type='apidoc', + ) + return None + dest_path = srcdir / destination + try: + dest_path.mkdir(parents=True, exist_ok=True) + except OSError as exc: + LOGGER.warning( + __('apidoc_modules item %i cannot create destination directory: %s'), + i, + exc.strerror, + type='apidoc', + ) + return None + + # exclude patterns should be absolute or relative to the conf directory + exclude_patterns: list[str] = [ + str(confdir / pattern) + for pattern in _check_collection_of_strings( + i, options, key='exclude_patterns', default=defaults.exclude_patterns + ) + ] + + # TODO template_dir + + max_depth = defaults.max_depth + if 'max_depth' in options: + if not isinstance(options['max_depth'], int): + LOGGER.warning( + __("apidoc_modules item %i '%s' must be an int"), + i, + 'max_depth', + type='apidoc', + ) + else: + max_depth = options['max_depth'] + + bool_options: dict[str, bool] = {} + for key in sorted(_BOOL_KEYS): + if key not in options: + bool_options[key] = getattr(defaults, key) + elif not isinstance(options[key], bool): + LOGGER.warning( + __("apidoc_modules item %i '%s' must be a boolean"), + i, + key, + type='apidoc', + ) + bool_options[key] = getattr(defaults, key) + else: + bool_options[key] = options[key] + + # TODO per-module automodule_options + automodule_options = frozenset( + _check_collection_of_strings( + i, options, key='automodule_options', default=defaults.automodule_options + ) + ) + + if diff := options.keys() - _ALLOWED_KEYS: + LOGGER.warning( + __('apidoc_modules item %i has unexpected keys: %s'), + i, + ', '.join(sorted(diff)), + type='apidoc', + ) + + return ApidocOptions( + dest_dir=dest_path, + module_path=module_path, + exclude_pattern=exclude_patterns, + automodule_options=automodule_options, + max_depth=max_depth, + quiet=True, + follow_links=bool_options['follow_links'], + separate_modules=bool_options['separate_modules'], + include_private=bool_options['include_private'], + no_headings=bool_options['no_headings'], + module_first=bool_options['module_first'], + implicit_namespaces=bool_options['implicit_namespaces'], + ) + + +def _check_collection_of_strings( + index: int, + options: dict[str, Any], + *, + key: str, + default: Collection[str], +) -> Collection[str]: + """Check that a key's value is a collection of strings in the options. + + :returns: The value of the key, or None if invalid. + """ + if key not in options: + return default + if not isinstance(options[key], list | tuple | set | frozenset): + LOGGER.warning( + __("apidoc_modules item %i '%s' must be a sequence"), + index, + key, + type='apidoc', + ) + return default + for item in options[key]: + if not isinstance(item, str): + LOGGER.warning( + __("apidoc_modules item %i '%s' must contain strings"), + index, + key, + type='apidoc', + ) + return default + return options[key] diff --git a/sphinx/ext/apidoc/_shared.py b/sphinx/ext/apidoc/_shared.py index 229b9a7226a..527f240f9df 100644 --- a/sphinx/ext/apidoc/_shared.py +++ b/sphinx/ext/apidoc/_shared.py @@ -9,7 +9,9 @@ if TYPE_CHECKING: from collections.abc import Sequence, Set from pathlib import Path - from typing import Final + from typing import Final, Self + + from sphinx.config import Config LOGGER: Final[logging.SphinxLoggerAdapter] = logging.getLogger('sphinx.ext.apidoc') @@ -35,15 +37,12 @@ def _remove_old_files( class ApidocOptions: """Options for apidoc.""" - module_path: Path dest_dir: Path + module_path: Path exclude_pattern: Sequence[str] = () - quiet: bool = False max_depth: int = 4 - force: bool = False follow_links: bool = False - dry_run: bool = False separate_modules: bool = False include_private: bool = False toc_file: str = 'modules' @@ -53,7 +52,11 @@ class ApidocOptions: automodule_options: Set[str] = dataclasses.field(default_factory=set) suffix: str = 'rst' - remove_old: bool = False + remove_old: bool = True + + quiet: bool = False + dry_run: bool = False + force: bool = True # --full only full: bool = False @@ -64,3 +67,33 @@ class ApidocOptions: release: str | None = None extensions: Sequence[str] | None = None template_dir: str | None = None + + +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) +class ApidocDefaults: + """Default values for apidoc options.""" + + exclude_patterns: list[str] + automodule_options: frozenset[str] + max_depth: int + follow_links: bool + separate_modules: bool + include_private: bool + no_headings: bool + module_first: bool + implicit_namespaces: bool + + @classmethod + def from_config(cls, config: Config, /) -> Self: + """Collect the default values for apidoc options.""" + return cls( + exclude_patterns=config.apidoc_exclude_patterns, + automodule_options=frozenset(config.apidoc_automodule_options), + max_depth=config.apidoc_max_depth, + follow_links=config.apidoc_follow_links, + separate_modules=config.apidoc_separate_modules, + include_private=config.apidoc_include_private, + no_headings=config.apidoc_no_headings, + module_first=config.apidoc_module_first, + implicit_namespaces=config.apidoc_implicit_namespaces, + ) diff --git a/tests/roots/test-ext-apidoc/conf.py b/tests/roots/test-ext-apidoc/conf.py new file mode 100644 index 00000000000..3cd0ceea431 --- /dev/null +++ b/tests/roots/test-ext-apidoc/conf.py @@ -0,0 +1,22 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path.cwd().resolve() / 'src')) + +extensions = ['sphinx.ext.apidoc'] + +apidoc_include_private = True +apidoc_follow_links = False +apidoc_separate_modules = True +apidoc_no_headings = False +apidoc_module_first = True +apidoc_modules = [ + { + 'path': 'src', + 'destination': 'generated', + 'exclude_patterns': ['src/exclude_package.py'], + 'automodule_options': ['members', 'undoc-members'], + 'max_depth': 3, + 'implicit_namespaces': False, + } +] diff --git a/tests/roots/test-ext-apidoc/index.rst b/tests/roots/test-ext-apidoc/index.rst new file mode 100644 index 00000000000..6660f8661b4 --- /dev/null +++ b/tests/roots/test-ext-apidoc/index.rst @@ -0,0 +1,6 @@ +Heading +======= + +.. toctree:: + + generated/modules diff --git a/tests/roots/test-ext-apidoc/src/exclude_package.py b/tests/roots/test-ext-apidoc/src/exclude_package.py new file mode 100644 index 00000000000..4aa8f8308e9 --- /dev/null +++ b/tests/roots/test-ext-apidoc/src/exclude_package.py @@ -0,0 +1 @@ +"""A module that should be excluded.""" diff --git a/tests/roots/test-ext-apidoc/src/my_package.py b/tests/roots/test-ext-apidoc/src/my_package.py new file mode 100644 index 00000000000..2f698f97260 --- /dev/null +++ b/tests/roots/test-ext-apidoc/src/my_package.py @@ -0,0 +1,6 @@ +"""An example module.""" + + +def example_function(a: str) -> str: + """An example function.""" + return a diff --git a/tests/test_extensions/test_ext_apidoc.py b/tests/test_extensions/test_ext_apidoc.py index e656779a968..7a9d6219415 100644 --- a/tests/test_extensions/test_ext_apidoc.py +++ b/tests/test_extensions/test_ext_apidoc.py @@ -13,6 +13,8 @@ if TYPE_CHECKING: from pathlib import Path + from sphinx.testing.util import SphinxTestApp + _apidoc = namedtuple('_apidoc', 'coderoot,outdir') # NoQA: PYI024 @@ -772,3 +774,26 @@ def test_remove_old_files(tmp_path: Path): apidoc_main(['--remove-old', '-o', str(gen_dir), str(module_dir)]) assert set(gen_dir.iterdir()) == {gen_dir / 'modules.rst', gen_dir / 'example.rst'} assert (gen_dir / 'example.rst').stat().st_mtime_ns == example_mtime + + +@pytest.mark.sphinx(testroot='ext-apidoc') +def test_sphinx_extension(app: SphinxTestApp) -> None: + """Test running apidoc as an extension.""" + app.build() + assert app.warning.getvalue() == '' + + assert set((app.srcdir / 'generated').iterdir()) == { + app.srcdir / 'generated' / 'modules.rst', + app.srcdir / 'generated' / 'my_package.rst', + } + assert 'show-inheritance' not in ( + app.srcdir / 'generated' / 'my_package.rst' + ).read_text(encoding='utf8') + assert (app.outdir / 'generated' / 'my_package.html').is_file() + + # test a re-build + app.build() + assert app.warning.getvalue() == '' + + # TODO check nothing got re-built + # TODO test that old files are removed