Use ansible-doc's --metadata-dump for ansible-core 2.13+

changelogs/fragments/72-ansible-core-2.13.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "Use the new ``--metadata-dump`` option for ansible-core 2.13+ to quickly dump and extract all module/plugin ``version_added`` values for the collection (https://github.com/ansible-community/antsibull-changelog/pull/72)."

src/antsibull_changelog/ansible.py

@@ -38,7 +38,7 @@
 
 def get_documentable_plugins() -> Tuple[str, ...]:
     """
-    Retrieve plugin types that can be documented. Does not include 'module'.
+    Retrieve plugin types that can be documented.
     """
     if HAS_ANSIBLE_CONSTANTS:
         return C.DOCUMENTABLE_PLUGINS

src/antsibull_changelog/plugins.py

@@ -10,13 +10,18 @@
 
 import json
 import os
+import re
 import shutil
 import subprocess
 import tempfile
 
 from typing import Any, Dict, List, Optional
 
-from .ansible import get_documentable_plugins, get_documentable_objects, PLUGIN_EXCEPTIONS
+import packaging.version
+
+from .ansible import (
+    get_ansible_release, get_documentable_plugins, get_documentable_objects, PLUGIN_EXCEPTIONS,
+)
 from .config import CollectionDetails, PathsConfig
 from .logger import LOGGER
 from .yaml import load_yaml, store_yaml
@@ -101,7 +106,8 @@ def extract_namespace(paths: PathsConfig, collection_name: Optional[str], filena
 def jsondoc_to_metadata(paths: PathsConfig,  # pylint: disable=too-many-arguments
                         collection_name: Optional[str],
                         plugin_type: str, name: str, data: Dict[str, Any],
-                        category: str = 'plugin') -> Dict[str, Any]:
+                        category: str = 'plugin',
+                        is_ansible_core_2_13: bool = False) -> Dict[str, Any]:
     """
     Convert ``ansible-doc --json`` output to plugin metadata.
 
@@ -111,6 +117,7 @@ def jsondoc_to_metadata(paths: PathsConfig,  # pylint: disable=too-many-argument
     :arg name: The plugin's name
     :arg data: The JSON for this plugin returned by ``ansible-doc --json``
     :arg category: Set to ``object`` for roles and playbooks
+    :arg is_ansible_core_2_13: Set to ``True`` for ``--metadata-dump`` output
     """
     namespace: Optional[str] = None
     if collection_name and name.startswith(collection_name + '.'):
@@ -121,12 +128,20 @@ def jsondoc_to_metadata(paths: PathsConfig,  # pylint: disable=too-many-argument
         if 'main' in entrypoints:
             docs = entrypoints['main']
     if category == 'plugin' and plugin_type == 'module':
-        filename: Optional[str] = docs.get('filename')
-        if filename:
-            namespace = extract_namespace(paths, collection_name, filename)
-        if '.' in name:
-            # Flatmapping
-            name = name[name.rfind('.') + 1:]
+        if is_ansible_core_2_13:
+            last_dot = name.rindex('.')
+            if last_dot >= 0:
+                namespace = name[:last_dot]
+                name = name[last_dot + 1:]
+            else:
+                namespace = ''
+        else:
+            filename: Optional[str] = docs.get('filename')
+            if filename:
+                namespace = extract_namespace(paths, collection_name, filename)
+            if '.' in name:
+                # Flatmapping
+                name = name[name.rfind('.') + 1:]
     return {
         'description': docs.get('short_description'),
         'name': name,
@@ -256,6 +271,20 @@ def run_ansible_doc(paths: PathsConfig, playbook_dir: Optional[str],
     return json.loads(output.decode('utf-8'))
 
 
+def run_ansible_doc_metadata_dump(paths: PathsConfig, playbook_dir: Optional[str],
+                                  collection_name: Optional[str]) -> dict:
+    """
+    Runs ansible-doc to retrieve documentation for all plugins in a collection.
+    """
+    command = [paths.ansible_doc_path, '--metadata-dump']
+    if collection_name:
+        command.append(collection_name)
+    if playbook_dir:
+        command.extend(['--playbook-dir', playbook_dir])
+    output = subprocess.check_output(command)
+    return json.loads(output.decode('utf-8'))
+
+
 def load_plugin_metadata(paths: PathsConfig,  # pylint: disable=too-many-arguments
                          playbook_dir: Optional[str],
                          plugin_type: str,
@@ -324,6 +353,40 @@ def __exit__(self, type_, value, traceback_):
         shutil.rmtree(self.dir, ignore_errors=True)
 
 
+def _load_plugins_2_13(plugins_data: Dict[str, Any],
+                       paths: PathsConfig,
+                       collection_name: str,
+                       playbook_dir: Optional[str] = None) -> None:
+    data = run_ansible_doc_metadata_dump(paths, playbook_dir, collection_name)['all']
+
+    for category, category_types in (
+        ('plugins', get_documentable_plugins()),
+        ('objects', get_documentable_objects()),
+    ):
+        for plugin_type in category_types:
+            if plugin_type in data:
+                plugins_data[category][plugin_type] = {}
+                for plugin_name, plugin_data in data[plugin_type].items():
+                    if plugin_name.startswith('ansible.builtin._'):
+                        plugin_name = plugin_name.replace('_', '', 1)
+                    processed_data = jsondoc_to_metadata(
+                        paths, collection_name, plugin_type, plugin_name,
+                        plugin_data, category=category[:-1])
+                    plugins_data[category][plugin_type][processed_data['name']] = processed_data
+
+
+def _load_collection_plugins_2_13(plugins_data: Dict[str, Any],
+                                  paths: PathsConfig,
+                                  collection_details: CollectionDetails) -> None:
+    collection_name = '{}.{}'.format(
+        collection_details.get_namespace(), collection_details.get_name())
+
+    with CollectionCopier(
+            paths, collection_details.get_namespace(), collection_details.get_name()
+    ) as (playbook_dir, new_paths):
+        _load_plugins_2_13(plugins_data, new_paths, collection_name, playbook_dir=playbook_dir)
+
+
 def _load_collection_plugins(plugins_data: Dict[str, Any],
                              paths: PathsConfig,
                              collection_details: CollectionDetails,
@@ -352,6 +415,23 @@ def _load_ansible_plugins(plugins_data: Dict[str, Any], paths: PathsConfig,
             paths, None, plugin_type, None, use_ansible_doc=use_ansible_doc)
 
 
+def _get_ansible_core_version(paths: PathsConfig) -> packaging.version.Version:
+    try:
+        version, _ = get_ansible_release()
+        return packaging.version.Version(version)
+    except ValueError:
+        pass
+
+    command = [paths.ansible_doc_path, '--version']
+    output = subprocess.check_output(command).decode('utf-8')
+    for regex in (r'^ansible-doc \[(?:core|base) ([^\]]+)\]', r'^ansible-doc ([^\s]+)'):
+        match = re.match(regex, output)
+        if match:
+            return packaging.version.Version(match.group(1))
+    raise Exception(
+        f'Cannot extract ansible-core version from ansible-doc --version output:\n{output}')
+
+
 def _refresh_plugin_cache(paths: PathsConfig,
                           collection_details: CollectionDetails,
                           version: str,
@@ -364,7 +444,13 @@ def _refresh_plugin_cache(paths: PathsConfig,
         'objects': {},
     }
 
-    if paths.is_collection:
+    core_version = _get_ansible_core_version(paths)
+    if core_version >= packaging.version.Version('2.13.0.dev0'):
+        if paths.is_collection:
+            _load_collection_plugins_2_13(plugins_data, paths, collection_details)
+        else:
+            _load_plugins_2_13(plugins_data, paths, 'ansible.builtin')
+    elif paths.is_collection:
         _load_collection_plugins(plugins_data, paths, collection_details, use_ansible_doc)
     else:
         _load_ansible_plugins(plugins_data, paths, use_ansible_doc)

tests/functional/fixtures.py

@@ -28,6 +28,17 @@
 # variable to True. Then for all changed files, a colorized diff will be printed.
 PRINT_DIFFS = False
 
+_ANSIBLE_DOC_VERSION_TEMPLATE = '''ansible-doc [core {ansible_core_version}]
+  config file = None
+  configured module search path = ['/home/me/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
+  ansible python module location = /usr/local/lib/python3.9/dist-packages/ansible
+  ansible collection location = /home/me/.ansible/collections:/usr/share/ansible/collections
+  executable location = /usr/local/bin/ansible-doc
+  python version = 3.9.2
+  jinja version = 3.0.3
+  libyaml = True
+'''
+
 
 def diff(old: str, new: str) -> str:
     seqm = difflib.SequenceMatcher(None, old, new)
@@ -131,6 +142,8 @@ def __init__(self, base_path: pathlib.Path, paths: PathsConfig):
         self.created_dirs = set([self.paths.base_dir])
         self.created_files = dict()
 
+        self.ansible_core_version = '2.12.2'
+
     def _write(self, path: str, data: bytes):
         with open(path, 'wb') as f:
             f.write(data)
@@ -341,6 +354,10 @@ def _plugin_base(self, plugin_type):
     def create_fake_subprocess_ansible_doc(self, plugin_data: Dict[str, Dict[str, Any]]
                                            ) -> Callable[[List[str]], str]:
         def fake_subprocess_ansible_doc(command: List[str]) -> str:
+            if command[0].endswith('ansible-doc') and command[1] == '--version':
+                return _ANSIBLE_DOC_VERSION_TEMPLATE.format(
+                    ansible_core_version=self.ansible_core_version,
+                ).encode('utf-8')
             if command[0].endswith('ansible-doc') and command[1] == '--json' and command[2] == '-t':
                 plugin_type = command[3]
                 args = command[4:]
@@ -392,6 +409,10 @@ def create_fake_subprocess_ansible_doc(self, plugin_data: Dict[str, Dict[str, An
                                            ) -> Callable[[List[str]], str]:
         def fake_subprocess_ansible_doc(command: List[str]) -> str:
             base_dir = self.paths.base_dir
+            if command[0].endswith('ansible-doc') and command[1] == '--version':
+                return _ANSIBLE_DOC_VERSION_TEMPLATE.format(
+                    ansible_core_version=self.ansible_core_version,
+                ).encode('utf-8')
             if command[0].endswith('ansible-doc') and command[1] == '--json' and command[2] == '-t':
                 plugin_type = command[3]
                 args = command[4:]