diff --git a/changelogs/fragments/72-ansible-core-2.13.yml b/changelogs/fragments/72-ansible-core-2.13.yml new file mode 100644 index 00000000..3397678b --- /dev/null +++ b/changelogs/fragments/72-ansible-core-2.13.yml @@ -0,0 +1,2 @@ +minor_changes: + - "Use the new ``--metadata-dump`` option for ansible-core 2.13+ to quickly dump and extract all module/plugin ``version_added`` values for the collection (https://github.com/ansible-community/antsibull-changelog/pull/72)." diff --git a/src/antsibull_changelog/ansible.py b/src/antsibull_changelog/ansible.py index 180c37fa..e335255c 100644 --- a/src/antsibull_changelog/ansible.py +++ b/src/antsibull_changelog/ansible.py @@ -38,7 +38,7 @@ def get_documentable_plugins() -> Tuple[str, ...]: """ - Retrieve plugin types that can be documented. Does not include 'module'. + Retrieve plugin types that can be documented. """ if HAS_ANSIBLE_CONSTANTS: return C.DOCUMENTABLE_PLUGINS diff --git a/src/antsibull_changelog/plugins.py b/src/antsibull_changelog/plugins.py index 026bc40a..c63764bc 100644 --- a/src/antsibull_changelog/plugins.py +++ b/src/antsibull_changelog/plugins.py @@ -10,13 +10,18 @@ import json import os +import re import shutil import subprocess import tempfile from typing import Any, Dict, List, Optional -from .ansible import get_documentable_plugins, get_documentable_objects, PLUGIN_EXCEPTIONS +import packaging.version + +from .ansible import ( + get_ansible_release, get_documentable_plugins, get_documentable_objects, PLUGIN_EXCEPTIONS, +) from .config import CollectionDetails, PathsConfig from .logger import LOGGER from .yaml import load_yaml, store_yaml @@ -101,7 +106,8 @@ def extract_namespace(paths: PathsConfig, collection_name: Optional[str], filena def jsondoc_to_metadata(paths: PathsConfig, # pylint: disable=too-many-arguments collection_name: Optional[str], plugin_type: str, name: str, data: Dict[str, Any], - category: str = 'plugin') -> Dict[str, Any]: + category: str = 'plugin', + is_ansible_core_2_13: bool = False) -> Dict[str, Any]: """ Convert ``ansible-doc --json`` output to plugin metadata. @@ -111,6 +117,7 @@ def jsondoc_to_metadata(paths: PathsConfig, # pylint: disable=too-many-argument :arg name: The plugin's name :arg data: The JSON for this plugin returned by ``ansible-doc --json`` :arg category: Set to ``object`` for roles and playbooks + :arg is_ansible_core_2_13: Set to ``True`` for ``--metadata-dump`` output """ namespace: Optional[str] = None if collection_name and name.startswith(collection_name + '.'): @@ -121,12 +128,20 @@ def jsondoc_to_metadata(paths: PathsConfig, # pylint: disable=too-many-argument if 'main' in entrypoints: docs = entrypoints['main'] if category == 'plugin' and plugin_type == 'module': - filename: Optional[str] = docs.get('filename') - if filename: - namespace = extract_namespace(paths, collection_name, filename) - if '.' in name: - # Flatmapping - name = name[name.rfind('.') + 1:] + if is_ansible_core_2_13: + last_dot = name.rindex('.') + if last_dot >= 0: + namespace = name[:last_dot] + name = name[last_dot + 1:] + else: + namespace = '' + else: + filename: Optional[str] = docs.get('filename') + if filename: + namespace = extract_namespace(paths, collection_name, filename) + if '.' in name: + # Flatmapping + name = name[name.rfind('.') + 1:] return { 'description': docs.get('short_description'), 'name': name, @@ -256,6 +271,20 @@ def run_ansible_doc(paths: PathsConfig, playbook_dir: Optional[str], return json.loads(output.decode('utf-8')) +def run_ansible_doc_metadata_dump(paths: PathsConfig, playbook_dir: Optional[str], + collection_name: Optional[str]) -> dict: + """ + Runs ansible-doc to retrieve documentation for all plugins in a collection. + """ + command = [paths.ansible_doc_path, '--metadata-dump'] + if collection_name: + command.append(collection_name) + if playbook_dir: + command.extend(['--playbook-dir', playbook_dir]) + output = subprocess.check_output(command) + return json.loads(output.decode('utf-8')) + + def load_plugin_metadata(paths: PathsConfig, # pylint: disable=too-many-arguments playbook_dir: Optional[str], plugin_type: str, @@ -324,6 +353,40 @@ def __exit__(self, type_, value, traceback_): shutil.rmtree(self.dir, ignore_errors=True) +def _load_plugins_2_13(plugins_data: Dict[str, Any], + paths: PathsConfig, + collection_name: str, + playbook_dir: Optional[str] = None) -> None: + data = run_ansible_doc_metadata_dump(paths, playbook_dir, collection_name)['all'] + + for category, category_types in ( + ('plugins', get_documentable_plugins()), + ('objects', get_documentable_objects()), + ): + for plugin_type in category_types: + if plugin_type in data: + plugins_data[category][plugin_type] = {} + for plugin_name, plugin_data in data[plugin_type].items(): + if plugin_name.startswith('ansible.builtin._'): + plugin_name = plugin_name.replace('_', '', 1) + processed_data = jsondoc_to_metadata( + paths, collection_name, plugin_type, plugin_name, + plugin_data, category=category[:-1]) + plugins_data[category][plugin_type][processed_data['name']] = processed_data + + +def _load_collection_plugins_2_13(plugins_data: Dict[str, Any], + paths: PathsConfig, + collection_details: CollectionDetails) -> None: + collection_name = '{}.{}'.format( + collection_details.get_namespace(), collection_details.get_name()) + + with CollectionCopier( + paths, collection_details.get_namespace(), collection_details.get_name() + ) as (playbook_dir, new_paths): + _load_plugins_2_13(plugins_data, new_paths, collection_name, playbook_dir=playbook_dir) + + def _load_collection_plugins(plugins_data: Dict[str, Any], paths: PathsConfig, collection_details: CollectionDetails, @@ -352,6 +415,23 @@ def _load_ansible_plugins(plugins_data: Dict[str, Any], paths: PathsConfig, paths, None, plugin_type, None, use_ansible_doc=use_ansible_doc) +def _get_ansible_core_version(paths: PathsConfig) -> packaging.version.Version: + try: + version, _ = get_ansible_release() + return packaging.version.Version(version) + except ValueError: + pass + + command = [paths.ansible_doc_path, '--version'] + output = subprocess.check_output(command).decode('utf-8') + for regex in (r'^ansible-doc \[(?:core|base) ([^\]]+)\]', r'^ansible-doc ([^\s]+)'): + match = re.match(regex, output) + if match: + return packaging.version.Version(match.group(1)) + raise Exception( + f'Cannot extract ansible-core version from ansible-doc --version output:\n{output}') + + def _refresh_plugin_cache(paths: PathsConfig, collection_details: CollectionDetails, version: str, @@ -364,7 +444,13 @@ def _refresh_plugin_cache(paths: PathsConfig, 'objects': {}, } - if paths.is_collection: + core_version = _get_ansible_core_version(paths) + if core_version >= packaging.version.Version('2.13.0.dev0'): + if paths.is_collection: + _load_collection_plugins_2_13(plugins_data, paths, collection_details) + else: + _load_plugins_2_13(plugins_data, paths, 'ansible.builtin') + elif paths.is_collection: _load_collection_plugins(plugins_data, paths, collection_details, use_ansible_doc) else: _load_ansible_plugins(plugins_data, paths, use_ansible_doc) diff --git a/tests/functional/fixtures.py b/tests/functional/fixtures.py index 4e44ef65..2838bb30 100644 --- a/tests/functional/fixtures.py +++ b/tests/functional/fixtures.py @@ -28,6 +28,17 @@ # variable to True. Then for all changed files, a colorized diff will be printed. PRINT_DIFFS = False +_ANSIBLE_DOC_VERSION_TEMPLATE = '''ansible-doc [core {ansible_core_version}] + config file = None + configured module search path = ['/home/me/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules'] + ansible python module location = /usr/local/lib/python3.9/dist-packages/ansible + ansible collection location = /home/me/.ansible/collections:/usr/share/ansible/collections + executable location = /usr/local/bin/ansible-doc + python version = 3.9.2 + jinja version = 3.0.3 + libyaml = True +''' + def diff(old: str, new: str) -> str: seqm = difflib.SequenceMatcher(None, old, new) @@ -131,6 +142,8 @@ def __init__(self, base_path: pathlib.Path, paths: PathsConfig): self.created_dirs = set([self.paths.base_dir]) self.created_files = dict() + self.ansible_core_version = '2.12.2' + def _write(self, path: str, data: bytes): with open(path, 'wb') as f: f.write(data) @@ -341,6 +354,10 @@ def _plugin_base(self, plugin_type): def create_fake_subprocess_ansible_doc(self, plugin_data: Dict[str, Dict[str, Any]] ) -> Callable[[List[str]], str]: def fake_subprocess_ansible_doc(command: List[str]) -> str: + if command[0].endswith('ansible-doc') and command[1] == '--version': + return _ANSIBLE_DOC_VERSION_TEMPLATE.format( + ansible_core_version=self.ansible_core_version, + ).encode('utf-8') if command[0].endswith('ansible-doc') and command[1] == '--json' and command[2] == '-t': plugin_type = command[3] args = command[4:] @@ -392,6 +409,10 @@ def create_fake_subprocess_ansible_doc(self, plugin_data: Dict[str, Dict[str, An ) -> Callable[[List[str]], str]: def fake_subprocess_ansible_doc(command: List[str]) -> str: base_dir = self.paths.base_dir + if command[0].endswith('ansible-doc') and command[1] == '--version': + return _ANSIBLE_DOC_VERSION_TEMPLATE.format( + ansible_core_version=self.ansible_core_version, + ).encode('utf-8') if command[0].endswith('ansible-doc') and command[1] == '--json' and command[2] == '-t': plugin_type = command[3] args = command[4:]