{amazon,community}.aws: cross-project testing

If there is a change in module_utils, run the CI only for the modules
affected by that change.
This also solves the case where in the same PR there is a change in a
module_utils file (e.g., rds) and a change on a module (e.g., ec2_vpc_igw),
returning the 'target_to_test' containing ec2_vpc_igw and all modules
affected by the change in module_utils. At the moment, 'target_to_test'
is returned with only ec2_vpc_igw.

Co-Authored-by: Alina Buzachis <[email protected]>

roles/ansible-test-splitter/defaults/main.yaml

@@ -1,3 +1,6 @@
 ---
 ansible_test_splitter__test_changed: false
 ansible_test_splitter__children_prefix: please_adjust_this
+# Git repositories of changed collections, only when
+# ansible_test_splitter__test_changed is true
+ansible_test_splitter__check_for_changes_in: []

roles/ansible-test-splitter/files/list_changed_targets.py

@@ -1,45 +1,247 @@
 #!/usr/bin/env python3
 
+import argparse
+import ast
+import json
 from pathlib import PosixPath
 import sys
 import subprocess
+import yaml
+
+
+parser = argparse.ArgumentParser(
+    description="Evaluate which targets need to be tested."
+)
+parser.add_argument(
+    "--branch", type=str, default="main", help="the default branch to test against"
+)
+
+parser.add_argument(
+    "--test-all-the-targets",
+    dest="test_all_the_targets",
+    action="store_true",
+    default=False,
+    help="list all the target available in the the collection",
+)
+
+parser.add_argument(
+    "--test-changed",
+    dest="test_changed",
+    action="store_true",
+    default=False,
+    help=("only test the targets impacted by the changes"),
+)
+
+
+parser.add_argument(
+    "collection_to_tests",
+    default=[],
+    nargs="+",
+    type=PosixPath,
+    help="the location of the collections to test. e.g: ~/.ansible/collections/ansible_collections/amazon/aws",
+)
 
 targets_to_test = []
 targets_dir = PosixPath("tests/integration/targets")
-zuul_branch = sys.argv[1]
-diff = subprocess.check_output(
-    ["git", "diff", f"origin/{zuul_branch}", "--name-only"]
-).decode()
-module_files = [PosixPath(d) for d in diff.split("\n") if d.startswith("plugins/")]
-for i in module_files:
-    if not i.is_file():
-        continue
-    target_name = i.stem
-
-    for t in targets_dir.iterdir():
-        aliases = t / "aliases"
-        if not aliases.is_file():
-            continue
-        # There is a target with the module name, let's take that
-        if t.name == target_name:
-            targets_to_test.append(target_name)
-            break
-        alias_content = aliases.read_text().split("\n")
-        # The target name is in the aliases file
-        if target_name in alias_content:
-            targets_to_test.append(target_name)
-            break
-
-target_files = [
-    PosixPath(d) for d in diff.split("\n") if d.startswith("tests/integration/targets/")
-]
-for i in target_files:
-    splitted = str(i).split("/")
-    if len(splitted) < 5:
-        continue
-    target_name = splitted[3]
-    aliases = targets_dir / target_name / "aliases"
-    if aliases.is_file():
-        targets_to_test.append(target_name)
-
-print(" ".join(list(set(targets_to_test))))
+
+
+def parse_args(raw_args):
+    return parser.parse_args(raw_args)
+
+
+def read_collection_name(path):
+    with (path / "galaxy.yml").open() as fd:
+        content = yaml.safe_load(fd)
+        return f'{content["namespace"]}.{content["name"]}'
+
+
+def list_pyimport(collection_name, module_content):
+    root = ast.parse(module_content)
+    for node in ast.iter_child_nodes(root):
+        if isinstance(node, ast.Import):
+            yield node.names[0].name
+        elif isinstance(node, ast.ImportFrom):
+            module = node.module.split(".")
+            prefix = (
+                f"ansible_collections.{collection_name}.plugins."
+                if node.level == 2
+                else ""
+            )
+            yield f"{prefix}{'.'.join(module)}"
+
+
+class WhatHaveChanged:
+    def __init__(self, path, branch):
+        assert isinstance(path, PosixPath)
+        self.collection_path = path
+        self.branch = branch
+        self.collection_name = lambda: read_collection_name(path)
+
+    def changed_files(self):
+        """List of changed files
+
+        Returns a list of pathlib.PosixPath
+        """
+        return [
+            PosixPath(p)
+            for p in (
+                subprocess.check_output(
+                    [
+                        "git",
+                        "diff",
+                        f"origin/{self.branch}",
+                        "--name-only",
+                    ],
+                    cwd=self.collection_path,
+                )
+                .decode()
+                .split("\n")
+            )
+        ]
+
+    def modules(self):
+        """List the modules impacted by the change"""
+        for d in self.changed_files():
+            if str(d).startswith("plugins/modules/"):
+                yield PosixPath(d)
+
+    def module_utils(self):
+        """List the Python modules impacted by the change"""
+        for d in self.changed_files():
+            if str(d).startswith("plugins/module_utils/"):
+                yield f"ansible_collections.{self.collection_name()}.plugins.module_utils.{d.stem}"
+
+
+class Target:
+    def __init__(self, path):
+        self.path = path
+        self.lines = [l.split("#")[0] for l in path.read_text().split("\n") if l]
+        self.name = path.parent.name
+
+    def is_alias_of(self, name):
+        return name in self.lines or self.name == name
+
+    def is_unstable(self):
+        if "unstable" in self.lines:
+            return True
+        return False
+
+    def is_disabled(self):
+        if "disabled" in self.lines:
+            return True
+        return False
+
+    def is_slow(self):
+        if "slow" in self.lines or "# reason: slow" in self.lines:
+            return True
+        return False
+
+    def is_ignored(self):
+        """Show the target be ignored by default?"""
+        ignore = set(["unsupported", "disabled", "unstable", "hidden"])
+        return not ignore.isdisjoint(set(self.lines))
+
+
+class WhatToTest:
+    def __init__(self, path):
+        self.collection_path = path
+        self._my_test_plan = []
+        self.collection_name = lambda: read_collection_name(path)
+
+    def _targets(self):
+        for a in self.collection_path.glob("tests/integration/targets/*/aliases"):
+            yield Target(a)
+
+    def cover_all(self):
+        """Cover all the targets available."""
+        for t in self._targets():
+            if t.is_ignored():
+                continue
+            self._my_test_plan.append(t)
+
+    def cover_module(self, path):
+        """Track the targets to run follow up to a modules changed."""
+        for t in self._targets():
+            if t.is_alias_of(path.stem):
+                self._my_test_plan.append(t)
+
+    def cover_module_utils(self, pymod):
+        """Track the targets to run follow up to a module_utils changed."""
+        for m in self.collection_path.glob("plugins/modules/*"):
+            for i in list_pyimport(self.collection_name(), m.read_text()):
+                if pymod == i:
+                    self.cover_module(m)
+
+    def slow_targets_to_test(self):
+        return list(set([a.name for a in self._my_test_plan if a.is_slow()]))
+
+    def regular_targets_to_test(self):
+        return list(set([a.name for a in self._my_test_plan if not a.is_slow()]))
+
+
+class ElGrandeSeparator:
+    def __init__(self, collections):
+        self.collections = collections
+        self.total_jobs = 10
+
+    def output(self):
+        batches = []
+        for c in collections:
+            slots = [
+                f"integration-{c.collection_name()}-{i+1}"
+                for i in range(self.total_jobs)
+            ]
+            slow_targets = c.slow_targets_to_test()
+            regular_targets = c.regular_targets_to_test()
+            for b in self.build_up_batches(slow_targets, regular_targets, slots):
+                batches.append(b)
+        result = self.build_result_struct(batches)
+        print(json.dumps(result))
+
+    def build_up_batches(self, slow_targets, regular_targets, slots):
+        jobs_per_slot = 16
+        my_slot_available = [s for s in slots]
+        for i in slow_targets:
+            my_slot = my_slot_available.pop(0)
+            yield (my_slot, [i])
+
+        while regular_targets:
+            my_slot = my_slot_available.pop(0)
+            yield (my_slot, regular_targets[0:jobs_per_slot])
+            for _ in range(jobs_per_slot):
+                if regular_targets:
+                    regular_targets.pop(0)
+
+    def build_result_struct(self, batches):
+        result = {
+            "data": {
+                "zuul": {"child_jobs": []},
+                "child": {"targets_to_test": {}},
+            }
+        }
+
+        for job, targets in batches:
+            result["data"]["zuul"]["child_jobs"].append(job)
+            result["data"]["child"]["targets_to_test"][job] = " ".join(targets)
+        return result
+
+
+if __name__ == "__main__":
+    args = parse_args(sys.argv[1:])
+
+    collections = [WhatToTest(i) for i in args.collection_to_tests]
+
+    if args.test_all_the_targets:
+        for c in collections:
+            c.cover_all()
+    else:
+        for whc in [WhatHaveChanged(i, args.branch) for i in args.collection_to_tests]:
+            for path in whc.modules():
+                for c in collections:
+                    c.cover_module(path)
+            for path in whc.module_utils():
+                for c in collections:
+                    c.cover_module_utils(path)
+
+    egs = ElGrandeSeparator(collections)
+    egs.output()

roles/ansible-test-splitter/files/test_list_changed_targets.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import io
+from pathlib import PosixPath
+from unittest.mock import MagicMock
+from list_changed_targets import (
+    list_pyimport,
+    read_collection_name,
+    WhatHaveChanged,
+    WhatToTest,
+    parse_args,
+)
+
+my_module = """
+from ..module_utils.core import AnsibleAWSModule
+from ipaddress import ipaddress
+import time
+import botocore.exceptions
+"""
+
+
+def test_read_collection_name():
+    m_galaxy_file = MagicMock()
+    m_galaxy_file.open = lambda: io.BytesIO(b"name: b\nnamespace: a\n")
+    m_path = MagicMock()
+    m_path.__truediv__.return_value = m_galaxy_file
+    assert read_collection_name(m_path) == "a.b"
+
+
+def test_list_pyimport():
+    assert list(list_pyimport("amazon.aws", my_module)) == [
+        "ansible_collections.amazon.aws.plugins.module_utils.core",
+        "ipaddress",
+        "time",
+        "botocore.exceptions",
+    ]
+
+
+def test_what_changed_files():
+    whc = WhatHaveChanged("a", "b")
+    whc.collection_name = lambda: "a.b"
+    whc.changed_files = lambda: [
+        PosixPath("tests/something"),
+        PosixPath("plugins/module_utils/core.py"),
+        PosixPath("plugins/modules/ec2.py"),
+    ]
+    assert list(whc.modules()) == [PosixPath("plugins/modules/ec2.py")]
+    assert list(whc.module_utils()) == [
+        "ansible_collections.a.b.plugins.module_utils.core"
+    ]
+
+
+def test_aliases():
+    def build_alias(name, text):
+        m_alias_file = MagicMock()
+        m_alias_file.read_text.return_value = text
+        m_alias_file.parent.name = name
+        return m_alias_file
+
+    wtt = WhatToTest(PosixPath("nowhere"))
+    m_c_path = MagicMock()
+    wtt.collection_path = m_c_path
+
+    m_c_path.glob.return_value = []
+    assert list(wtt.aliases()) == []
+
+    m_c_path.glob.return_value = [build_alias("a", "ec2\n")]
+    assert list(wtt.aliases()) == [["a", "ec2"]]
+
+    m_c_path.glob.return_value = [build_alias("a", "#ec2\n")]
+    assert list(wtt.aliases()) == [["a"]]
+
+    m_c_path.glob.return_value = [build_alias("a", "disabled\n")]
+    assert list(wtt.aliases()) == []
+
+
+def test_argparse():
+    args = parse_args(["a"])
+    assert args.changed_collections == []
+
+    args = parse_args(
+        "col --changed-collection somewhere --changed-collection somewhere-else".split(
+            " "
+        )
+    )
+    assert args.changed_collections == [
+        PosixPath("somewhere"),
+        PosixPath("somewhere-else"),
+    ]