From b61ce09820abc5638c238be553760861fbc403ce Mon Sep 17 00:00:00 2001 From: Adrian Clay Lake Date: Wed, 11 Dec 2024 16:46:19 +0100 Subject: [PATCH] fix: release identification in prepare_single_image_build_matrix.py (#308) * fix(src/image/prepare_single_image_build_matrix.py): release identification * refactor: remove extra print * ci: automatically update oci/mock-rock/_releases.json, from https://github.com/canonical/oci-factory/actions/runs/12259827303 * trigger_tessts * trigger_tessts * trigger_tessts * ci: automatically update oci/mock-rock/_releases.json, from https://github.com/canonical/oci-factory/actions/runs/12260216075 * fix: permit optional track release field * ci: automatically update oci/mock-rock/_releases.json, from https://github.com/canonical/oci-factory/actions/runs/12273962224 * feat: add tests for checking the number of release * ci: automatically update oci/mock-rock/_releases.json, from https://github.com/canonical/oci-factory/actions/runs/12279591872 --------- Co-authored-by: clay-lake --- .github/workflows/_Test-OCI-Factory.yaml | 1 - oci/mock-rock/_releases.json | 14 +-- .../prepare_single_image_build_matrix.py | 34 +++++--- tests/data/image_all_eol_tracks.yaml | 37 ++++++++ .../image_all_eol_tracks_with_release.yaml | 45 ++++++++++ tests/data/image_no_track_releases.yaml | 14 +++ tests/data/image_single_track_release.yaml | 19 ++++ tests/data/image_with_release.yaml | 45 ++++++++++ tests/data/image_without_release.yaml | 37 ++++++++ .../test_prepare_single_image_build_matrix.py | 86 +++++++++++++++++++ 10 files changed, 314 insertions(+), 18 deletions(-) create mode 100644 tests/data/image_all_eol_tracks.yaml create mode 100644 tests/data/image_all_eol_tracks_with_release.yaml create mode 100644 tests/data/image_no_track_releases.yaml create mode 100644 tests/data/image_single_track_release.yaml create mode 100644 tests/data/image_with_release.yaml create mode 100644 tests/data/image_without_release.yaml create mode 100644 tests/integration/test_prepare_single_image_build_matrix.py diff --git a/.github/workflows/_Test-OCI-Factory.yaml b/.github/workflows/_Test-OCI-Factory.yaml index 001e5a5c..a54122c8 100644 --- a/.github/workflows/_Test-OCI-Factory.yaml +++ b/.github/workflows/_Test-OCI-Factory.yaml @@ -117,7 +117,6 @@ jobs: # test-oci-compliance: true # test-vulnerabilities: true - # # Test workflow used in continuous testing # test-vulnerability-scan: # name: Test vulnerability scan workflow diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json index 62d7445f..97a4ddda 100644 --- a/oci/mock-rock/_releases.json +++ b/oci/mock-rock/_releases.json @@ -35,31 +35,31 @@ "1.1-22.04": { "end-of-life": "2030-05-01T00:00:00Z", "candidate": { - "target": "966" + "target": "984" }, "beta": { - "target": "966" + "target": "984" }, "edge": { - "target": "966" + "target": "984" } }, "1-22.04": { "end-of-life": "2030-05-01T00:00:00Z", "candidate": { - "target": "966" + "target": "984" }, "beta": { - "target": "966" + "target": "984" }, "edge": { - "target": "966" + "target": "984" } }, "1.2-22.04": { "end-of-life": "2030-05-01T00:00:00Z", "beta": { - "target": "967" + "target": "985" }, "edge": { "target": "1.2-22.04_beta" diff --git a/src/image/prepare_single_image_build_matrix.py b/src/image/prepare_single_image_build_matrix.py index 52432701..c144b205 100755 --- a/src/image/prepare_single_image_build_matrix.py +++ b/src/image/prepare_single_image_build_matrix.py @@ -107,11 +107,20 @@ def filter_eol_tracks(build: dict[str, Any]) -> dict[str, Any]: def filter_eol_builds(builds: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Remove any builds with no tracks specified.""" - # remove any end of life tracks - builds = [filter_eol_tracks(build) for build in builds] + """Remove any builds with eol tracks.""" - return [build for build in builds if len(build["release"])] + # if no release exists and therefore no eol is specified, do nothing + non_release_builds = [build for build in builds if "release" not in build] + + # if we have release info, then filter based on eol + release_builds = [ + filtered_build + for build in builds + if "release" in build + and (filtered_build := filter_eol_tracks(build))["release"] + ] + + return non_release_builds + release_builds def write_revision_data(data_dir: Path, build: dict[str, Any]): @@ -157,7 +166,7 @@ def write_github_output( """Write script result to GITHUB_OUTPUT.""" outputs = { "build-matrix": {"include": builds}, - "release-to": release_to, + "release-to": "true" if release_to else "", "revision-data-dir": str(revision_data_dir), } with GithubOutput() as github_output: @@ -221,16 +230,21 @@ def main(): f"Generating matrix for following builds: \n {json.dumps(builds, indent=4)}" ) + # trigger a release if specified in the image_trigger root + release = "release" in image_trigger + for build in builds: write_revision_data(args.revision_data_dir, build) - # the workflow GH matrix has a problem parsing nested JSON dicts - # so let's remove this field since we don't need it for the builds - del build["release"] + if "release" in build: + # trigger a release if specified in any of the builds + release = True - release_to = "true" if "release" in image_trigger else "" + # the workflow GH matrix has a problem parsing nested JSON dicts + # so let's remove this field since we don't need it for the builds themselves + del build["release"] - write_github_output(release_to, builds, args.revision_data_dir) + write_github_output(release, builds, args.revision_data_dir) if __name__ == "__main__": diff --git a/tests/data/image_all_eol_tracks.yaml b/tests/data/image_all_eol_tracks.yaml new file mode 100644 index 00000000..2a0550b3 --- /dev/null +++ b/tests/data/image_all_eol_tracks.yaml @@ -0,0 +1,37 @@ +version: 1 + +upload: + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.0 + release: + 1.0-22.04: + end-of-life: "2020-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.1 + release: + 1.1-22.04: + end-of-life: "2000-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + 1-22.04: + end-of-life: "2000-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.2 + release: + 1.2-22.04: + end-of-life: "2000-05-01T00:00:00Z" + risks: + - beta diff --git a/tests/data/image_all_eol_tracks_with_release.yaml b/tests/data/image_all_eol_tracks_with_release.yaml new file mode 100644 index 00000000..b92f7b27 --- /dev/null +++ b/tests/data/image_all_eol_tracks_with_release.yaml @@ -0,0 +1,45 @@ +version: 1 + +release: + latest: + end-of-life: "2030-05-01T00:00:00Z" + candidate: 1.2-22.04_beta + test: + end-of-life: "2030-05-01T00:00:00Z" + beta: 1.1-22.04_beta + +upload: + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.0 + release: + 1.0-22.04: + end-of-life: "2020-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.1 + release: + 1.1-22.04: + end-of-life: "2000-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + 1-22.04: + end-of-life: "2000-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.2 + release: + 1.2-22.04: + end-of-life: "2000-05-01T00:00:00Z" + risks: + - beta diff --git a/tests/data/image_no_track_releases.yaml b/tests/data/image_no_track_releases.yaml new file mode 100644 index 00000000..941bcd17 --- /dev/null +++ b/tests/data/image_no_track_releases.yaml @@ -0,0 +1,14 @@ +version: 1 + +upload: + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.0 + + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.1 + + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.2 diff --git a/tests/data/image_single_track_release.yaml b/tests/data/image_single_track_release.yaml new file mode 100644 index 00000000..47efa36b --- /dev/null +++ b/tests/data/image_single_track_release.yaml @@ -0,0 +1,19 @@ +version: 1 + +upload: + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.0 + + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.1 + + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.2 + release: + 1.2-22.04: + end-of-life: "2030-05-01T00:00:00Z" + risks: + - beta diff --git a/tests/data/image_with_release.yaml b/tests/data/image_with_release.yaml new file mode 100644 index 00000000..72382b32 --- /dev/null +++ b/tests/data/image_with_release.yaml @@ -0,0 +1,45 @@ +version: 1 + +release: + latest: + end-of-life: "2030-05-01T00:00:00Z" + candidate: 1.2-22.04_beta + test: + end-of-life: "2030-05-01T00:00:00Z" + beta: 1.1-22.04_beta + +upload: + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.0 + release: + 1.0-22.04: + end-of-life: "2024-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.1 + release: + 1.1-22.04: + end-of-life: "2030-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + 1-22.04: + end-of-life: "2030-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.2 + release: + 1.2-22.04: + end-of-life: "2030-05-01T00:00:00Z" + risks: + - beta diff --git a/tests/data/image_without_release.yaml b/tests/data/image_without_release.yaml new file mode 100644 index 00000000..488174c5 --- /dev/null +++ b/tests/data/image_without_release.yaml @@ -0,0 +1,37 @@ +version: 1 + +upload: + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.0 + release: + 1.0-22.04: + end-of-life: "2024-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.1 + release: + 1.1-22.04: + end-of-life: "2030-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + 1-22.04: + end-of-life: "2030-05-01T00:00:00Z" + risks: + - candidate + - edge + - beta + - source: "canonical/rocks-toolbox" + commit: 17916dd5de270e61a6a3fd3f4661a6413a50fd6f + directory: mock_rock/1.2 + release: + 1.2-22.04: + end-of-life: "2030-05-01T00:00:00Z" + risks: + - beta diff --git a/tests/integration/test_prepare_single_image_build_matrix.py b/tests/integration/test_prepare_single_image_build_matrix.py new file mode 100644 index 00000000..95082977 --- /dev/null +++ b/tests/integration/test_prepare_single_image_build_matrix.py @@ -0,0 +1,86 @@ +# from pathlib import Path +# import pytest + +import json +import re +import shutil +import sys +from pathlib import Path + +import pytest + +from src.image.prepare_single_image_build_matrix import \ + main as prepare_build_matrix + +from .. import DATA_DIR + + +@pytest.fixture +def prep_execution(tmpdir, monkeypatch, request): + + image_trigger_sample = getattr(request, "param", None) + + # configure files/env requried for the test + github_output = tmpdir / "github_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(github_output)) + + revision_data_dir = tmpdir / "revision-data" + revision_data_dir.mkdir() + + oci_trigger_dir = tmpdir / "image_trigger" + oci_trigger_dir.mkdir() + shutil.copy( + image_trigger_sample, + oci_trigger_dir / "image.yaml", + ) + + # patch the arv for the test. script.py can be anything + args = ( + f"--oci-path {oci_trigger_dir} --revision-data-dir {revision_data_dir}".split( + " " + ) + ) + monkeypatch.setattr(sys, "argv", ["script.py"] + args) + + return revision_data_dir, github_output + + +@pytest.mark.parametrize( + "prep_execution, expected_release_to, expected_release_count", + [ + (DATA_DIR / "image_all_eol_tracks_with_release.yaml", True, 0), + (DATA_DIR / "image_all_eol_tracks.yaml", False, 0), + (DATA_DIR / "image_no_track_releases.yaml", False, 0), + (DATA_DIR / "image_single_track_release.yaml", True, 1), + (DATA_DIR / "image_with_release.yaml", True, 3), + (DATA_DIR / "image_without_release.yaml", True, 3), + ], + indirect=["prep_execution"], +) +def test_release_to(prep_execution, expected_release_to, expected_release_count): + """Test state of release-to in github output after running prepare_single_image_build_matrix""" + revision_data_dir, github_output = prep_execution + + # run main from prepare_single_image_build_matrix + prepare_build_matrix() + + github_output_content = github_output.read_text("utf8") + + assert re.search( + f'^release-to={"true" if expected_release_to else ""}$', + github_output_content, + re.M, + ), "Invalid release-to value" + + revision_files = Path(revision_data_dir).glob("*") + + release_count = 0 + + for file in revision_files: + revision_data = json.loads(file.read_text()) + if release_list := revision_data.get("release"): + release_count += len(release_list) + + assert ( + expected_release_count == release_count + ), "Invalid number of builds to release"