WIP: Run CI tests against multiple versions

.github/workflows/ci.yml

@@ -115,96 +115,172 @@ jobs:
           SHELLCHECK_OPTS: --exclude=SC2086 --exclude=SC2046 --exclude=SC2004 --exclude=SC2129
 
 
-  unit-tests:
-    strategy:
-      matrix:
-        python-version:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
-    if: |
-      always() && !cancelled() &&
-      !contains(needs.*.result, 'failure') &&
-      !contains(needs.*.result, 'cancelled') &&
-      needs.files-changed.outputs.python == 'true'
-    needs: ["files-changed", "yaml-lint", "python-lint"]
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - name: "Check out repository code"
-        uses: "actions/checkout@v4"
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: "Setup environment"
-        run: |
-          pipx install poetry==1.8.5
-          poetry config virtualenvs.create true --local
-          pip install invoke toml codecov
-      - name: "Install Package"
-        run: "poetry install --all-extras"
-      - name: "Mypy Tests"
-        run: "poetry run mypy --show-error-codes infrahub_sdk/"
-      # - name: "Pylint Tests"
-      #   run: "poetry run pylint infrahub_sdk/"
-      - name: "Unit Tests"
-        run: "poetry run pytest --cov infrahub_sdk tests/unit/"
-      - name: "Upload coverage to Codecov"
-        run: |
-          codecov --flags python-${{ matrix.python-version }}
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-      # The pytest-cov plugin doesn't quite work with content that is
-      # auto-imported by our own infrahub pytest plugin, this workaround
-      # reports the missing lines
-      - name: "Report coverage for pytest-plugin"
-        if: matrix.python-version == '3.12'
-        run: |
-          source $(poetry env info --path)/bin/activate
-          coverage run --source=infrahub_sdk -m pytest tests/unit/pytest_plugin
-          coverage report -m
-          coverage xml
-          codecov --flags python-filler-${{ matrix.python-version }}
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+#  unit-tests:
+#    strategy:
+#      matrix:
+#        python-version:
+#          - "3.9"
+#          - "3.10"
+#          - "3.11"
+#          - "3.12"
+#          - "3.13"
+#    if: |
+#      always() && !cancelled() &&
+#      !contains(needs.*.result, 'failure') &&
+#      !contains(needs.*.result, 'cancelled') &&
+#      needs.files-changed.outputs.python == 'true'
+#    needs: ["files-changed", "yaml-lint", "python-lint"]
+#    runs-on: ubuntu-latest
+#    timeout-minutes: 30
+#    steps:
+#      - name: "Check out repository code"
+#        uses: "actions/checkout@v4"
+#      - name: Set up Python ${{ matrix.python-version }}
+#        uses: actions/setup-python@v5
+#        with:
+#          python-version: ${{ matrix.python-version }}
+#      - name: "Setup environment"
+#        run: |
+#          pipx install poetry==1.8.5
+#          poetry config virtualenvs.create true --local
+#          pip install invoke toml codecov
+#      - name: "Install Package"
+#        run: "poetry install --all-extras"
+#      - name: "Mypy Tests"
+#        run: "poetry run mypy --show-error-codes infrahub_sdk/"
+#      # - name: "Pylint Tests"
+#      #   run: "poetry run pylint infrahub_sdk/"
+#      - name: "Unit Tests"
+#        run: "poetry run pytest --cov infrahub_sdk tests/unit/"
+#      - name: "Upload coverage to Codecov"
+#        run: |
+#          codecov --flags python-${{ matrix.python-version }}
+#        env:
+#          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+#
+#      # The pytest-cov plugin doesn't quite work with content that is
+#      # auto-imported by our own infrahub pytest plugin, this workaround
+#      # reports the missing lines
+#      - name: "Report coverage for pytest-plugin"
+#        if: matrix.python-version == '3.12'
+#        run: |
+#          source $(poetry env info --path)/bin/activate
+#          coverage run --source=infrahub_sdk -m pytest tests/unit/pytest_plugin
+#          coverage report -m
+#          coverage xml
+#          codecov --flags python-filler-${{ matrix.python-version }}
+#        env:
+#          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   # ------------------------------------------ Integration Tests  ------------------------------------------
-  integration-tests:
+#  integration-tests:
+#    if: |
+#      always() && !cancelled() &&
+#      !contains(needs.*.result, 'failure') &&
+#      !contains(needs.*.result, 'cancelled') &&
+#      needs.files-changed.outputs.python == 'true'
+#    needs: ["files-changed", "yaml-lint", "python-lint"]
+#    runs-on:
+#      group: "huge-runners"
+#    timeout-minutes: 30
+#    strategy:
+#      matrix:
+#        version: ["1.0.0", "1.1.0"]
+#    steps:
+#      - name: "Check out repository code"
+#        uses: "actions/checkout@v4"
+#      - name: Set up Python
+#        uses: actions/setup-python@v5
+#        with:
+#          python-version: "3.12"
+#      - name: "Set environment variables"
+#        run: |
+#          RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/')
+#          echo "PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp" >> $GITHUB_ENV
+#      - name: "Setup environment"
+#        run: |
+#          pipx install poetry==1.8.5
+#          poetry config virtualenvs.create true --local
+#          pip install invoke toml codecov
+#      - name: "Install Package"
+#        run: "poetry install --all-extras"
+#      - name: "Set environment variables for python_testcontainers"
+#        run: |
+#          echo INFRAHUB_TESTING_IMAGE_VER=${{ matrix.version }} >> $GITHUB_ENV
+#      - name: "Integration Tests"
+#        run: |
+#          echo "Running tests for version: ${{ matrix.version }}"
+#          poetry run pytest --cov infrahub_sdk tests/integration/
+#      - name: "Upload coverage to Codecov"
+#        run: |
+#          codecov --flags integration-tests
+#        env:
+#          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+  integration-tests-against-local-infrahub-build:
     if: |
       always() && !cancelled() &&
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled') &&
       needs.files-changed.outputs.python == 'true'
-    needs: ["files-changed", "yaml-lint", "python-lint"]
+    needs: [ "files-changed", "yaml-lint", "python-lint" ]
     runs-on:
       group: "huge-runners"
     timeout-minutes: 30
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
+
+      - name: "Checkout infrahub repository"
+        uses: "actions/checkout@v4"
+        with:
+          repository: "opsmill/infrahub"
+          path: "infrahub-server"
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - name: "Set environment variables"
+
+      - name: "Setup git credentials prior dev.build"
         run: |
+          cd infrahub-server
+          git config --global user.name 'Infrahub'
+          git config --global user.email '[email protected]'
+          git config --global --add safe.directory '*'
+          git config --global credential.usehttppath true
+          git config --global credential.helper /usr/local/bin/infrahub-git-credential
+
+      - name: "Set environment variables prior dev.build"
+        run: |
+          echo "INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }}" >> $GITHUB_ENV
           RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/')
           echo "PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp" >> $GITHUB_ENV
+          echo "INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "INFRAHUB_TESTING_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "INFRAHUB_TESTING_DOCKER_IMAGE=opsmill/infrahub" >> $GITHUB_ENV
+
+      - name: "Build container"
+        run: |
+          cd infrahub-server
+          inv dev.build
+
       - name: "Setup environment"
         run: |
           pipx install poetry==1.8.5
           poetry config virtualenvs.create true --local
           pip install invoke toml codecov
+
       - name: "Install Package"
         run: "poetry install --all-extras"
+
       - name: "Integration Tests"
-        run: "poetry run pytest --cov infrahub_sdk tests/integration/"
+        run: |
+          echo "Running tests for version: $INFRAHUB_TESTING_DOCKER_IMAGE"
+          poetry run pytest --cov infrahub_sdk tests/integration/
+
       - name: "Upload coverage to Codecov"
         run: |
           codecov --flags integration-tests
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+

infrahub_sdk/testing/docker.py

@@ -1,10 +1,42 @@
+from __future__ import annotations
+
+import os
+
 import pytest
 from infrahub_testcontainers.helpers import TestInfrahubDocker
+from packaging.version import InvalidVersion, Version
 
 from .. import Config, InfrahubClient, InfrahubClientSync
 
+DEFAULT_INFRAHUB_VERSION = "todo"
+INFRAHUB_VERSION = os.getenv("INFRAHUB_TESTING_IMAGE_TAG", DEFAULT_INFRAHUB_VERSION)
+
+
+# TODO test it works
+def skip_version(min_infrahub_version: str | None = None, max_infrahub_version: str | None = None) -> bool:
+    """
+    Check if a test should be skipped depending on infrahub version.
+    """
+    try:
+        version = Version(INFRAHUB_VERSION)
+    except InvalidVersion:
+        # We would typically end up here for development purpose while running a CI test against
+        # unreleased versions of infrahub, like `stable` or `develop` branch.
+        # For now, we consider this means we are testing against the most recent version of infrahub,
+        # so we skip if the test should not be ran against a maximum version.
+        return max_infrahub_version is None
+
+    if min_infrahub_version is not None and version <= Version(min_infrahub_version):
+        return True
+
+    return max_infrahub_version is not None and version <= Version(max_infrahub_version)
+
 
 class TestInfrahubDockerClient(TestInfrahubDocker):
+    @pytest.fixture(scope="class")
+    def infrahub_version(self) -> str:
+        return INFRAHUB_VERSION
+
     @pytest.fixture(scope="class")
     def client(self, infrahub_port: int) -> InfrahubClient:
         return InfrahubClient(

tests/integration/test_infrahub_client.py

@@ -16,10 +16,6 @@
 
 
 class TestInfrahubNode(TestInfrahubDockerClient, SchemaAnimal):
-    @pytest.fixture(scope="class")
-    def infrahub_version(self) -> str:
-        return "1.1.0"
-
     @pytest.fixture(scope="class")
     async def base_dataset(
         self,

tests/integration/test_node.py

@@ -11,10 +11,6 @@
 
 
 class TestInfrahubNode(TestInfrahubDockerClient, SchemaCarPerson):
-    @pytest.fixture(scope="class")
-    def infrahub_version(self) -> str:
-        return "1.1.0"
-
     @pytest.fixture(scope="class")
     async def initial_schema(self, default_branch: str, client: InfrahubClient, schema_base: SchemaRoot) -> None:
         await client.schema.wait_until_converged(branch=default_branch)

tests/integration/test_repository.py

@@ -2,8 +2,6 @@
 
 from typing import TYPE_CHECKING
 
-import pytest
-
 from infrahub_sdk.testing.docker import TestInfrahubDockerClient
 from infrahub_sdk.testing.repository import GitRepo
 from infrahub_sdk.utils import get_fixtures_dir
@@ -13,10 +11,6 @@
 
 
 class TestInfrahubRepository(TestInfrahubDockerClient):
-    @pytest.fixture(scope="class")
-    def infrahub_version(self) -> str:
-        return "1.1.0"
-
     async def test_add_repository(self, client: InfrahubClient, remote_repos_dir):
         src_directory = get_fixtures_dir() / "integration/mock_repo"
         repo = GitRepo(name="mock_repo", src_directory=src_directory, dst_directory=remote_repos_dir)