From 680f491b443e789614a7c36bcb13ea3ebcb25b68 Mon Sep 17 00:00:00 2001 From: Patrick Ogenstad Date: Thu, 9 Jan 2025 12:48:47 +0100 Subject: [PATCH 01/14] Avoid providing a falsy fallback to `dict.get()` --- infrahub_sdk/analyzer.py | 2 +- infrahub_sdk/node.py | 4 ++-- pyproject.toml | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/infrahub_sdk/analyzer.py b/infrahub_sdk/analyzer.py index f2582fb..5bca02e 100644 --- a/infrahub_sdk/analyzer.py +++ b/infrahub_sdk/analyzer.py @@ -91,7 +91,7 @@ def variables(self) -> list[GraphQLQueryVariable]: else: data["default_value"] = variable.default_value.value - if not data.get("default_value", None) and non_null: + if not data.get("default_value") and non_null: data["required"] = True response.append(GraphQLQueryVariable(**data)) diff --git a/infrahub_sdk/node.py b/infrahub_sdk/node.py index 3d84fb9..7a929d3 100644 --- a/infrahub_sdk/node.py +++ b/infrahub_sdk/node.py @@ -1074,7 +1074,7 @@ async def from_graphql( timeout: int | None = None, ) -> Self: if not schema: - node_kind = data.get("__typename", None) or data.get("node", {}).get("__typename", None) + node_kind = data.get("__typename") or data.get("node", {}).get("__typename", None) if not node_kind: raise ValueError("Unable to determine the type of the node, __typename not present in data") schema = await client.schema.get(kind=node_kind, branch=branch, timeout=timeout) @@ -1594,7 +1594,7 @@ def from_graphql( timeout: int | None = None, ) -> Self: if not schema: - node_kind = data.get("__typename", None) or data.get("node", {}).get("__typename", None) + node_kind = data.get("__typename") or data.get("node", {}).get("__typename", None) if not node_kind: raise ValueError("Unable to determine the type of the node, __typename not present in data") schema = client.schema.get(kind=node_kind, branch=branch, timeout=timeout) diff --git a/pyproject.toml b/pyproject.toml index f89684d..05566a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -273,7 +273,6 @@ ignore = [ "RUF005", # Consider `[*path, str(key)]` instead of concatenation "RUF015", # Prefer `next(iter(input_data["variables"].keys()))` over single element slice "RUF029", # Function is declared `async`, but doesn't `await` or use `async` features. - "RUF056", # [*] Avoid providing a falsy fallback to `dict.get()` in boolean test positions. The default fallback `None` is already falsy. "S108", # Probable insecure usage of temporary file or directory "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes "S701", # By default, jinja2 sets `autoescape` to `False`. Consider using `autoescape=True` @@ -284,7 +283,6 @@ ignore = [ "SIM114", # Combine `if` branches using logical `or` operator "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements "SIM118", # Use `key in dict` instead of `key in dict.keys) - "SIM910", # Use `data.get(key)` instead of `data.get(key, None)` "TC003", # Move standard library import `collections.abc.Iterable` into a type-checking block "UP031", # Use format specifiers instead of percent format ] From 67072ef3270634168cfa195f6ef852561729a6cc Mon Sep 17 00:00:00 2001 From: wvandeun Date: Fri, 10 Jan 2025 16:30:37 +0100 Subject: [PATCH 02/14] add filters to SDK client count method --- changelog/count-method-filters.changed.md | 1 + infrahub_sdk/client.py | 14 ++++++++++++-- tests/unit/sdk/test_client.py | 10 ++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 changelog/count-method-filters.changed.md diff --git a/changelog/count-method-filters.changed.md b/changelog/count-method-filters.changed.md new file mode 100644 index 0000000..13a15c5 --- /dev/null +++ b/changelog/count-method-filters.changed.md @@ -0,0 +1 @@ +Added possibility to use filters for the SDK client's count method diff --git a/infrahub_sdk/client.py b/infrahub_sdk/client.py index 839f1ec..d90b1b3 100644 --- a/infrahub_sdk/client.py +++ b/infrahub_sdk/client.py @@ -547,8 +547,10 @@ async def count( at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, + **kwargs: Any, ) -> int: """Return the number of nodes of a given kind.""" + filters = kwargs schema = await self.schema.get(kind=kind, branch=branch) branch = branch or self.default_branch @@ -556,7 +558,10 @@ async def count( at = Timestamp(at) response = await self.execute_graphql( - query=Query(query={schema.kind: {"count": None}}).render(), branch_name=branch, at=at, timeout=timeout + query=Query(query={schema.kind: {"count": None, "@filters": filters}}).render(), + branch_name=branch, + at=at, + timeout=timeout, ) return int(response.get(schema.kind, {}).get("count", 0)) @@ -1651,8 +1656,10 @@ def count( at: Timestamp | None = None, branch: str | None = None, timeout: int | None = None, + **kwargs: Any, ) -> int: """Return the number of nodes of a given kind.""" + filters = kwargs schema = self.schema.get(kind=kind, branch=branch) branch = branch or self.default_branch @@ -1660,7 +1667,10 @@ def count( at = Timestamp(at) response = self.execute_graphql( - query=Query(query={schema.kind: {"count": None}}).render(), branch_name=branch, at=at, timeout=timeout + query=Query(query={schema.kind: {"count": None, "@filters": filters}}).render(), + branch_name=branch, + at=at, + timeout=timeout, ) return int(response.get(schema.kind, {}).get("count", 0)) diff --git a/tests/unit/sdk/test_client.py b/tests/unit/sdk/test_client.py index 3f08780..f8624c1 100644 --- a/tests/unit/sdk/test_client.py +++ b/tests/unit/sdk/test_client.py @@ -83,6 +83,16 @@ async def test_method_count(clients, mock_query_repository_count, client_type): assert count == 5 +@pytest.mark.parametrize("client_type", client_types) +async def test_method_count_with_filter(clients, mock_query_repository_count, client_type): # pylint: disable=unused-argument + if client_type == "standard": + count = await clients.standard.count(kind="CoreRepository", name__value="test") + else: + count = clients.sync.count(kind="CoreRepository", name__value="test") + + assert count == 5 + + @pytest.mark.parametrize("client_type", client_types) async def test_method_get_version(clients, mock_query_infrahub_version, client_type): # pylint: disable=unused-argument if client_type == "standard": From 922ed7f660f88e07440facb3642f03a7954b96a3 Mon Sep 17 00:00:00 2001 From: wvandeun Date: Mon, 13 Jan 2025 00:55:08 +0100 Subject: [PATCH 03/14] add integration test --- tests/integration/test_infrahub_client.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/test_infrahub_client.py b/tests/integration/test_infrahub_client.py index c18bc5f..3c60751 100644 --- a/tests/integration/test_infrahub_client.py +++ b/tests/integration/test_infrahub_client.py @@ -142,6 +142,14 @@ async def test_create_branch_async(self, client: InfrahubClient, base_dataset): task_id = await client.branch.create(branch_name="new-branch-2", wait_until_completion=False) assert isinstance(task_id, str) + async def test_count(self, client: InfrahubClient, base_dataset): + count = await client.count(kind=TESTING_PERSON) + assert count == 3 + + async def test_count_with_filter(self, client: InfrahubClient, base_dataset): + count = await client.count(kind=TESTING_PERSON, name__values=["Liam Walker", "Ethan Carter"]) + assert count == 2 + # async def test_get_generic_filter_source(self, client: InfrahubClient, base_dataset): # admin = await client.get(kind="CoreAccount", name__value="admin") From 5d2857684850bdb8bdf69cb76487e85966da4ada Mon Sep 17 00:00:00 2001 From: Alex Gittings Date: Mon, 13 Jan 2025 10:27:36 +0000 Subject: [PATCH 04/14] Resolves #130: Replace GitPython with dulwich (#218) * resolves #130: Replace GitPython with dulwich * Remove debugging * Add better testing for initializing git repos * Refactor RepoManager * Add integration test for dulwich * Add minimum version for dulwich * Fix linting for exception match * Add debug for ci * Move asserts so it fails early * Add debugging for integration test * Return docker compose config in CI * Add more debugs for files * Run command on docker host * Get container logs * Comment out integration test for git repo * Remove print --- changelog/130.added.md | 1 + infrahub_sdk/checks.py | 8 +- infrahub_sdk/generator.py | 6 +- infrahub_sdk/pytest_plugin/items/base.py | 5 - infrahub_sdk/repository.py | 33 +++++ infrahub_sdk/testing/repository.py | 22 +-- infrahub_sdk/transforms.py | 6 +- infrahub_sdk/utils.py | 5 +- poetry.lock | 132 ++++++++++++------ pyproject.toml | 2 +- .../integration/mock_repo/blank_schema.yml | 0 tests/integration/test_repository.py | 31 ++++ tests/unit/ctl/test_transform_app.py | 5 +- tests/unit/sdk/test_repository.py | 67 +++++++++ 14 files changed, 250 insertions(+), 73 deletions(-) create mode 100644 changelog/130.added.md create mode 100644 infrahub_sdk/repository.py create mode 100644 tests/fixtures/integration/mock_repo/blank_schema.yml create mode 100644 tests/integration/test_repository.py create mode 100644 tests/unit/sdk/test_repository.py diff --git a/changelog/130.added.md b/changelog/130.added.md new file mode 100644 index 0000000..266f732 --- /dev/null +++ b/changelog/130.added.md @@ -0,0 +1 @@ +Replace GitPython with dulwich \ No newline at end of file diff --git a/infrahub_sdk/checks.py b/infrahub_sdk/checks.py index 2ca3eb2..ad69253 100644 --- a/infrahub_sdk/checks.py +++ b/infrahub_sdk/checks.py @@ -8,9 +8,10 @@ from typing import TYPE_CHECKING, Any import ujson -from git.repo import Repo from pydantic import BaseModel, Field +from infrahub_sdk.repository import GitRepoManager + from .exceptions import UninitializedError if TYPE_CHECKING: @@ -43,7 +44,7 @@ def __init__( params: dict | None = None, client: InfrahubClient | None = None, ): - self.git: Repo | None = None + self.git: GitRepoManager | None = None self.initializer = initializer or InfrahubCheckInitializer() self.logs: list[dict[str, Any]] = [] @@ -137,10 +138,9 @@ def branch_name(self) -> str: return self.branch if not self.git: - self.git = Repo(self.root_directory) + self.git = GitRepoManager(self.root_directory) self.branch = str(self.git.active_branch) - return self.branch @abstractmethod diff --git a/infrahub_sdk/generator.py b/infrahub_sdk/generator.py index 3c57bd2..e6448b3 100644 --- a/infrahub_sdk/generator.py +++ b/infrahub_sdk/generator.py @@ -4,7 +4,7 @@ from abc import abstractmethod from typing import TYPE_CHECKING -from git.repo import Repo +from infrahub_sdk.repository import GitRepoManager from .exceptions import UninitializedError @@ -30,7 +30,7 @@ def __init__( ) -> None: self.query = query self.branch = branch - self.git: Repo | None = None + self.git: GitRepoManager | None = None self.params = params or {} self.root_directory = root_directory or os.getcwd() self.generator_instance = generator_instance @@ -81,7 +81,7 @@ def branch_name(self) -> str: return self.branch if not self.git: - self.git = Repo(self.root_directory) + self.git = GitRepoManager(self.root_directory) self.branch = str(self.git.active_branch) diff --git a/infrahub_sdk/pytest_plugin/items/base.py b/infrahub_sdk/pytest_plugin/items/base.py index 35eca9a..e00db68 100644 --- a/infrahub_sdk/pytest_plugin/items/base.py +++ b/infrahub_sdk/pytest_plugin/items/base.py @@ -6,7 +6,6 @@ import pytest import ujson -from git.exc import InvalidGitRepositoryError from ..exceptions import InvalidResourceConfigError from ..models import InfrahubInputOutputTest @@ -28,7 +27,6 @@ def __init__( **kwargs: dict[str, Any], ): super().__init__(*args, **kwargs) # type: ignore[arg-type] - self.resource_name: str = resource_name self.resource_config: InfrahubRepositoryConfigElement = resource_config self.test: InfrahubTest = test @@ -68,9 +66,6 @@ def runtest(self) -> None: """Run the test logic.""" def repr_failure(self, excinfo: pytest.ExceptionInfo, style: str | None = None) -> str: # noqa: ARG002 - if isinstance(excinfo.value, InvalidGitRepositoryError): - return f"Invalid Git repository at {excinfo.value}" - return str(excinfo.value) def reportinfo(self) -> tuple[Path | str, int | None, str]: diff --git a/infrahub_sdk/repository.py b/infrahub_sdk/repository.py new file mode 100644 index 0000000..9472c4f --- /dev/null +++ b/infrahub_sdk/repository.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from pathlib import Path + +from dulwich import porcelain +from dulwich.repo import Repo + + +class GitRepoManager: + def __init__(self, root_directory: str, branch: str = "main"): + self.root_directory = root_directory + self.branch = branch + self.git: Repo = self.initialize_repo() + + def initialize_repo(self) -> Repo: + # Check if the directory already has a repository + + root_path = Path(self.root_directory) + + if root_path.exists() and (root_path / ".git").is_dir(): + repo = Repo(self.root_directory) # Open existing repo + else: + repo = Repo.init(self.root_directory, default_branch=self.branch.encode("utf-8")) + + if not repo: + raise ValueError("Failed to initialize or open a repository.") + + return repo + + @property + def active_branch(self) -> str | None: + active_branch = porcelain.active_branch(self.root_directory).decode("utf-8") + return active_branch diff --git a/infrahub_sdk/testing/repository.py b/infrahub_sdk/testing/repository.py index dcdb787..9e97416 100644 --- a/infrahub_sdk/testing/repository.py +++ b/infrahub_sdk/testing/repository.py @@ -7,10 +7,11 @@ from pathlib import Path from typing import TYPE_CHECKING -from git.repo import Repo +from dulwich import porcelain from infrahub_sdk.graphql import Mutation from infrahub_sdk.protocols import CoreGenericRepository +from infrahub_sdk.repository import GitRepoManager if TYPE_CHECKING: from infrahub_sdk import InfrahubClient @@ -37,14 +38,14 @@ class GitRepo: type: GitRepoType = GitRepoType.INTEGRATED - _repo: Repo | None = None + _repo: GitRepoManager | None = None initial_branch: str = "main" directories_to_ignore: list[str] = field(default_factory=list) remote_directory_name: str = "/remote" _branches: list[str] = field(default_factory=list) @property - def repo(self) -> Repo: + def repo(self) -> GitRepoManager: if self._repo: return self._repo raise ValueError("Repo hasn't been initialized yet") @@ -62,12 +63,17 @@ def init(self) -> None: dst=self.dst_directory / self.name, ignore=shutil.ignore_patterns(".git"), ) - self._repo = Repo.init(self.dst_directory / self.name, initial_branch=self.initial_branch) - for untracked in self.repo.untracked_files: - self.repo.index.add(untracked) - self.repo.index.commit("First commit") - self.repo.git.checkout(self.initial_branch) + self._repo = GitRepoManager(str(Path(self.dst_directory / self.name)), branch=self.initial_branch) + + files = list( + porcelain.get_untracked_paths(self._repo.git.path, self._repo.git.path, self._repo.git.open_index()) + ) + files_to_add = [str(Path(self._repo.git.path) / t) for t in files] + if files_to_add: + porcelain.add(repo=self._repo.git.path, paths=files_to_add) + porcelain.commit(repo=self._repo.git.path, message="First commit") + porcelain.checkout_branch(self._repo.git, self.initial_branch.encode("utf-8")) async def add_to_infrahub(self, client: InfrahubClient, branch: str | None = None) -> dict: input_data = { diff --git a/infrahub_sdk/transforms.py b/infrahub_sdk/transforms.py index 3073b7b..79ad74f 100644 --- a/infrahub_sdk/transforms.py +++ b/infrahub_sdk/transforms.py @@ -5,7 +5,7 @@ from abc import abstractmethod from typing import TYPE_CHECKING, Any -from git import Repo +from infrahub_sdk.repository import GitRepoManager from .exceptions import UninitializedError @@ -27,7 +27,7 @@ def __init__( server_url: str = "", client: InfrahubClient | None = None, ): - self.git: Repo + self.git: GitRepoManager self.branch = branch self.server_url = server_url or os.environ.get("INFRAHUB_URL", "http://127.0.0.1:8000") @@ -56,7 +56,7 @@ def branch_name(self) -> str: return self.branch if not hasattr(self, "git") or not self.git: - self.git = Repo(self.root_directory) + self.git = GitRepoManager(self.root_directory) self.branch = str(self.git.active_branch) diff --git a/infrahub_sdk/utils.py b/infrahub_sdk/utils.py index 1231dae..ee93edf 100644 --- a/infrahub_sdk/utils.py +++ b/infrahub_sdk/utils.py @@ -9,13 +9,14 @@ import httpx import ujson -from git.repo import Repo from graphql import ( FieldNode, InlineFragmentNode, SelectionSetNode, ) +from infrahub_sdk.repository import GitRepoManager + from .exceptions import JsonDecodeError if TYPE_CHECKING: @@ -246,7 +247,7 @@ def get_branch(branch: str | None = None, directory: str | Path = ".") -> str: if branch: return branch - repo = Repo(directory) + repo = GitRepoManager(directory) return str(repo.active_branch) diff --git a/poetry.lock b/poetry.lock index f04afb0..9954139 100644 --- a/poetry.lock +++ b/poetry.lock @@ -357,6 +357,93 @@ docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] ssh = ["paramiko (>=2.4.3)"] websockets = ["websocket-client (>=1.3.0)"] +[[package]] +name = "dulwich" +version = "0.21.7" +description = "Python Git Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dulwich-0.21.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d4c0110798099bb7d36a110090f2688050703065448895c4f53ade808d889dd3"}, + {file = "dulwich-0.21.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2bc12697f0918bee324c18836053644035362bb3983dc1b210318f2fed1d7132"}, + {file = "dulwich-0.21.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:471305af74790827fcbafe330fc2e8bdcee4fb56ca1177c8c481b1c8f806c4a4"}, + {file = "dulwich-0.21.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d54c9d0e845be26f65f954dff13a1cd3f2b9739820c19064257b8fd7435ab263"}, + {file = "dulwich-0.21.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12d61334a575474e707614f2e93d6ed4cdae9eb47214f9277076d9e5615171d3"}, + {file = "dulwich-0.21.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e274cebaf345f0b1e3b70197f2651de92b652386b68020cfd3bf61bc30f6eaaa"}, + {file = "dulwich-0.21.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:817822f970e196e757ae01281ecbf21369383285b9f4a83496312204cf889b8c"}, + {file = "dulwich-0.21.7-cp310-cp310-win32.whl", hash = "sha256:7836da3f4110ce684dcd53489015fb7fa94ed33c5276e3318b8b1cbcb5b71e08"}, + {file = "dulwich-0.21.7-cp310-cp310-win_amd64.whl", hash = "sha256:4a043b90958cec866b4edc6aef5fe3c2c96a664d0b357e1682a46f6c477273c4"}, + {file = "dulwich-0.21.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ce8db196e79c1f381469410d26fb1d8b89c6b87a4e7f00ff418c22a35121405c"}, + {file = "dulwich-0.21.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:62bfb26bdce869cd40be443dfd93143caea7089b165d2dcc33de40f6ac9d812a"}, + {file = "dulwich-0.21.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c01a735b9a171dcb634a97a3cec1b174cfbfa8e840156870384b633da0460f18"}, + {file = "dulwich-0.21.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa4d14767cf7a49c9231c2e52cb2a3e90d0c83f843eb6a2ca2b5d81d254cf6b9"}, + {file = "dulwich-0.21.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bca4b86e96d6ef18c5bc39828ea349efb5be2f9b1f6ac9863f90589bac1084d"}, + {file = "dulwich-0.21.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a7b5624b02ef808cdc62dabd47eb10cd4ac15e8ac6df9e2e88b6ac6b40133673"}, + {file = "dulwich-0.21.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c3a539b4696a42fbdb7412cb7b66a4d4d332761299d3613d90a642923c7560e1"}, + {file = "dulwich-0.21.7-cp311-cp311-win32.whl", hash = "sha256:675a612ce913081beb0f37b286891e795d905691dfccfb9bf73721dca6757cde"}, + {file = "dulwich-0.21.7-cp311-cp311-win_amd64.whl", hash = "sha256:460ba74bdb19f8d498786ae7776745875059b1178066208c0fd509792d7f7bfc"}, + {file = "dulwich-0.21.7-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4c51058ec4c0b45dc5189225b9e0c671b96ca9713c1daf71d622c13b0ab07681"}, + {file = "dulwich-0.21.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4bc4c5366eaf26dda3fdffe160a3b515666ed27c2419f1d483da285ac1411de0"}, + {file = "dulwich-0.21.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a0650ec77d89cb947e3e4bbd4841c96f74e52b4650830112c3057a8ca891dc2f"}, + {file = "dulwich-0.21.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f18f0a311fb7734b033a3101292b932158cade54b74d1c44db519e42825e5a2"}, + {file = "dulwich-0.21.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c589468e5c0cd84e97eb7ec209ab005a2cb69399e8c5861c3edfe38989ac3a8"}, + {file = "dulwich-0.21.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d62446797163317a397a10080c6397ffaaca51a7804c0120b334f8165736c56a"}, + {file = "dulwich-0.21.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e84cc606b1f581733df4350ca4070e6a8b30be3662bbb81a590b177d0c996c91"}, + {file = "dulwich-0.21.7-cp312-cp312-win32.whl", hash = "sha256:c3d1685f320907a52c40fd5890627945c51f3a5fa4bcfe10edb24fec79caadec"}, + {file = "dulwich-0.21.7-cp312-cp312-win_amd64.whl", hash = "sha256:6bd69921fdd813b7469a3c77bc75c1783cc1d8d72ab15a406598e5a3ba1a1503"}, + {file = "dulwich-0.21.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7d8ab29c660125db52106775caa1f8f7f77a69ed1fe8bc4b42bdf115731a25bf"}, + {file = "dulwich-0.21.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0d2e4485b98695bf95350ce9d38b1bb0aaac2c34ad00a0df789aa33c934469b"}, + {file = "dulwich-0.21.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e138d516baa6b5bafbe8f030eccc544d0d486d6819b82387fc0e285e62ef5261"}, + {file = "dulwich-0.21.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f34bf9b9fa9308376263fd9ac43143c7c09da9bc75037bb75c6c2423a151b92c"}, + {file = "dulwich-0.21.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2e2c66888207b71cd1daa2acb06d3984a6bc13787b837397a64117aa9fc5936a"}, + {file = "dulwich-0.21.7-cp37-cp37m-win32.whl", hash = "sha256:10893105c6566fc95bc2a67b61df7cc1e8f9126d02a1df6a8b2b82eb59db8ab9"}, + {file = "dulwich-0.21.7-cp37-cp37m-win_amd64.whl", hash = "sha256:460b3849d5c3d3818a80743b4f7a0094c893c559f678e56a02fff570b49a644a"}, + {file = "dulwich-0.21.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:74700e4c7d532877355743336c36f51b414d01e92ba7d304c4f8d9a5946dbc81"}, + {file = "dulwich-0.21.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c92e72c43c9e9e936b01a57167e0ea77d3fd2d82416edf9489faa87278a1cdf7"}, + {file = "dulwich-0.21.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d097e963eb6b9fa53266146471531ad9c6765bf390849230311514546ed64db2"}, + {file = "dulwich-0.21.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:808e8b9cc0aa9ac74870b49db4f9f39a52fb61694573f84b9c0613c928d4caf8"}, + {file = "dulwich-0.21.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1957b65f96e36c301e419d7adaadcff47647c30eb072468901bb683b1000bc5"}, + {file = "dulwich-0.21.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4b09bc3a64fb70132ec14326ecbe6e0555381108caff3496898962c4136a48c6"}, + {file = "dulwich-0.21.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5882e70b74ac3c736a42d3fdd4f5f2e6570637f59ad5d3e684760290b58f041"}, + {file = "dulwich-0.21.7-cp38-cp38-win32.whl", hash = "sha256:29bb5c1d70eba155ded41ed8a62be2f72edbb3c77b08f65b89c03976292f6d1b"}, + {file = "dulwich-0.21.7-cp38-cp38-win_amd64.whl", hash = "sha256:25c3ab8fb2e201ad2031ddd32e4c68b7c03cb34b24a5ff477b7a7dcef86372f5"}, + {file = "dulwich-0.21.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8929c37986c83deb4eb500c766ee28b6670285b512402647ee02a857320e377c"}, + {file = "dulwich-0.21.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc1e11be527ac06316539b57a7688bcb1b6a3e53933bc2f844397bc50734e9ae"}, + {file = "dulwich-0.21.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fc3078a1ba04c588fabb0969d3530efd5cd1ce2cf248eefb6baf7cbc15fc285"}, + {file = "dulwich-0.21.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40dcbd29ba30ba2c5bfbab07a61a5f20095541d5ac66d813056c122244df4ac0"}, + {file = "dulwich-0.21.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8869fc8ec3dda743e03d06d698ad489b3705775fe62825e00fa95aa158097fc0"}, + {file = "dulwich-0.21.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d96ca5e0dde49376fbcb44f10eddb6c30284a87bd03bb577c59bb0a1f63903fa"}, + {file = "dulwich-0.21.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0064363bd5e814359657ae32517fa8001e8573d9d040bd997908d488ab886ed"}, + {file = "dulwich-0.21.7-cp39-cp39-win32.whl", hash = "sha256:869eb7be48243e695673b07905d18b73d1054a85e1f6e298fe63ba2843bb2ca1"}, + {file = "dulwich-0.21.7-cp39-cp39-win_amd64.whl", hash = "sha256:404b8edeb3c3a86c47c0a498699fc064c93fa1f8bab2ffe919e8ab03eafaaad3"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e598d743c6c0548ebcd2baf94aa9c8bfacb787ea671eeeb5828cfbd7d56b552f"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a2d76c96426e791556836ef43542b639def81be4f1d6d4322cd886c115eae1"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6c88acb60a1f4d31bd6d13bfba465853b3df940ee4a0f2a3d6c7a0778c705b7"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ecd315847dea406a4decfa39d388a2521e4e31acde3bd9c2609c989e817c6d62"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d05d3c781bc74e2c2a2a8f4e4e2ed693540fbe88e6ac36df81deac574a6dad99"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6de6f8de4a453fdbae8062a6faa652255d22a3d8bce0cd6d2d6701305c75f2b3"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e25953c7acbbe4e19650d0225af1c0c0e6882f8bddd2056f75c1cc2b109b88ad"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:4637cbd8ed1012f67e1068aaed19fcc8b649bcf3e9e26649826a303298c89b9d"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:858842b30ad6486aacaa607d60bab9c9a29e7c59dc2d9cb77ae5a94053878c08"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:739b191f61e1c4ce18ac7d520e7a7cbda00e182c3489552408237200ce8411ad"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:274c18ec3599a92a9b67abaf110e4f181a4f779ee1aaab9e23a72e89d71b2bd9"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:2590e9b431efa94fc356ae33b38f5e64f1834ec3a94a6ac3a64283b206d07aa3"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed60d1f610ef6437586f7768254c2a93820ccbd4cfdac7d182cf2d6e615969bb"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8278835e168dd097089f9e53088c7a69c6ca0841aef580d9603eafe9aea8c358"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffc27fb063f740712e02b4d2f826aee8bbed737ed799962fef625e2ce56e2d29"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:61e3451bd3d3844f2dca53f131982553be4d1b1e1ebd9db701843dd76c4dba31"}, + {file = "dulwich-0.21.7.tar.gz", hash = "sha256:a9e9c66833cea580c3ac12927e4b9711985d76afca98da971405d414de60e968"}, +] + +[package.dependencies] +urllib3 = ">=1.25" + +[package.extras] +fastimport = ["fastimport"] +https = ["urllib3 (>=1.24.1)"] +paramiko = ["paramiko"] +pgp = ["gpg"] + [[package]] name = "eval-type-backport" version = "0.2.2" @@ -429,38 +516,6 @@ docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2. testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.1.1)", "pytest (>=8.3.2)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.3)"] typing = ["typing-extensions (>=4.12.2)"] -[[package]] -name = "gitdb" -version = "4.0.11" -description = "Git Object Database" -optional = false -python-versions = ">=3.7" -files = [ - {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, - {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, -] - -[package.dependencies] -smmap = ">=3.0.1,<6" - -[[package]] -name = "gitpython" -version = "3.1.43" -description = "GitPython is a Python library used to interact with Git repositories" -optional = false -python-versions = ">=3.7" -files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, -] - -[package.dependencies] -gitdb = ">=4.0.1,<5" - -[package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] - [[package]] name = "graphql-core" version = "3.2.4" @@ -1776,17 +1831,6 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -[[package]] -name = "smmap" -version = "5.0.1" -description = "A pure Python implementation of a sliding window memory map manager" -optional = false -python-versions = ">=3.7" -files = [ - {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, - {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, -] - [[package]] name = "sniffio" version = "1.3.1" @@ -2275,4 +2319,4 @@ tests = ["Jinja2", "pytest", "pyyaml", "rich"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "9ada2423e2a3604259f5a477b93ed1a20de107a8cf94d6758e8b9739e78d1bc8" +content-hash = "97d089383df1d2c713508ce441ebae49ff3f3371247174e723d465ca535bc2c1" diff --git a/pyproject.toml b/pyproject.toml index b647be7..2e968f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ pendulum = [ { version = ">=2", python = ">=3.9,<3.12" }, { version = ">=3", python = ">=3.12" }, ] -gitpython = "^3" ujson = "^5" Jinja2 = { version = "^3", optional = true } numpy = [ @@ -51,6 +50,7 @@ typer = { version = "^0.12.3", optional = true } pytest = { version = "*", optional = true } pyyaml = { version = "^6", optional = true } eval-type-backport = { version = "^0.2.2", python = "~3.9" } +dulwich = "^0.21.4" [tool.poetry.group.dev.dependencies] pytest = "*" diff --git a/tests/fixtures/integration/mock_repo/blank_schema.yml b/tests/fixtures/integration/mock_repo/blank_schema.yml new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py new file mode 100644 index 0000000..9de48b9 --- /dev/null +++ b/tests/integration/test_repository.py @@ -0,0 +1,31 @@ +# from __future__ import annotations + +# from typing import TYPE_CHECKING + +# import pytest + +# from infrahub_sdk.testing.docker import TestInfrahubDockerClient +# from infrahub_sdk.testing.repository import GitRepo +# from infrahub_sdk.utils import get_fixtures_dir + +# if TYPE_CHECKING: +# from infrahub_sdk import InfrahubClient + + +# class TestInfrahubRepository(TestInfrahubDockerClient): +# @pytest.fixture(scope="class") +# def infrahub_version(self) -> str: +# return "1.1.0" + +# async def test_add_repository(self, client: InfrahubClient, remote_repos_dir): +# src_directory = get_fixtures_dir() / "integration/mock_repo" +# repo = GitRepo(name="mock_repo", src_directory=src_directory, dst_directory=remote_repos_dir) +# commit = repo._repo.git[repo._repo.git.head()] +# assert len(list(repo._repo.git.get_walker())) == 1 +# assert commit.message.decode("utf-8") == "First commit" + +# response = await repo.add_to_infrahub(client=client) +# assert response.get(f"{repo.type.value}Create", {}).get("ok") + +# repos = await client.all(kind=repo.type) +# assert repos diff --git a/tests/unit/ctl/test_transform_app.py b/tests/unit/ctl/test_transform_app.py index ea45289..6b10341 100644 --- a/tests/unit/ctl/test_transform_app.py +++ b/tests/unit/ctl/test_transform_app.py @@ -8,11 +8,11 @@ from pathlib import Path import pytest -from git import Repo from pytest_httpx._httpx_mock import HTTPXMock from typer.testing import CliRunner from infrahub_sdk.ctl.cli_commands import app +from infrahub_sdk.repository import GitRepoManager from tests.helpers.utils import change_directory, strip_color runner = CliRunner() @@ -41,8 +41,7 @@ def tags_transform_dir(): fixture_path = Path(FIXTURE_BASE_DIR / "tags_transform") shutil.copytree(fixture_path, temp_dir, dirs_exist_ok=True) # Initialize fixture as git repo. This is necessary to run some infrahubctl commands. - with change_directory(temp_dir): - Repo.init(".", initial_branch="main") + GitRepoManager(temp_dir) yield temp_dir diff --git a/tests/unit/sdk/test_repository.py b/tests/unit/sdk/test_repository.py new file mode 100644 index 0000000..43ef388 --- /dev/null +++ b/tests/unit/sdk/test_repository.py @@ -0,0 +1,67 @@ +import tempfile +from pathlib import Path + +import pytest +from dulwich.repo import Repo + +from infrahub_sdk.repository import GitRepoManager +from infrahub_sdk.testing.repository import GitRepo +from infrahub_sdk.utils import get_fixtures_dir + + +@pytest.fixture +def temp_dir(): + """Fixture to create a temporary directory for testing.""" + with tempfile.TemporaryDirectory() as tmp_dir: + yield tmp_dir + + +def test_initialize_repo_creates_new_repo(temp_dir): + """Test that a new Git repository is created if none exists.""" + manager = GitRepoManager(root_directory=temp_dir, branch="main") + + # Verify .git directory is created + assert (Path(temp_dir) / ".git").is_dir() + + # Verify the repository is initialized + assert manager.git is not None + assert isinstance(manager.git, Repo) + + +def test_initialize_repo_uses_existing_repo(temp_dir): + """Test that the GitRepoManager uses an existing repository without an active branch.""" + # Manually initialize a repo + Repo.init(temp_dir, default_branch=b"main") + + manager = GitRepoManager(temp_dir) + assert manager.git is not None + assert isinstance(manager.git, Repo) + assert (Path(temp_dir) / ".git").is_dir() + + +def test_active_branch_returns_correct_branch(temp_dir): + """Test that the active branch is correctly returned.""" + manager = GitRepoManager(temp_dir, branch="develop") + + # Verify the active branch is "develop" + assert manager.active_branch == "develop" + + +def test_initialize_repo_raises_error_on_failure(monkeypatch, temp_dir): + """Test that an error is raised if the repository cannot be initialized.""" + + def mock_init(*args, **kwargs): # noqa: ANN002, ANN003 + return None # Simulate failure + + monkeypatch.setattr(Repo, "init", mock_init) + + with pytest.raises(ValueError, match=r"Failed to initialize or open a repository\."): + GitRepoManager(temp_dir) + + +def test_gitrepo_init(temp_dir): + src_directory = get_fixtures_dir() / "integration/mock_repo" + repo = GitRepo(name="mock_repo", src_directory=src_directory, dst_directory=Path(temp_dir)) + assert len(list(repo._repo.git.get_walker())) == 1 + commit = repo._repo.git[repo._repo.git.head()] + assert commit.message.decode("utf-8") == "First commit" From efcf4723eb43f502d32c73fb356573a119b3cf69 Mon Sep 17 00:00:00 2001 From: wartraxx51 <107243342+wartraxx51@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:33:50 +0100 Subject: [PATCH 05/14] Add variable to fix repository issue from ci (#234) --- .github/workflows/ci.yml | 4 +++ tests/integration/test_repository.py | 44 ++++++++++++++-------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cdf3f3..a875d0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,6 +190,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + - name: "Set environment variables" + run: | + RUNNER_NUM=$(hostname | grep -o '[0-9][0-9]' | sed 's/^0*//') + echo PYTEST_DEBUG_TEMPROOT=/var/lib/github/ghrunner_${RUNNER_NUM}/_temp >> $GITHUB_ENV - name: "Setup environment" run: | pipx install poetry==1.8.5 diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index 9de48b9..359f1e6 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,31 +1,31 @@ -# from __future__ import annotations +from __future__ import annotations -# from typing import TYPE_CHECKING +from typing import TYPE_CHECKING -# import pytest +import pytest -# from infrahub_sdk.testing.docker import TestInfrahubDockerClient -# from infrahub_sdk.testing.repository import GitRepo -# from infrahub_sdk.utils import get_fixtures_dir +from infrahub_sdk.testing.docker import TestInfrahubDockerClient +from infrahub_sdk.testing.repository import GitRepo +from infrahub_sdk.utils import get_fixtures_dir -# if TYPE_CHECKING: -# from infrahub_sdk import InfrahubClient +if TYPE_CHECKING: + from infrahub_sdk import InfrahubClient -# class TestInfrahubRepository(TestInfrahubDockerClient): -# @pytest.fixture(scope="class") -# def infrahub_version(self) -> str: -# return "1.1.0" +class TestInfrahubRepository(TestInfrahubDockerClient): + @pytest.fixture(scope="class") + def infrahub_version(self) -> str: + return "1.1.0" -# async def test_add_repository(self, client: InfrahubClient, remote_repos_dir): -# src_directory = get_fixtures_dir() / "integration/mock_repo" -# repo = GitRepo(name="mock_repo", src_directory=src_directory, dst_directory=remote_repos_dir) -# commit = repo._repo.git[repo._repo.git.head()] -# assert len(list(repo._repo.git.get_walker())) == 1 -# assert commit.message.decode("utf-8") == "First commit" + async def test_add_repository(self, client: InfrahubClient, remote_repos_dir): + src_directory = get_fixtures_dir() / "integration/mock_repo" + repo = GitRepo(name="mock_repo", src_directory=src_directory, dst_directory=remote_repos_dir) + commit = repo._repo.git[repo._repo.git.head()] + assert len(list(repo._repo.git.get_walker())) == 1 + assert commit.message.decode("utf-8") == "First commit" -# response = await repo.add_to_infrahub(client=client) -# assert response.get(f"{repo.type.value}Create", {}).get("ok") + response = await repo.add_to_infrahub(client=client) + assert response.get(f"{repo.type.value}Create", {}).get("ok") -# repos = await client.all(kind=repo.type) -# assert repos + repos = await client.all(kind=repo.type) + assert repos From 6c5057f8a4b30dc71b6d010d69f4b5077de64523 Mon Sep 17 00:00:00 2001 From: jeremy brisson Date: Mon, 13 Jan 2025 13:35:16 +0100 Subject: [PATCH 06/14] fix ci for repository --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a875d0c..1cf7066 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,8 +192,8 @@ jobs: python-version: "3.12" - name: "Set environment variables" run: | - RUNNER_NUM=$(hostname | grep -o '[0-9][0-9]' | sed 's/^0*//') - echo PYTEST_DEBUG_TEMPROOT=/var/lib/github/ghrunner_${RUNNER_NUM}/_temp >> $GITHUB_ENV + RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/') + echo PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp >> $GITHUB_ENV - name: "Setup environment" run: | pipx install poetry==1.8.5 From 68f00c6a376c4809101a07a9b473b52649ca94ed Mon Sep 17 00:00:00 2001 From: jeremy brisson Date: Mon, 13 Jan 2025 13:39:58 +0100 Subject: [PATCH 07/14] fix ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cf7066..b8a348a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,7 +193,7 @@ jobs: - name: "Set environment variables" run: | RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/') - echo PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp >> $GITHUB_ENV + echo "PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp" >> $GITHUB_ENV - name: "Setup environment" run: | pipx install poetry==1.8.5 From 4947c708041c540a7cbb19faae21f47e8e3a3236 Mon Sep 17 00:00:00 2001 From: wvandeun Date: Fri, 10 Jan 2025 16:47:25 +0100 Subject: [PATCH 08/14] fixes parallel query execution not considering filters --- changelog/query-parallel.fixed.md | 1 + infrahub_sdk/client.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog/query-parallel.fixed.md diff --git a/changelog/query-parallel.fixed.md b/changelog/query-parallel.fixed.md new file mode 100644 index 0000000..e337ce9 --- /dev/null +++ b/changelog/query-parallel.fixed.md @@ -0,0 +1 @@ +fixes issue where using `parallel` query execution could lead to excessive and unneeded GraphQL queries diff --git a/infrahub_sdk/client.py b/infrahub_sdk/client.py index d90b1b3..14111f4 100644 --- a/infrahub_sdk/client.py +++ b/infrahub_sdk/client.py @@ -786,7 +786,7 @@ async def process_batch() -> tuple[list[InfrahubNode], list[InfrahubNode]]: nodes = [] related_nodes = [] batch_process = await self.create_batch() - count = await self.count(kind=schema.kind) + count = await self.count(kind=schema.kind, **filters) total_pages = (count + pagination_size - 1) // pagination_size for page_number in range(1, total_pages + 1): @@ -1930,7 +1930,7 @@ def process_batch() -> tuple[list[InfrahubNodeSync], list[InfrahubNodeSync]]: related_nodes = [] batch_process = self.create_batch() - count = self.count(kind=schema.kind) + count = self.count(kind=schema.kind, **filters) total_pages = (count + pagination_size - 1) // pagination_size for page_number in range(1, total_pages + 1): From df5cff6cd374bff80c41d4f3c1fefab6b973b502 Mon Sep 17 00:00:00 2001 From: Patrick Ogenstad Date: Tue, 14 Jan 2025 10:13:52 +0100 Subject: [PATCH 09/14] Replace `Literal[None]` with `None` --- infrahub_sdk/client.py | 24 ++++++++++++------------ pyproject.toml | 1 - 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/infrahub_sdk/client.py b/infrahub_sdk/client.py index 14111f4..4dda3eb 100644 --- a/infrahub_sdk/client.py +++ b/infrahub_sdk/client.py @@ -1186,7 +1186,7 @@ async def allocate_next_ip_address( async def allocate_next_ip_address( self, resource_pool: CoreNode, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., address_type: str | None = ..., @@ -1201,7 +1201,7 @@ async def allocate_next_ip_address( async def allocate_next_ip_address( self, resource_pool: CoreNode, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., address_type: str | None = ..., @@ -1216,7 +1216,7 @@ async def allocate_next_ip_address( async def allocate_next_ip_address( self, resource_pool: CoreNode, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., address_type: str | None = ..., @@ -1333,7 +1333,7 @@ async def allocate_next_ip_prefix( async def allocate_next_ip_prefix( self, resource_pool: CoreNode, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., member_type: str | None = ..., @@ -1349,7 +1349,7 @@ async def allocate_next_ip_prefix( async def allocate_next_ip_prefix( self, resource_pool: CoreNode, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., member_type: str | None = ..., @@ -1365,7 +1365,7 @@ async def allocate_next_ip_prefix( async def allocate_next_ip_prefix( self, resource_pool: CoreNode, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., member_type: str | None = ..., @@ -2306,7 +2306,7 @@ def allocate_next_ip_address( def allocate_next_ip_address( self, resource_pool: CoreNodeSync, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., address_type: str | None = ..., @@ -2321,7 +2321,7 @@ def allocate_next_ip_address( def allocate_next_ip_address( self, resource_pool: CoreNodeSync, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., address_type: str | None = ..., @@ -2336,7 +2336,7 @@ def allocate_next_ip_address( def allocate_next_ip_address( self, resource_pool: CoreNodeSync, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., address_type: str | None = ..., @@ -2449,7 +2449,7 @@ def allocate_next_ip_prefix( def allocate_next_ip_prefix( self, resource_pool: CoreNodeSync, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., member_type: str | None = ..., @@ -2465,7 +2465,7 @@ def allocate_next_ip_prefix( def allocate_next_ip_prefix( self, resource_pool: CoreNodeSync, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., member_type: str | None = ..., @@ -2481,7 +2481,7 @@ def allocate_next_ip_prefix( def allocate_next_ip_prefix( self, resource_pool: CoreNodeSync, - kind: Literal[None] = ..., + kind: None = ..., identifier: str | None = ..., prefix_length: int | None = ..., member_type: str | None = ..., diff --git a/pyproject.toml b/pyproject.toml index 2e968f0..c9a61d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,7 +254,6 @@ ignore = [ "PLW1641", # Object does not implement `__hash__` method "PTH100", # `os.path.abspath()` should be replaced by `Path.resolve()` "PTH109", # `os.getcwd()` should be replaced by `Path.cwd()` - "PYI061", # [*] `Literal[None]` can be replaced with `None` "RET504", # Unnecessary assignment to `data` before `return` statement "RUF005", # Consider `[*path, str(key)]` instead of concatenation "RUF015", # Prefer `next(iter(input_data["variables"].keys()))` over single element slice From 6c6bd0a45e499f1dd91b1b68e6b1784121a834b1 Mon Sep 17 00:00:00 2001 From: Lucas Guillermou Date: Fri, 15 Nov 2024 11:39:34 +0100 Subject: [PATCH 10/14] Remove task_report.py # Conflicts: # infrahub_sdk/task_report.py --- infrahub_sdk/task_report.py | 208 ------------------------------------ 1 file changed, 208 deletions(-) delete mode 100644 infrahub_sdk/task_report.py diff --git a/infrahub_sdk/task_report.py b/infrahub_sdk/task_report.py deleted file mode 100644 index 317c296..0000000 --- a/infrahub_sdk/task_report.py +++ /dev/null @@ -1,208 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, Union, runtime_checkable - -from typing_extensions import Self - -from .uuidt import generate_uuid - -if TYPE_CHECKING: - from types import TracebackType - - from .client import InfrahubClient - - -class Log(TypedDict): - message: str - severity: str - - -TaskLogs = Union[list[Log], Log] - - -class TaskReport: - def __init__( - self, - client: InfrahubClient, - logger: InfrahubLogger, - related_node: str, - title: str, - task_id: str | None = None, - created_by: str | None = None, - create_with_context: bool = True, - ): - self.client = client - self.title = title - self.task_id: Final = task_id or generate_uuid() - self.related_node: Final = related_node - self.created_by: Final = created_by - self.has_failures: bool = False - self.finalized: bool = False - self.created: bool = False - self.create_with_context = create_with_context - self.log = logger - - async def __aenter__(self) -> Self: - if self.create_with_context: - await self.create() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - if exc_type: - self.finalized = True - await self.update(conclusion="FAILURE", logs={"message": str(exc_value), "severity": "ERROR"}) - - if self.finalized or not self.created: - return - - conclusion = "FAILURE" if self.has_failures else "SUCCESS" - await self.update(conclusion=conclusion) - - async def create(self, title: str | None = None, conclusion: str = "UNKNOWN", logs: TaskLogs | None = None) -> None: - variables: dict[str, Any] = { - "related_node": self.related_node, - "task_id": self.task_id, - "title": title or self.title, - "conclusion": conclusion, - } - if self.created_by: - variables["created_by"] = self.created_by - if logs: - variables["logs"] = logs - - await self.client.execute_graphql( - query=CREATE_TASK, - variables=variables, - ) - self.created = True - - async def info(self, event: str, *args: Any, **kw: Any) -> None: - self.log.info(event, *args, **kw) - await self.update(logs={"severity": "INFO", "message": event}) - - async def warning(self, event: str, *args: Any, **kw: Any) -> None: - self.log.warning(event, *args, **kw) - await self.update(logs={"severity": "WARNING", "message": event}) - - async def error(self, event: str, *args: Any, **kw: Any) -> None: - self.log.error(event, *args, **kw) - self.has_failures = True - await self.update(logs={"severity": "ERROR", "message": event}) - - async def critical(self, event: str, *args: Any, **kw: Any) -> None: - self.log.critical(event, *args, **kw) - self.has_failures = True - await self.update(logs={"severity": "CRITICAL", "message": event}) - - async def exception(self, event: str, *args: Any, **kw: Any) -> None: - self.log.critical(event, *args, **kw) - self.has_failures = True - await self.update(logs={"severity": "CRITICAL", "message": event}) - - async def finalise( - self, title: str | None = None, conclusion: str = "SUCCESS", logs: TaskLogs | None = None - ) -> None: - self.finalized = True - await self.update(title=title, conclusion=conclusion, logs=logs) - - async def update( - self, title: str | None = None, conclusion: str | None = None, logs: TaskLogs | None = None - ) -> None: - if not self.created: - await self.create() - variables: dict[str, Any] = {"task_id": self.task_id} - if conclusion: - variables["conclusion"] = conclusion - if title: - variables["title"] = title - if logs: - variables["logs"] = logs - await self.client.execute_graphql(query=UPDATE_TASK, variables=variables) - - -class InfrahubLogger(Protocol): - def debug(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send a debug event""" - - def info(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send an info event""" - - def warning(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send a warning event""" - - def error(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send an error event.""" - - def critical(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send a critical event.""" - - def exception(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send an exception event.""" - - -@runtime_checkable -class InfrahubTaskReportLogger(Protocol): - async def info(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send an info event""" - - async def warning(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send a warning event""" - - async def error(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send an error event.""" - - async def critical(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send a critical event.""" - - async def exception(self, event: str | None = None, *args: Any, **kw: Any) -> Any: - """Send an exception event.""" - - -CREATE_TASK = """ -mutation CreateTask( - $conclusion: TaskConclusion!, - $title: String!, - $task_id: UUID, - $related_node: String!, - $created_by: String, - $logs: [RelatedTaskLogCreateInput] - ) { - InfrahubTaskCreate( - data: { - id: $task_id, - title: $title, - related_node: $related_node, - conclusion: $conclusion, - created_by: $created_by, - logs: $logs - } - ) { - ok - } -} -""" - -UPDATE_TASK = """ -mutation UpdateTask( - $conclusion: TaskConclusion, - $title: String, - $task_id: UUID!, - $logs: [RelatedTaskLogCreateInput] - ) { - InfrahubTaskUpdate( - data: { - id: $task_id, - title: $title, - conclusion: $conclusion, - logs: $logs - } - ) { - ok - } -} -""" From 2ca31cd1aa1aff023659a9f86b2039c2ad159ad0 Mon Sep 17 00:00:00 2001 From: Guillaume Mazoyer Date: Wed, 15 Jan 2025 09:37:59 +0100 Subject: [PATCH 11/14] Allow merging lists within dicts merge (#238) --- infrahub_sdk/utils.py | 6 +++++- tests/unit/sdk/test_utils.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/infrahub_sdk/utils.py b/infrahub_sdk/utils.py index ee93edf..aa065e2 100644 --- a/infrahub_sdk/utils.py +++ b/infrahub_sdk/utils.py @@ -136,8 +136,12 @@ def deep_merge_dict(dicta: dict, dictb: dict, path: list | None = None) -> dict: if key in dicta: if isinstance(dicta[key], dict) and isinstance(dictb[key], dict): deep_merge_dict(dicta[key], dictb[key], path + [str(key)]) + elif isinstance(dicta[key], list) and isinstance(dictb[key], list): + # Merge lists + # Cannot use compare_list because list of dicts won't work (dict not hashable) + dicta[key] = [i for i in dicta[key] if i not in dictb[key]] + dictb[key] elif dicta[key] == dictb[key]: - pass + continue else: raise ValueError("Conflict at %s" % ".".join(path + [str(key)])) else: diff --git a/tests/unit/sdk/test_utils.py b/tests/unit/sdk/test_utils.py index 33d2414..df424d2 100644 --- a/tests/unit/sdk/test_utils.py +++ b/tests/unit/sdk/test_utils.py @@ -87,8 +87,11 @@ def test_deep_merge_dict(): a = {"keyA": 1} b = {"keyB": {"sub1": 10}} c = {"keyB": {"sub2": 20}} + d = {"keyA": [10, 20], "keyB": "foo"} + e = {"keyA": [20, 30], "keyB": "foo"} assert deep_merge_dict(a, b) == {"keyA": 1, "keyB": {"sub1": 10}} assert deep_merge_dict(c, b) == {"keyB": {"sub1": 10, "sub2": 20}} + assert deep_merge_dict(d, e) == {"keyA": [10, 20, 30], "keyB": "foo"} def test_str_to_bool(): From 5f73796e22649286b966cd764e815700aaa7562a Mon Sep 17 00:00:00 2001 From: Brett Lykins Date: Thu, 16 Jan 2025 10:56:42 -0500 Subject: [PATCH 12/14] v1.6 prep (#239) * v1.6 prep --- CHANGELOG.md | 14 ++++++++++++++ changelog/130.added.md | 1 - changelog/count-method-filters.changed.md | 1 - changelog/query-parallel.fixed.md | 1 - pyproject.toml | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) delete mode 100644 changelog/130.added.md delete mode 100644 changelog/count-method-filters.changed.md delete mode 100644 changelog/query-parallel.fixed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index dfe6e86..a24c5ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,20 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang +## [1.6.0](https://github.com/opsmill/infrahub-sdk-python/tree/v1.6.0) - 2025-01-16 + +### Added + +- Replace GitPython with dulwich ([#130](https://github.com/opsmill/infrahub-sdk-python/issues/130)) + +### Changed + +- Added possibility to use filters for the SDK client's count method + +### Fixed + +- Fixes issue where using `parallel` query execution could lead to excessive and unneeded GraphQL queries + ## [1.5.0](https://github.com/opsmill/infrahub-sdk-python/tree/v1.5.0) - 2025-01-09 ### Added diff --git a/changelog/130.added.md b/changelog/130.added.md deleted file mode 100644 index 266f732..0000000 --- a/changelog/130.added.md +++ /dev/null @@ -1 +0,0 @@ -Replace GitPython with dulwich \ No newline at end of file diff --git a/changelog/count-method-filters.changed.md b/changelog/count-method-filters.changed.md deleted file mode 100644 index 13a15c5..0000000 --- a/changelog/count-method-filters.changed.md +++ /dev/null @@ -1 +0,0 @@ -Added possibility to use filters for the SDK client's count method diff --git a/changelog/query-parallel.fixed.md b/changelog/query-parallel.fixed.md deleted file mode 100644 index e337ce9..0000000 --- a/changelog/query-parallel.fixed.md +++ /dev/null @@ -1 +0,0 @@ -fixes issue where using `parallel` query execution could lead to excessive and unneeded GraphQL queries diff --git a/pyproject.toml b/pyproject.toml index c9a61d2..646a276 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.9" [tool.poetry] name = "infrahub-sdk" -version = "1.5.0" +version = "1.6.0" description = "Python Client to interact with Infrahub" authors = ["OpsMill "] readme = "README.md" From 3c4664eab181098e0f224352e682228f6d8db0f8 Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 16 Jan 2025 22:17:44 +0100 Subject: [PATCH 13/14] bump version to 1.6.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 646a276..b9221c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.9" [tool.poetry] name = "infrahub-sdk" -version = "1.6.0" +version = "1.6.1" description = "Python Client to interact with Infrahub" authors = ["OpsMill "] readme = "README.md" From ddadf3ad376a4ea827c47432cded4e7d0ae81beb Mon Sep 17 00:00:00 2001 From: wvandeun Date: Thu, 16 Jan 2025 22:20:38 +0100 Subject: [PATCH 14/14] add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a24c5ce..5701e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang +## [1.6.1](https://github.com/opsmill/infrahub-sdk-python/tree/v1.6.1) - 2025-01-16 + +Fixes release of v1.6.0 + ## [1.6.0](https://github.com/opsmill/infrahub-sdk-python/tree/v1.6.0) - 2025-01-16 ### Added