diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py index 67de2c6791b0e..51b871cb060e5 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py @@ -6,17 +6,17 @@ import json from typing import List, Union +import semver import sentry_sdk -from dagster import AutoMaterializePolicy, MetadataValue, OpExecutionContext, Output, asset +from dagster import MetadataValue, OpExecutionContext, Output, asset from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 from metadata_service.models.transform import to_json_sanitized_dict -from orchestrator.assets.registry_entry import ConnectorTypePrimaryKey, ConnectorTypes, read_registry_entry_blob +from orchestrator.assets.registry_entry import ConnectorTypePrimaryKey, ConnectorTypes from orchestrator.logging import sentry from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus -from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition from orchestrator.utils.object_helpers import default_none_to_dict from pydash.objects import set_with @@ -89,12 +89,18 @@ def apply_release_candidates( latest_registry_entry: dict, release_candidate_registry_entry: PolymorphicRegistryEntry, ) -> dict: - updated_registry_entry = copy.deepcopy(latest_registry_entry) - updated_registry_entry.setdefault("releases", {}) - updated_registry_entry["releases"]["releaseCandidates"] = { - release_candidate_registry_entry.dockerImageTag: to_json_sanitized_dict(release_candidate_registry_entry) - } - return updated_registry_entry + # Ensure that the release candidate is newer than the latest registry entry + if semver.Version.parse(release_candidate_registry_entry.dockerImageTag) > semver.Version.parse( + latest_registry_entry["dockerImageTag"] + ): + updated_registry_entry = copy.deepcopy(latest_registry_entry) + updated_registry_entry.setdefault("releases", {}) + updated_registry_entry["releases"]["releaseCandidates"] = { + release_candidate_registry_entry.dockerImageTag: to_json_sanitized_dict(release_candidate_registry_entry) + } + return updated_registry_entry + else: + return latest_registry_entry def apply_release_candidate_entries(registry_entry_dict: dict, docker_repository_to_rc_registry_entry: dict) -> dict: diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock b/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock index c3bc71da47bd5..f51b69f261c80 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock +++ b/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock @@ -2571,6 +2571,23 @@ tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -3691,4 +3708,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.10, <3.12" -content-hash = "cb5c8267e7d15c230b0f265678ee27ab4959ebbbd644ab45c118623fd7b42a37" +content-hash = "cea377cfb9ff4465d05311f1cc722d10ee11e25ff1610b47f5e242530bd1fb96" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml b/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml index 7c950506a90a0..876172dc4e477 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml +++ b/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "orchestrator" -version = "0.5.7" +version = "0.5.8" description = "" authors = ["Ben Church "] readme = "README.md" @@ -36,6 +36,7 @@ pendulum = "<3.0.0" [tool.poetry.group.dev.dependencies] pytest = "^8" +pytest-mock = "^3.14.0" [build-system] requires = ["poetry-core"] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py index d47510947188a..9615c4bcc39f5 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import copy from unittest import mock from uuid import UUID @@ -11,6 +12,7 @@ from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 +from orchestrator.assets import registry from orchestrator.assets.registry_entry import ( get_connector_type_from_registry_entry, get_registry_entry_write_path, @@ -485,3 +487,56 @@ def test_language_from_tags_does_not_override_top_level_language(): mock_metadata_entry.dependency_file_url = "test-dependency-file-url" result = metadata_to_registry_entry(mock_metadata_entry, "oss") assert result["language"] == "python" + + +def test_apply_release_candidates_with_older_rc(mocker): + uuid = UUID(int=1) + latest_registry_entry = { + "name": "source-test", + "sourceDefinitionId": str(uuid), + "dockerRepository": "test-repo", + "documentationUrl": "https://test_documentation_url.com", + "spec": {}, + "dockerImageTag": "1.1.0", + "releases": {}, + } + rc_registry_entry = ConnectorRegistrySourceDefinition.parse_obj( + { + "name": "source-test", + "sourceDefinitionId": str(uuid), + "dockerRepository": "test-repo", + "documentationUrl": "https://test_documentation_url.com", + "spec": {}, + "dockerImageTag": "1.1.0-rc.1", + "releases": {"isReleaseCandidate": True}, + } + ) + + result = registry.apply_release_candidates(latest_registry_entry, rc_registry_entry) + assert not result["releases"] + + +def test_apply_release_candidates_newer_version(mocker): + uuid = UUID(int=1) + latest_registry_entry = { + "name": "source-test", + "sourceDefinitionId": str(uuid), + "dockerRepository": "test-repo", + "documentationUrl": "https://test_documentation_url.com", + "spec": {}, + "dockerImageTag": "1.0.0", + "releases": {}, + } + rc_registry_entry = ConnectorRegistrySourceDefinition.parse_obj( + { + "name": "source-test", + "sourceDefinitionId": str(uuid), + "dockerRepository": "test-repo", + "documentationUrl": "https://test_documentation_url.com", + "spec": {}, + "dockerImageTag": "1.1.0-rc.1", + "releases": {"isReleaseCandidate": True}, + } + ) + result = registry.apply_release_candidates(latest_registry_entry, rc_registry_entry) + assert "1.1.0-rc.1" in result["releases"]["releaseCandidates"]