airbytehq · artem1205 · Jan 30, 2025 · Feb 23, 2025 · Feb 23, 2025 · Mar 3, 2025
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -29,7 +29,7 @@ repos:
       - id: ruff
         args:
           - --fix
-          - --select=I
+          - --select=I,F
 
       # Run the formatter.
       - id: ruff-format

@@ -3,7 +3,7 @@
 #
 from __future__ import annotations
 
-from typing import Callable, Final
+from typing import Final
 
 import dagger
 
@@ -48,7 +48,7 @@ def get_container(self, platform: dagger.Platform) -> dagger.Container:
                         # tar is equired to untar java connector binary distributions.
                         # openssl is required because we need to ssh and scp sometimes.
                         # findutils is required for xargs, which is shipped as part of findutils.
-                        f"yum install -y shadow-utils tar openssl findutils",
+                        "yum install -y shadow-utils tar openssl findutils",
                         # Update first, but in the same .with_exec step as the package installation.
                         # Otherwise, we risk caching stale package URLs.
                         "yum update -y --security",

@@ -7,7 +7,7 @@
 import json
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, List, Mapping, Optional, Tuple, Type
+from typing import Dict, List, Optional, Tuple, Type
 
 import dagger
 import semver

@@ -61,7 +61,7 @@ def get_ci_on_master_report(connector: Connector) -> Dict | None:
                 json_report = get_ci_json_report(json_report_url)
                 if json_report["connector_version"] == connector.version:
                     return json_report
-            except Exception as e:
+            except Exception:
                 continue
             return None
 

@@ -6,8 +6,6 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-import requests
-
 if TYPE_CHECKING:
     from typing import Callable, List, Set, Tuple
 

@@ -5,7 +5,6 @@
 from pathlib import Path
 from typing import Any
 
-from connector_ops.utils import Connector  # type: ignore
 from jinja2 import Environment, FileSystemLoader
 from markdown_it import MarkdownIt
 from markdown_it.tree import SyntaxTreeNode

@@ -2,7 +2,7 @@
 
 import re
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Dict, List
 
 from connector_ops.utils import Connector  # type: ignore
 from jinja2 import Environment, FileSystemLoader

@@ -1,6 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
-import os
 from datetime import datetime, timedelta
 
 import toml

@@ -481,4 +481,4 @@ def test_pass_when_documentation_file_has_changelog_entry(self, mocker, tmp_path
 
         # Assert
         assert result.status == CheckStatus.PASSED
-        assert f"Changelog entry found for version 1.0.0" in result.message
+        assert "Changelog entry found for version 1.0.0" in result.message
@@ -1,10 +1,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 from __future__ import annotations
 
-import os
-
-import pytest
-
 from connectors_qa import consts
 from connectors_qa.checks import metadata
 from connectors_qa.models import CheckStatus

@@ -80,7 +80,7 @@ def test_pass_when_https_url(self, mocker, tmp_path):
         # Arrange
         connector = mocker.MagicMock(code_directory=tmp_path)
         file_with_http_url = tmp_path / "file.py"
-        file_with_http_url.write_text(f"https://example.com")
+        file_with_http_url.write_text("https://example.com")
 
         # Act
         result = security.CheckConnectorUsesHTTPSOnly()._run(connector)

@@ -139,7 +139,7 @@ def get_connection_objects(
         get_configured_catalog_from_path(custom_configured_catalog_path, selected_streams) if custom_configured_catalog_path else None
     )
     custom_state = get_state_from_path(custom_state_path) if custom_state_path else None
-    is_ci = os.getenv("CI", False)
+    os.getenv("CI", False)
 
     if connection_id:
         if not retrieval_reason:

@@ -68,4 +68,4 @@ def test_write(tmp_path, messages, expected_writes):
     for expected_file, expected_content in expected_writes:
         expected_path = Path(tmp_path / expected_file)
         assert expected_path.exists()
-        content = expected_path.read_text()
+        expected_path.read_text()
@@ -1,11 +1,8 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 import pytest
-import requests
 import semver
-import yaml
 
-from metadata_service.docker_hub import get_latest_version_on_dockerhub
 from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
 from metadata_service.validators import metadata_validator
 
@@ -118,8 +115,6 @@ def incremented_version(current_version):
 
 
 def test_validation_fail_on_docker_image_tag_decrement(metadata_definition, decremented_version):
-    current_version = metadata_definition.data.dockerImageTag
-
     metadata_definition.data.dockerImageTag = decremented_version
     success, error_message = metadata_validator.validate_docker_image_tag_is_not_decremented(metadata_definition, None)
     assert not success

@@ -4,7 +4,6 @@
 
 import json
 import os
-import re
 from datetime import datetime
 from typing import List, Type, TypeVar
 

@@ -5,15 +5,12 @@
 import os
 from typing import List
 
-import numpy as np
-import pandas as pd
 import yaml
 from dagster import OpExecutionContext, Output, asset
 from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME
-from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
 from orchestrator.config import get_public_url_for_gcs_file
 from orchestrator.logging import sentry
-from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition, PartialMetadataDefinition
+from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition
 from orchestrator.utils.object_helpers import are_values_equal, merge_values
 
 

@@ -11,7 +11,6 @@
 
 import orchestrator.hacks as HACKS
 import pandas as pd
-import semver
 import sentry_sdk
 from dagster import AutoMaterializePolicy, DynamicPartitionsDefinition, MetadataValue, OpExecutionContext, Output, asset
 from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager

@@ -5,7 +5,7 @@
 import os
 
 import pandas as pd
-from dagster import AutoMaterializePolicy, FreshnessPolicy, OpExecutionContext, Output, asset
+from dagster import AutoMaterializePolicy, FreshnessPolicy, OpExecutionContext, asset
 from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe
 
 

@@ -17,7 +17,7 @@
 NIGHTLY_INDIVIDUAL_TEST_REPORT_FILE_NAME = "output.json"
 NIGHTLY_GHA_WORKFLOW_ID = "connector_nightly_builds_dagger.yml"
 CI_TEST_REPORT_PREFIX = "airbyte-ci/connectors/test"
-CI_MASTER_TEST_OUTPUT_REGEX = f".*master.*output.json$"
+CI_MASTER_TEST_OUTPUT_REGEX = ".*master.*output.json$"
 ANALYTICS_BUCKET = "ab-analytics-connector-metrics"
 ANALYTICS_FOLDER = "data/connector_quality_metrics"
 

@@ -3,7 +3,7 @@
 #
 
 from dagster import AssetSelection, SkipReason, define_asset_job, job, op
-from orchestrator.assets import metadata, registry, registry_entry, specs_secrets_mask
+from orchestrator.assets import registry_entry
 from orchestrator.config import HIGH_QUEUE_PRIORITY, MAX_METADATA_PARTITION_RUN_REQUEST
 from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus
 
@@ -103,7 +103,7 @@ def add_new_metadata_partitions_op(context):
     context.log.info(f"New etags found: {new_etags_found}")
 
     if not new_etags_found:
-        return SkipReason(f"No new metadata files to process in GCS bucket")
+        return SkipReason("No new metadata files to process in GCS bucket")
 
     # if there are more than the MAX_METADATA_PARTITION_RUN_REQUEST, we need to split them into multiple runs
     etags_to_process = new_etags_found

@@ -82,7 +82,7 @@ def _get_context_from_args_kwargs(args, kwargs):
 
     # otherwise raise an error
     raise Exception(
-        f"No context provided to Sentry Transaction. When using @instrument, ensure that the asset/op has a context as the first argument."
+        "No context provided to Sentry Transaction. When using @instrument, ensure that the asset/op has a context as the first argument."
     )
 
 

@@ -4,9 +4,7 @@
 
 import os
 
-from dagster import OpExecutionContext, op
-from dagster_slack import SlackResource
-from slack_sdk import WebhookClient
+from dagster import OpExecutionContext
 
 
 def chunk_messages(report):

@@ -8,7 +8,7 @@
 from typing import List, Optional, Union
 
 import dagster._check as check
-from dagster import BoolSource, Field, InitResourceContext, Noneable, StringSource, resource
+from dagster import Field, InitResourceContext, Noneable, StringSource, resource
 from dagster._core.storage.file_manager import check_file_like_obj
 from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager
 from google.cloud import storage

@@ -6,8 +6,7 @@
 from typing import List
 
 from dagster import InitResourceContext, StringSource, resource
-from dateutil.parser import parse
-from github import ContentFile, Github, GitTreeElement, Repository
+from github import ContentFile, Github, Repository
 from metadata_service.constants import METADATA_FILE_NAME
 from orchestrator.config import CONNECTORS_PATH
 
@@ -64,7 +63,7 @@ def get_latest_commit_for_file(github_connector_repo: Repository, path: str) ->
     config_schema={"connectors_path": StringSource},
 )
 def github_connectors_metadata_files(resource_context: InitResourceContext) -> List[dict]:
-    resource_context.log.info(f"retrieving github metadata files")
+    resource_context.log.info("retrieving github metadata files")
 
     github_connector_repo = resource_context.resources.github_connector_repo
     repo_file_tree = github_connector_repo.get_git_tree("master", recursive=True).tree
@@ -78,7 +77,7 @@ def github_connectors_metadata_files(resource_context: InitResourceContext) -> L
         if _valid_metadata_file_path(github_file.path)
     ]
 
-    resource_context.log.info(f"finished retrieving github metadata files")
+    resource_context.log.info("finished retrieving github metadata files")
     return metadata_file_paths
 
 

@@ -3,7 +3,7 @@
 #
 
 import hashlib
-from typing import List, Optional
+from typing import List
 
 import pandas as pd
 from dagster import MetadataValue, Output

@@ -8,30 +8,24 @@
 
 from typing import TYPE_CHECKING
 
-import anyio
-from dagger import File, QueryError
-
 from pipelines.airbyte_ci.connectors.build_image.steps.common import LoadContainerToLocalDockerHost
 from pipelines.airbyte_ci.connectors.build_image.steps.java_connectors import (
     BuildConnectorDistributionTar,
     BuildConnectorImages,
-    dist_tar_directory_path,
 )
 from pipelines.airbyte_ci.connectors.build_image.steps.normalization import BuildOrPullNormalization
 from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID
 from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext
 from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests
 from pipelines.airbyte_ci.steps.gradle import GradleTask
 from pipelines.consts import LOCAL_BUILD_PLATFORM
-from pipelines.dagger.actions.system import docker
 from pipelines.helpers.execution.run_steps import StepToRun
-from pipelines.helpers.utils import export_container_to_tarball
-from pipelines.models.steps import STEP_PARAMS, StepResult, StepStatus
+from pipelines.models.steps import STEP_PARAMS
 
 if TYPE_CHECKING:
-    from typing import Callable, Dict, List, Optional
+    from typing import List
 
-    from pipelines.helpers.execution.run_steps import RESULTS_DICT, STEP_TREE
+    from pipelines.helpers.execution.run_steps import STEP_TREE
 
 
 class IntegrationTests(GradleTask):

@@ -12,7 +12,7 @@
 from pathlib import Path
 from typing import Any, ClassVar, Dict, Generator, List, Optional, Tuple, cast
 
-from dagger import Container, ExecError
+from dagger import ExecError
 
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.helpers.utils import dagger_directory_as_zip_file

@@ -4,7 +4,6 @@
 
 from pathlib import Path
 
-import asyncclick as click
 import pytest
 
 from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, python_connectors

@@ -3,13 +3,10 @@
 #
 import datetime
 
-import asyncclick as click
 import pytest
 import requests
 
-from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.dagger.actions.python import common
 
 pytestmark = [
     pytest.mark.anyio,

@@ -1,10 +1,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
 import enum
-import time
-
-import anyio
-import pytest
 
 from pipelines.helpers.execution import argument_parsing
 

@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 import json
-import os
 import random
 from typing import List
 

@@ -7,7 +7,6 @@
 import time
 from typing import List
 
-import asyncclick as click
 import dagger
 import pytest
 import yaml

@@ -4,7 +4,6 @@
 
 from unittest.mock import patch
 
-import asyncclick as click
 import pytest
 from connector_ops.utils import Connector, ConnectorLanguage
 

@@ -2,20 +2,13 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-import json
-import random
-from pathlib import Path
-from typing import List
 from unittest.mock import AsyncMock, MagicMock
 
-import anyio
-import asyncclick as click
 import pytest
-from connector_ops.utils import Connector, ConnectorLanguage
+from connector_ops.utils import Connector
 from dagger import Directory
 
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline
 from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline
 from pipelines.models.steps import StepStatus