diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1170125df75e8..6be8af77173cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: ruff args: - --fix - - --select=I + - --select=I,F # Run the formatter. - id: ruff-format diff --git a/airbyte-ci/connectors/base_images/base_images/java/bases.py b/airbyte-ci/connectors/base_images/base_images/java/bases.py index 096a309015a24..d8ae644a60832 100644 --- a/airbyte-ci/connectors/base_images/base_images/java/bases.py +++ b/airbyte-ci/connectors/base_images/base_images/java/bases.py @@ -3,7 +3,7 @@ # from __future__ import annotations -from typing import Callable, Final +from typing import Final import dagger @@ -48,7 +48,7 @@ def get_container(self, platform: dagger.Platform) -> dagger.Container: # tar is equired to untar java connector binary distributions. # openssl is required because we need to ssh and scp sometimes. # findutils is required for xargs, which is shipped as part of findutils. - f"yum install -y shadow-utils tar openssl findutils", + "yum install -y shadow-utils tar openssl findutils", # Update first, but in the same .with_exec step as the package installation. # Otherwise, we risk caching stale package URLs. "yum update -y --security", diff --git a/airbyte-ci/connectors/base_images/base_images/version_registry.py b/airbyte-ci/connectors/base_images/base_images/version_registry.py index df2a137a6c179..d900996927af0 100644 --- a/airbyte-ci/connectors/base_images/base_images/version_registry.py +++ b/airbyte-ci/connectors/base_images/base_images/version_registry.py @@ -7,7 +7,7 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Dict, List, Mapping, Optional, Tuple, Type +from typing import Dict, List, Optional, Tuple, Type import dagger import semver diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/hacks.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/hacks.py index d521a1d8b8db2..fb49c0c621a01 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/hacks.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/hacks.py @@ -61,7 +61,7 @@ def get_ci_on_master_report(connector: Connector) -> Dict | None: json_report = get_ci_json_report(json_report_url) if json_report["connector_version"] == connector.version: return json_report - except Exception as e: + except Exception: continue return None diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/utils.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/utils.py index b68e62b61670c..1ad85afa8c80a 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/utils.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/utils.py @@ -6,8 +6,6 @@ from pathlib import Path from typing import TYPE_CHECKING -import requests - if TYPE_CHECKING: from typing import Callable, List, Set, Tuple diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/helpers.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/helpers.py index 1c947d0313447..51001a301f9ed 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/helpers.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/helpers.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import Any -from connector_ops.utils import Connector # type: ignore from jinja2 import Environment, FileSystemLoader from markdown_it import MarkdownIt from markdown_it.tree import SyntaxTreeNode diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/models.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/models.py index cfa7fa82a4789..5212dd20084e8 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/models.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/models.py @@ -2,7 +2,7 @@ import re from pathlib import Path -from typing import Any, Dict, List +from typing import Dict, List from connector_ops.utils import Connector # type: ignore from jinja2 import Environment, FileSystemLoader diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py index d7f0cd0c5df72..edef340f2c83e 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py @@ -1,6 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import os from datetime import datetime, timedelta import toml diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py index f5f58912489a7..1f613c193b342 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py @@ -481,4 +481,4 @@ def test_pass_when_documentation_file_has_changelog_entry(self, mocker, tmp_path # Assert assert result.status == CheckStatus.PASSED - assert f"Changelog entry found for version 1.0.0" in result.message + assert "Changelog entry found for version 1.0.0" in result.message diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py index 482a2536b2414..a66cc2bdca107 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py @@ -1,10 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. from __future__ import annotations -import os - -import pytest - from connectors_qa import consts from connectors_qa.checks import metadata from connectors_qa.models import CheckStatus diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_security.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_security.py index 37287b0779f09..508e4cf4d83ae 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_security.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_security.py @@ -80,7 +80,7 @@ def test_pass_when_https_url(self, mocker, tmp_path): # Arrange connector = mocker.MagicMock(code_directory=tmp_path) file_with_http_url = tmp_path / "file.py" - file_with_http_url.write_text(f"https://example.com") + file_with_http_url.write_text("https://example.com") # Act result = security.CheckConnectorUsesHTTPSOnly()._run(connector) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py index b531e3425be8e..a4b24d15167ed 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py @@ -139,7 +139,7 @@ def get_connection_objects( get_configured_catalog_from_path(custom_configured_catalog_path, selected_streams) if custom_configured_catalog_path else None ) custom_state = get_state_from_path(custom_state_path) if custom_state_path else None - is_ci = os.getenv("CI", False) + os.getenv("CI", False) if connection_id: if not retrieval_reason: diff --git a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py index fbd8f03bc4f7f..7cb5ca796078e 100644 --- a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py +++ b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py @@ -68,4 +68,4 @@ def test_write(tmp_path, messages, expected_writes): for expected_file, expected_content in expected_writes: expected_path = Path(tmp_path / expected_file) assert expected_path.exists() - content = expected_path.read_text() + expected_path.read_text() diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py index bc51db02988d5..c25ce341e2229 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py @@ -1,11 +1,8 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import pytest -import requests import semver -import yaml -from metadata_service.docker_hub import get_latest_version_on_dockerhub from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.validators import metadata_validator @@ -118,8 +115,6 @@ def incremented_version(current_version): def test_validation_fail_on_docker_image_tag_decrement(metadata_definition, decremented_version): - current_version = metadata_definition.data.dockerImageTag - metadata_definition.data.dockerImageTag = decremented_version success, error_message = metadata_validator.validate_docker_image_tag_is_not_decremented(metadata_definition, None) assert not success diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py index 1fe1ada3c5f18..045599cfc5af2 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py @@ -4,7 +4,6 @@ import json import os -import re from datetime import datetime from typing import List, Type, TypeVar diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py index 783f16614b44f..277bda3d4038b 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py @@ -5,15 +5,12 @@ import os from typing import List -import numpy as np -import pandas as pd import yaml from dagster import OpExecutionContext, Output, asset from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME -from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from orchestrator.config import get_public_url_for_gcs_file from orchestrator.logging import sentry -from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition, PartialMetadataDefinition +from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition from orchestrator.utils.object_helpers import are_values_equal, merge_values diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py index d642f5179bda9..21eb6602aa521 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py @@ -11,7 +11,6 @@ import orchestrator.hacks as HACKS import pandas as pd -import semver import sentry_sdk from dagster import AutoMaterializePolicy, DynamicPartitionsDefinition, MetadataValue, OpExecutionContext, Output, asset from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py index 2bbcbd2834511..cde178eccc305 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py @@ -5,7 +5,7 @@ import os import pandas as pd -from dagster import AutoMaterializePolicy, FreshnessPolicy, OpExecutionContext, Output, asset +from dagster import AutoMaterializePolicy, FreshnessPolicy, OpExecutionContext, asset from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py index b7f803334efe9..0206ddf3231e8 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py @@ -17,7 +17,7 @@ NIGHTLY_INDIVIDUAL_TEST_REPORT_FILE_NAME = "output.json" NIGHTLY_GHA_WORKFLOW_ID = "connector_nightly_builds_dagger.yml" CI_TEST_REPORT_PREFIX = "airbyte-ci/connectors/test" -CI_MASTER_TEST_OUTPUT_REGEX = f".*master.*output.json$" +CI_MASTER_TEST_OUTPUT_REGEX = ".*master.*output.json$" ANALYTICS_BUCKET = "ab-analytics-connector-metrics" ANALYTICS_FOLDER = "data/connector_quality_metrics" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py index 3756f7ac8a87b..3296f6329cf64 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py @@ -3,7 +3,7 @@ # from dagster import AssetSelection, SkipReason, define_asset_job, job, op -from orchestrator.assets import metadata, registry, registry_entry, specs_secrets_mask +from orchestrator.assets import registry_entry from orchestrator.config import HIGH_QUEUE_PRIORITY, MAX_METADATA_PARTITION_RUN_REQUEST from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus @@ -103,7 +103,7 @@ def add_new_metadata_partitions_op(context): context.log.info(f"New etags found: {new_etags_found}") if not new_etags_found: - return SkipReason(f"No new metadata files to process in GCS bucket") + return SkipReason("No new metadata files to process in GCS bucket") # if there are more than the MAX_METADATA_PARTITION_RUN_REQUEST, we need to split them into multiple runs etags_to_process = new_etags_found diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py index 8e908183e80bc..cc34a4a00e480 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py @@ -82,7 +82,7 @@ def _get_context_from_args_kwargs(args, kwargs): # otherwise raise an error raise Exception( - f"No context provided to Sentry Transaction. When using @instrument, ensure that the asset/op has a context as the first argument." + "No context provided to Sentry Transaction. When using @instrument, ensure that the asset/op has a context as the first argument." ) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py index cbcfdd6b82db9..049b07720c531 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py @@ -4,9 +4,7 @@ import os -from dagster import OpExecutionContext, op -from dagster_slack import SlackResource -from slack_sdk import WebhookClient +from dagster import OpExecutionContext def chunk_messages(report): diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py index 9dadf166b5c2b..fd538b51e9666 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py @@ -8,7 +8,7 @@ from typing import List, Optional, Union import dagster._check as check -from dagster import BoolSource, Field, InitResourceContext, Noneable, StringSource, resource +from dagster import Field, InitResourceContext, Noneable, StringSource, resource from dagster._core.storage.file_manager import check_file_like_obj from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager from google.cloud import storage diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py index a02c130d80214..5c2666ce5645f 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py @@ -6,8 +6,7 @@ from typing import List from dagster import InitResourceContext, StringSource, resource -from dateutil.parser import parse -from github import ContentFile, Github, GitTreeElement, Repository +from github import ContentFile, Github, Repository from metadata_service.constants import METADATA_FILE_NAME from orchestrator.config import CONNECTORS_PATH @@ -64,7 +63,7 @@ def get_latest_commit_for_file(github_connector_repo: Repository, path: str) -> config_schema={"connectors_path": StringSource}, ) def github_connectors_metadata_files(resource_context: InitResourceContext) -> List[dict]: - resource_context.log.info(f"retrieving github metadata files") + resource_context.log.info("retrieving github metadata files") github_connector_repo = resource_context.resources.github_connector_repo repo_file_tree = github_connector_repo.get_git_tree("master", recursive=True).tree @@ -78,7 +77,7 @@ def github_connectors_metadata_files(resource_context: InitResourceContext) -> L if _valid_metadata_file_path(github_file.path) ] - resource_context.log.info(f"finished retrieving github metadata files") + resource_context.log.info("finished retrieving github metadata files") return metadata_file_paths diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py index b5c996fe26c00..bcbeb1622d4fd 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py @@ -3,7 +3,7 @@ # import hashlib -from typing import List, Optional +from typing import List import pandas as pd from dagster import MetadataValue, Output diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py index 5c5ea5dcbe4bd..628b70800ecc7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py @@ -8,14 +8,10 @@ from typing import TYPE_CHECKING -import anyio -from dagger import File, QueryError - from pipelines.airbyte_ci.connectors.build_image.steps.common import LoadContainerToLocalDockerHost from pipelines.airbyte_ci.connectors.build_image.steps.java_connectors import ( BuildConnectorDistributionTar, BuildConnectorImages, - dist_tar_directory_path, ) from pipelines.airbyte_ci.connectors.build_image.steps.normalization import BuildOrPullNormalization from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID @@ -23,15 +19,13 @@ from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests from pipelines.airbyte_ci.steps.gradle import GradleTask from pipelines.consts import LOCAL_BUILD_PLATFORM -from pipelines.dagger.actions.system import docker from pipelines.helpers.execution.run_steps import StepToRun -from pipelines.helpers.utils import export_container_to_tarball -from pipelines.models.steps import STEP_PARAMS, StepResult, StepStatus +from pipelines.models.steps import STEP_PARAMS if TYPE_CHECKING: - from typing import Callable, Dict, List, Optional + from typing import List - from pipelines.helpers.execution.run_steps import RESULTS_DICT, STEP_TREE + from pipelines.helpers.execution.run_steps import STEP_TREE class IntegrationTests(GradleTask): diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 32aa134639078..4cc4785fe6ff0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any, ClassVar, Dict, Generator, List, Optional, Tuple, cast -from dagger import Container, ExecError +from dagger import ExecError from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.utils import dagger_directory_as_zip_file diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py index f03496b7d99a4..b84c2968d57e3 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py @@ -4,7 +4,6 @@ from pathlib import Path -import asyncclick as click import pytest from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, python_connectors diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py index cbba74169eea2..e81f5a3d6fdb3 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py @@ -3,13 +3,10 @@ # import datetime -import asyncclick as click import pytest import requests -from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.dagger.actions.python import common pytestmark = [ pytest.mark.anyio, diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py index a8630cb7988bc..7fd405c1d4d99 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py @@ -1,10 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import enum -import time - -import anyio -import pytest from pipelines.helpers.execution import argument_parsing diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index 97600973da2ce..b949d8c2010a7 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import json -import os import random from typing import List diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py index 4019951179a9f..4e74ae7bb37eb 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py @@ -7,7 +7,6 @@ import time from typing import List -import asyncclick as click import dagger import pytest import yaml diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py index 7980768e586da..dd91b8ea81e80 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -4,7 +4,6 @@ from unittest.mock import patch -import asyncclick as click import pytest from connector_ops.utils import Connector, ConnectorLanguage diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py index 4e515605107dd..b42c1c4593d7a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py @@ -2,20 +2,13 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json -import random -from pathlib import Path -from typing import List from unittest.mock import AsyncMock, MagicMock -import anyio -import asyncclick as click import pytest -from connector_ops.utils import Connector, ConnectorLanguage +from connector_ops.utils import Connector from dagger import Directory from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline from pipelines.models.steps import StepStatus diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py index 74e7a667eac40..fc068a23a5c20 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py @@ -5,8 +5,6 @@ import functools from typing import List, Mapping, Optional -import dpath.exceptions -import dpath.util import icdiff import py from pprintpp import pformat diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index d4dcec8c98552..1bd0d7ca55ed3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -1768,7 +1768,7 @@ async def test_read_validate_stream_statuses_exceptions(mocker, output): configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream.parse_obj({"name": f"test_stream_0", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}), + stream=AirbyteStream.parse_obj({"name": "test_stream_0", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}), sync_mode="full_refresh", destination_sync_mode="overwrite", ) @@ -1919,7 +1919,7 @@ async def test_read_validate_async_output_state_messages(mocker, state_message_p configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream.parse_obj({"name": f"test_stream_0", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}), + stream=AirbyteStream.parse_obj({"name": "test_stream_0", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}), sync_mode="full_refresh", destination_sync_mode="overwrite", ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py index bc76474d1c75d..be48a7089d3f0 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py @@ -12,7 +12,6 @@ Config, EmptyStreamConfiguration, ExpectedRecordsConfig, - IgnoredFieldsConfiguration, ) from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py index 7b9307f26fc37..e53bc4dd691d3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py @@ -3,7 +3,6 @@ # import json -import operator from contextlib import nullcontext as does_not_raise from logging import Logger from pathlib import Path diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py index 80ac4d0459b2a..8600fb1e623b9 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py @@ -56,7 +56,7 @@ def test_pytest_generate_tests(mocker, parametrize_skip_or_fail_return): HIGH_TEST_STRICTNESS_LEVEL, None, FAIL_ACTION, - f"MyTestClass.dumb_test_function failed: it was not configured but must be according to the current TestStrictnessLevel.high test strictness level.", + "MyTestClass.dumb_test_function failed: it was not configured but must be according to the current TestStrictnessLevel.high test strictness level.", id="Discovered test is mandatory in high test strictness level, we're in high test strictness level, it was not configured: FAIL", ), pytest.param( diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py index d9e1628a29a02..5597c0c9fae6b 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py @@ -7,7 +7,6 @@ import string import tempfile import time -from functools import partial from pathlib import Path from typing import Iterable from unittest.mock import Mock diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py index 2ab044fbdf47d..5cadba75327a6 100644 --- a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py @@ -42,5 +42,5 @@ def test_write(self): message1 = self._record("mystream", "text data 1", 1) message2 = self._record("mystream", "text data 2", 2) - outcome = list(DestinationAstra().write(self.config, catalog, [message1, message2])) + list(DestinationAstra().write(self.config, catalog, [message1, message2])) assert astra_client.count_documents(db_creds.collection) == 2 diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py index 312956df50060..e9f4e04097fc8 100644 --- a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py @@ -1,7 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import ANY, MagicMock, Mock, patch +from unittest.mock import ANY, MagicMock, Mock import pytest import urllib3 diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py index facbeb3344c9c..d23644f26d142 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py @@ -144,10 +144,10 @@ def write( def _safe_write(*, con: duckdb.DuckDBPyConnection, buffer: Dict[str, Dict[str, List[Any]]], schema_name: str, stream_name: str): table_name = f"_airbyte_raw_{stream_name}" try: - pa_table = pa.Table.from_pydict(buffer[stream_name]) + pa.Table.from_pydict(buffer[stream_name]) except: logger.exception( - f"Writing with pyarrow view failed, falling back to writing with executemany. Expect some performance degradation." + "Writing with pyarrow view failed, falling back to writing with executemany. Expect some performance degradation." ) query = f""" INSERT INTO {schema_name}.{table_name} diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py index b3320efb79fee..8a65e5a60cc3e 100644 --- a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py @@ -7,7 +7,6 @@ import random import string import tempfile -import time from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, Generator, Iterable diff --git a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py index 7a2616579d570..70423f9be58ea 100644 --- a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py +++ b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py @@ -1,7 +1,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. from __future__ import annotations -import os import tempfile from unittest.mock import Mock, patch @@ -22,7 +21,7 @@ def test_validated_sql_name() -> None: def test_check(mock_connect, mock_makedirs) -> None: mock_connect.return_value.execute.return_value = True logger = Mock() - temp_dir = tempfile.mkdtemp() + tempfile.mkdtemp() config = {"destination_path": "/local/test"} # config = {"destination_path": f"{temp_dir}/testdb.db"} destination = DestinationDuckdb() diff --git a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py index 9ae7e741e9bd1..55b8663acced3 100644 --- a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py @@ -12,7 +12,6 @@ from typing import Any, Callable, Dict, Generator, Iterable from unittest.mock import MagicMock -import duckdb import pytest from destination_motherduck import DestinationMotherDuck from destination_motherduck.destination import CONFIG_MOTHERDUCK_API_KEY diff --git a/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py b/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py index c062b05670530..cdd59db9e4045 100644 --- a/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py +++ b/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py @@ -21,7 +21,7 @@ def test_validated_sql_name() -> None: def test_check(mock_connect, mock_makedirs) -> None: mock_connect.return_value.execute.return_value = True logger = Mock() - temp_dir = tempfile.mkdtemp() + tempfile.mkdtemp() config = {"destination_path": "/local/test"} # config = {"destination_path": f"{temp_dir}/testdb.db"} destination = DestinationMotherDuck() diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py index b51c3ba18131d..5a68b25913f7c 100644 --- a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py @@ -19,7 +19,6 @@ from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, - AirbyteStateMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py index 3fb46fa23a8fc..d256cc68cd421 100644 --- a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py @@ -11,7 +11,6 @@ from destination_pinecone.indexer import PineconeIndexer from pinecone import IndexDescription, exceptions from pinecone.grpc import PineconeGRPC -from pinecone.models import IndexList from airbyte_cdk.models import ConfiguredAirbyteCatalog @@ -19,7 +18,7 @@ def create_pinecone_indexer(embedding_dimensions=3, side_effect=None): config = PineconeIndexingModel(mode="pinecone", pinecone_environment="myenv", pinecone_key="mykey", index="myindex") - with patch.object(PineconeGRPC, "Index") as mock_index: + with patch.object(PineconeGRPC, "Index"): indexer = PineconeIndexer(config, 3) indexer.pc.list_indexes = MagicMock() diff --git a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/run.py b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/run.py index 88382951b2208..9073a0a3ed980 100644 --- a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/run.py +++ b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/run.py @@ -5,7 +5,6 @@ import sys -from airbyte_cdk.entrypoint import launch from destination_timeplus import DestinationTimeplus diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py index a46a518bea0fe..e820aa412b2a3 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py @@ -181,7 +181,7 @@ def index_documents(self, documents): response.get("status").get("code") == "OK" or response.get("status").get("statusDetail") == "Document should have at least one part." ) - except AssertionError as e: + except AssertionError: # Handle the assertion error pass else: diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py index c0429365569e5..4ff0fd408118a 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py @@ -3,10 +3,7 @@ # -from unittest.mock import MagicMock - import pytest -import requests from source_airtable import SourceAirtable from airbyte_cdk.models import FailureType, SyncMode @@ -30,7 +27,7 @@ def test_pagination(self, tables_requests_mock, airtable_streams_with_pagination def test_read_records_403_error(self, tables_requests_mock, airtable_streams_403_status_code_requests_mock): with pytest.raises(MessageRepresentationAirbyteTracedErrors) as exc_info: stream = SourceAirtable(catalog={}, config=self.config, state={}).streams(config=self.config)[0] - data = list(stream.read_records(sync_mode=SyncMode.full_refresh)) + list(stream.read_records(sync_mode=SyncMode.full_refresh)) assert exc_info.value.failure_type == FailureType.config_error assert exc_info.value.message == "Permission denied or entity is unprocessable." diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_v3_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_v3_request_builder.py index 0987e20379b80..929e7a2ced703 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_v3_request_builder.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_brands_report_v3_request_builder.py @@ -21,7 +21,7 @@ def _init_report_endpoint( report_date: Optional[str] = None, ) -> "SponsoredBrandsV3ReportRequestBuilder": return ( - cls(f"reporting/reports") + cls("reporting/reports") .with_client_id(client_id) .with_client_access_token(client_access_token) .with_profile_id(profile_id) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_display_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_display_report_request_builder.py index 93a52c8e80bf4..3aaa010511d02 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_display_report_request_builder.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_display_report_request_builder.py @@ -21,7 +21,7 @@ def _init_report_endpoint( report_date: Optional[str] = None, ) -> "SponsoredDisplayReportRequestBuilder": return ( - cls(f"reporting/reports") + cls("reporting/reports") .with_client_id(client_id) .with_client_access_token(client_access_token) .with_profile_id(profile_id) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_products_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_products_report_request_builder.py index ce5c4dd52b06c..5900326e88e12 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_products_report_request_builder.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/sponsored_products_report_request_builder.py @@ -21,7 +21,7 @@ def _init_report_endpoint( report_date: Optional[str] = None, ) -> "SponsoredProductsReportRequestBuilder": return ( - cls(f"reporting/reports") + cls("reporting/reports") .with_client_id(client_id) .with_client_access_token(client_access_token) .with_profile_id(profile_id) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_download_response_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_download_response_builder.py index 93f08ef88ec19..713a1002d2e4d 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_download_response_builder.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_responses/report_download_response_builder.py @@ -1,7 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import gzip -import json from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, find_template diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py index 96874e60649ba..ef51bdc8a4499 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py @@ -10,7 +10,6 @@ from airbyte_cdk.test.mock_http.response_builder import ( FieldPath, HttpResponseBuilder, - NestedPath, PaginationStrategy, RecordBuilder, create_record_builder, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index 021d619d692ad..3904b1372aae9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -4,7 +4,6 @@ from typing import Any, List, Mapping, Optional, Tuple -import pendulum from airbyte_protocol_dataclasses.models import ConfiguredAirbyteCatalog from airbyte_cdk import TState diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py index 306425e085607..e176abe2eb28b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import logging -import time from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py index 425b2cbf4017a..811706e01dc0b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -4,11 +4,8 @@ import json -import os -import tempfile from pathlib import Path from typing import Any, Mapping -from unittest import TestCase import pytest from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions, MigrateStreamNameOption diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index cf831cd45cb16..692a44ad48d76 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -4,12 +4,9 @@ import logging -from unittest.mock import patch -import freezegun import pytest from source_amazon_seller_partner import SourceAmazonSellerPartner -from source_amazon_seller_partner.components import AmazonSPOauthAuthenticator from source_amazon_seller_partner.utils import AmazonConfigException from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/components.py b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/components.py index de75871ebbd8e..31eb3412ad597 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/components.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/components.py @@ -57,7 +57,7 @@ def __call__(self, request: requests.PreparedRequest) -> requests.PreparedReques try: body_json = json.loads(request.body.decode("utf-8")) if request.body else {} - except json.JSONDecodeError as e: + except json.JSONDecodeError: body_json = {} body_json["QueueUrl"] = payload["QueueUrl"] body_json["AttributeNames"] = payload["AttributeNames"] diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py index 63c39b3f65be4..40f5623f71ea3 100755 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py @@ -48,7 +48,7 @@ def config(): @pytest.fixture(scope="module") def streams(config): - catalog = ( + ( CatalogBuilder() .with_stream("annotations_stream", sync_mode=SyncMode.full_refresh) .with_stream("cohorts_stream", sync_mode=SyncMode.full_refresh) diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/components.py b/airbyte-integrations/connectors/source-aws-cloudtrail/components.py index b99a2a62f3012..4eca46b8ba6c0 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/components.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/components.py @@ -6,7 +6,7 @@ import hashlib import hmac import json -from dataclasses import InitVar, dataclass +from dataclasses import dataclass from typing import Any, Mapping, Union import requests diff --git a/airbyte-integrations/connectors/source-braze/components.py b/airbyte-integrations/connectors/source-braze/components.py index 636a4ffc5e56c..a3968d7db59f8 100644 --- a/airbyte-integrations/connectors/source-braze/components.py +++ b/airbyte-integrations/connectors/source-braze/components.py @@ -2,8 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime -import operator from dataclasses import dataclass, field from typing import Any, Iterable, Mapping, MutableMapping, Optional diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_components.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_components.py index f23f166518969..f13fefead5116 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_components.py @@ -3,8 +3,6 @@ # import pytest -from airbyte_cdk.sources.types import Record, StreamSlice - @pytest.mark.parametrize( "record, expected_record", diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py index 9da2278ef3463..7ea529b162f76 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py @@ -3,7 +3,6 @@ # import logging -from dataclasses import dataclass import backoff import requests diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py index 7fc5a7a0158fa..ec2b6ba2eb8b0 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py @@ -6,7 +6,6 @@ from unittest.mock import MagicMock import pytest -import requests import responses from source_convex.source import ConvexStream diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 8bdbe9ddd9cca..3c30677ab33ea 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -165,7 +165,7 @@ def _job_start_request( "action_attribution_windows": ["1d_click", "7d_click", "28d_click", "1d_view", "7d_view", "28d_view"], "filtering": [ { - "field": f"ad.effective_status", + "field": "ad.effective_status", "operator": "IN", "value": [ "ACTIVE", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py index cd6702add9f15..d00723dfc7baa 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py @@ -91,7 +91,7 @@ def test_ads_stream(self, http_mocker: HttpMocker): output = self._read(config().with_ad_statuses(self.statuses), "ads") assert len(output.records) == 1 account_state = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state")[self.account_id] - assert self.filter_statuses_flag in account_state, f"State should include `filter_statuses` flag to track new records in the past." + assert self.filter_statuses_flag in account_state, "State should include `filter_statuses` flag to track new records in the past." assert account_state == {"filter_statuses": self.statuses, "updated_time": "2023-03-21T22:41:46-0700"} @HttpMocker() @@ -141,7 +141,7 @@ def test_campaigns_stream(self, http_mocker: HttpMocker): assert len(output.records) == 1 account_state = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state")[self.account_id] - assert self.filter_statuses_flag in account_state, f"State should include `filter_statuses` flag to track new records in the past." + assert self.filter_statuses_flag in account_state, "State should include `filter_statuses` flag to track new records in the past." assert account_state == {"filter_statuses": self.statuses, "updated_time": "2024-03-12T15:02:47-0700"} @HttpMocker() @@ -185,5 +185,5 @@ def test_ad_sets_stream(self, http_mocker: HttpMocker): assert len(output.records) == 1 account_state = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state")[self.account_id] - assert self.filter_statuses_flag in account_state, f"State should include `filter_statuses` flag to track new records in the past." + assert self.filter_statuses_flag in account_state, "State should include `filter_statuses` flag to track new records in the past." assert account_state == {"filter_statuses": self.statuses, "updated_time": "2024-03-02T15:02:47-0700"} diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py index 326f6475326e8..6b69470a042f9 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py @@ -8,7 +8,7 @@ import pandas as pd import pytest -from pandas import read_csv, read_excel, testing +from pandas import read_csv, read_excel from paramiko import SSHException from source_file.client import Client, URLFile from source_file.utils import backoff_handler diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py index 639577406d69f..56c9e9efabac8 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py @@ -3,7 +3,6 @@ import shutil import time import uuid -from typing import Any, Mapping import docker import google diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py index eab24e5e8d145..49386c6716ef4 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py @@ -1,6 +1,5 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -from os import write from source_gcs import Config diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py index 7047f82d93175..72816227c956f 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_helpers.py @@ -1,6 +1,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -from unittest.mock import Mock, patch +from unittest.mock import Mock from source_gcs.helpers import get_stream_name diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_legacy_config_transformer.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_legacy_config_transformer.py index 77734d09bd06d..280584a7c3115 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_legacy_config_transformer.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_legacy_config_transformer.py @@ -2,8 +2,7 @@ from unittest.mock import Mock, patch -from pyarrow import MockOutputStream -from source_gcs import LegacyConfigTransformer, helpers +from source_gcs import LegacyConfigTransformer from source_gcs.spec import SourceGCSSpec diff --git a/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py b/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py index ba36daec8ced9..caef6d600f8e9 100644 --- a/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py +++ b/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py @@ -7,7 +7,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction +from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index cc8d8d8ee2b2a..0fed7f62ba058 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -15,9 +15,9 @@ from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.checkpoint.substream_resumable_full_refresh_cursor import SubstreamResumableFullRefreshCursor -from airbyte_cdk.sources.streams.core import CheckpointMixin, Stream +from airbyte_cdk.sources.streams.core import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction +from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index 1ee7c865bbfb2..6919f19cd4436 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -52,7 +52,7 @@ from source_github.utils import read_full_refresh from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode -from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction +from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, ResponseAction from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException, UserDefinedBackoffException from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py index de602c5991481..7c9d4cf09e0ac 100644 --- a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py +++ b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import unittest -from json import dumps, load +from json import load from typing import Dict from components import IncrementalSingleBodyFilterCursor diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py index 61990be3b7b95..7e2c44cf33aab 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py @@ -5,7 +5,7 @@ from typing import Any, List, Mapping -from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message +from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.models import FailureType from airbyte_cdk.sources import Source diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py index 1ed3e035984c7..c21eaf1cd9b4a 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import copy import datetime import json import logging @@ -575,7 +574,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if hasattr(ex, "failure_type") and ex.failure_type == FailureType.config_error: # bad request and forbidden are set in mapper as config errors raise ex - logger.error(f"Check failed", exc_info=ex) + logger.error("Check failed", exc_info=ex) if not metadata: return False, "Failed to get metadata, over quota, try later" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py index aac51ada0dcc5..b35474b7a4ffe 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py @@ -7,7 +7,6 @@ import os from typing import Any, Mapping -import dpath.util from source_google_analytics_data_api.config_migrations import MigrateCustomReportsCohortSpec from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py index 97cb79d8b64de..d32ca41c52907 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py @@ -14,7 +14,7 @@ from requests.models import Response from source_google_analytics_data_api.source import GoogleAnalyticsDataApiBaseStream, SourceGoogleAnalyticsDataApi -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from .utils import read_incremental diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py index 39d7a1bfafdb4..a013909904923 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py @@ -22,7 +22,7 @@ from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from source_google_drive.utils import get_folder_id from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata @@ -266,7 +266,7 @@ def file_size(self, file: GoogleDriveRemoteFile) -> int: file_metadata = self.google_drive_service.files().get(fileId=file.id, fields="size", supportsAllDrives=True).execute() return int(file_metadata["size"]) except KeyError: - raise ErrorFetchingMetadata(f"Size was expected in metadata response but was missing") + raise ErrorFetchingMetadata("Size was expected in metadata response but was missing") except Exception as e: raise ErrorFetchingMetadata(f"An error occurred while retrieving file size: {str(e)}") diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/components/extractors.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/components/extractors.py index 54b8f8d9be72a..d811a2e41493c 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/components/extractors.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/components/extractors.py @@ -12,7 +12,7 @@ from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.types import Config -from source_google_sheets.utils import name_conversion, safe_name_conversion +from source_google_sheets.utils import safe_name_conversion class RawSchemaParser: diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/conftest/entrypoint_wrapper_helper.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/conftest/entrypoint_wrapper_helper.py index 880bc9712d5bf..4306448a028aa 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/conftest/entrypoint_wrapper_helper.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/conftest/entrypoint_wrapper_helper.py @@ -7,7 +7,7 @@ from typing import Any, Mapping from airbyte_cdk.sources import Source -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, _run_command, make_file, read +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, _run_command, make_file def check( diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py index 74fc489b97d71..1558e3f0184f4 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py @@ -2,16 +2,13 @@ # Copyright (c) 2025 Airbyte, Inc., all rights reserved. # -import json from copy import deepcopy from unittest.mock import ANY import pytest -from requests.status_codes import codes as status_codes from airbyte_cdk.models import ( AirbyteCatalog, - AirbyteConnectionStatus, AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, @@ -19,10 +16,8 @@ AirbyteStream, AirbyteStreamStatusTraceMessage, AirbyteTraceMessage, - ConfiguredAirbyteCatalog, FailureType, Level, - Status, StreamDescriptor, SyncMode, TraceType, @@ -30,7 +25,7 @@ ) from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, AirbyteStreamStatus from airbyte_cdk.test.catalog_builder import CatalogBuilder, ConfiguredAirbyteStreamBuilder -from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import find_template from .conftest import GoogleSheetsBaseTest, oauth_credentials, service_account_credentials @@ -67,7 +62,7 @@ def test_check_expected_to_read_data_from_1_sheet(self, http_mocker: HttpMocker) TestSourceCheck.get_spreadsheet_info_and_sheets(http_mocker, "check_succeeded_meta", 200) TestSourceCheck.get_sheet_first_row(http_mocker, "check_wrong_range", 200) error_message = ( - f"Unable to read the schema of sheet. Error: Unexpected return result: Sheet was expected to contain data on exactly 1 sheet." + "Unable to read the schema of sheet. Error: Unexpected return result: Sheet was expected to contain data on exactly 1 sheet." ) output = self._check(self._config, expecting_exception=True) trace_message = AirbyteTraceMessage( @@ -89,9 +84,9 @@ def test_check_duplicated_headers(self, http_mocker: HttpMocker) -> None: TestSourceCheck.get_spreadsheet_info_and_sheets(http_mocker, "check_succeeded_meta", 200) TestSourceCheck.get_sheet_first_row(http_mocker, "check_duplicate_headers", 200) - error_message = f"The following duplicate headers were found in the sheet. Please fix them to continue: ['header1']" + error_message = "The following duplicate headers were found in the sheet. Please fix them to continue: ['header1']" output = self._check(self._config, expecting_exception=True) - trace_message = AirbyteTraceMessage( + AirbyteTraceMessage( type=TraceType.ERROR, emitted_at=ANY, error=AirbyteErrorTraceMessage( @@ -168,7 +163,7 @@ def test_discover_empty_column_return_expected_schema(self, http_mocker: HttpMoc _STREAM_NAME: {"name": {"type": ["null", "string"]}, "age": {"type": ["null", "string"]}}, } GoogleSheetsBaseTest.get_spreadsheet_info_and_sheets(http_mocker, "discover_with_empty_column_spreadsheet_info_and_sheets", 200) - GoogleSheetsBaseTest.get_sheet_first_row(http_mocker, f"discover_with_empty_column_get_sheet_first_row", 200) + GoogleSheetsBaseTest.get_sheet_first_row(http_mocker, "discover_with_empty_column_get_sheet_first_row", 200) expected_streams = [] for expected_stream_name, expected_stream_properties in expected_schemas_properties.items(): @@ -746,7 +741,7 @@ def test_read_empty_sheet(self, http_mocker: HttpMocker) -> None: output = self._read(self._config, catalog=configured_catalog, expecting_exception=True) expected_message = ( - f"Unable to read the schema of sheet. Error: Unexpected return result: Sheet was expected to contain data on exactly 1 sheet." + "Unable to read the schema of sheet. Error: Unexpected return result: Sheet was expected to contain data on exactly 1 sheet." ) assert output.errors[0].trace.error.message == expected_message @@ -767,7 +762,7 @@ def test_read_expected_data_on_1_sheet(self, http_mocker: HttpMocker) -> None: output = self._read(self._config, catalog=configured_catalog, expecting_exception=True) expected_message = ( - f"Unable to read the schema of sheet. Error: Unexpected return result: Sheet was expected to contain data on exactly 1 sheet." + "Unable to read the schema of sheet. Error: Unexpected return result: Sheet was expected to contain data on exactly 1 sheet." ) assert output.errors[0].trace.error.message == expected_message diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py index 463a456b4aaab..3e6fdf887ba36 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock import pytest from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 195ac0335d213..21fbf5b5b1cc3 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -3,10 +3,9 @@ # import logging -import traceback from http import HTTPStatus from itertools import chain -from typing import Any, Generator, List, Mapping, Optional, Tuple, Union +from typing import Any, Generator, List, Mapping, Optional, Tuple from requests import HTTPError diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py index b5c2cc79e61bd..8e5c68932bf91 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py @@ -2,7 +2,7 @@ import json from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py index fdf05847617cd..eb2d4be187e12 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py @@ -2,8 +2,6 @@ from unittest import TestCase -import freezegun - from airbyte_cdk.test.mock_http import HttpMocker from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py index 3c21d185a6502..b97a8162157ff 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py @@ -3,8 +3,6 @@ from datetime import datetime, timedelta, timezone from unittest import TestCase -import freezegun - from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import SyncMode diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py index 8b76e13cc6c45..a6e7eb20060ac 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py @@ -2,8 +2,6 @@ from unittest import TestCase -import freezegun - from airbyte_cdk.test.mock_http import HttpMocker from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 2033d84ab4338..9a802f87ba70a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -503,8 +503,6 @@ def test_search_based_incremental_stream_should_sort_by_id(requests_mock, common def random_date(start, end): return pendulum.from_timestamp(random.randint(start, end) / 1000).to_iso8601_string() - after = 0 - # Custom callback to mock search endpoint filter and sort behavior, returns 100 records per request. # See _process_search in stream.py for details on the structure of the filter amd sort parameters. # The generated records will have an id that is the sum of the current id and the current "after" value @@ -795,7 +793,7 @@ def test_get_granted_scopes_retry(requests_mock, mocker): ] requests_mock.register_uri("GET", mock_url, response) - actual_scopes = SourceHubspot().get_granted_scopes(authenticator) + SourceHubspot().get_granted_scopes(authenticator) assert len(requests_mock.request_history) > 1 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index 064f98512101c..0f86f21c226b3 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -2,8 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json -from unittest.mock import patch import pendulum import pytest diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py index ef3ea86c51dbb..12e0aa95cf245 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py @@ -5,7 +5,7 @@ from __future__ import annotations -from typing import Any, List, MutableMapping +from typing import Any, MutableMapping ACCESS_TOKEN = "test_access_token" diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py index 218692c282f4b..a19fb6066cc89 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py @@ -15,9 +15,9 @@ ) from airbyte_protocol.models import SyncMode -from .config import BUSINESS_ACCOUNT_ID, ConfigBuilder +from .config import ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, InstagramPaginationStrategy -from .request_builder import RequestBuilder, get_account_request +from .request_builder import get_account_request from .response_builder import get_account_response from .utils import config, read_output diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py index 09e8331291558..72b292b321f24 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py @@ -2,11 +2,8 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # import json -import unittest from unittest import TestCase -import pytest - from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import ( diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py index 5e71182a23499..4460f9778e817 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py @@ -2,11 +2,8 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # import json -import unittest from unittest import TestCase -import pytest - from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import ( diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 74fc743575246..aa9aeebd2cc78 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from datetime import datetime from unittest.mock import MagicMock import pytest @@ -83,7 +82,6 @@ def test_user_insights_state(api, user_insights, values, slice_dates, expected): """ This test shows how `STATE` is managed based on the scenario for Incremental Read. """ - import pendulum # UserInsights stream stream = UserInsights(api=api, start_date="2023-01-01T01:01:01Z") diff --git a/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py b/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py index 0019904673281..2aa8477db96b5 100644 --- a/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py @@ -6,7 +6,6 @@ import pytest -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py index 682607c649339..26422ccdd425e 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py @@ -2,14 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import patch import pytest import requests -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig -from airbyte_cdk.sources.streams import Stream - @pytest.mark.parametrize( "rate_limit_header, backoff_time", diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py index bf39a54ae053c..01251c595acbf 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import json import os -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from typing import Any, Dict from unittest import TestCase @@ -18,7 +18,6 @@ RecordBuilder, create_record_builder, create_response_builder, - find_template, ) from airbyte_cdk.test.state_builder import StateBuilder from integration.config import ConfigBuilder diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index fd4f10dfd98ac..933d8f25e0bb1 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -964,7 +964,7 @@ def test_project_versions_stream(config, mock_non_deleted_projects_responses, pr ) authenticator = SourceJira(config=config, catalog=None, state=None).get_authenticator(config=config) - args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = find_stream("project_versions", config) records = list(read_full_refresh(stream)) diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py new file mode 100644 index 0000000000000..2d4fe4c8da8e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Mapping, Optional + +from airbyte_cdk import TState +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + + +class SourceKlaviyo(YamlDeclarativeSource): + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: TState, **kwargs): + super().__init__(catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py new file mode 100644 index 0000000000000..4f9c08cc02d19 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py @@ -0,0 +1,96 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import datetime +from typing import Any, Dict, Optional +from unittest import TestCase + +from integration.config import KlaviyoConfigBuilder +from source_klaviyo import SourceKlaviyo + +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) + + +_ENDPOINT_TEMPLATE_NAME = "profiles" +_START_DATE = datetime.datetime(2021, 1, 1, tzinfo=datetime.timezone.utc) +_STREAM_NAME = "profiles" +_RECORDS_PATH = FieldPath("data") + + +def _config() -> KlaviyoConfigBuilder: + return KlaviyoConfigBuilder().with_start_date(_START_DATE) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _a_profile_request(start_date: datetime) -> HttpRequest: + return HttpRequest( + url="https://a.klaviyo.com/api/profiles", + query_params={ + "additional-fields[profile]": "predictive_analytics", + "page[size]": "100", + "filter": f"greater-than(updated,{start_date.strftime('%Y-%m-%dT%H:%M:%S%z')})", + "sort": "updated", + }, + ) + + +def _a_profile() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + record_id_path=FieldPath("id"), + record_cursor_path=NestedPath(["attributes", "updated"]), + ) + + +def _profiles_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + _RECORDS_PATH, + ) + + +def _read( + config_builder: KlaviyoConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(SourceKlaviyo(catalog, config, state), config, catalog, state, expecting_exception) + + +class FullRefreshTest(TestCase): + @HttpMocker() + def test_when_read_then_extract_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_profile_request(_START_DATE), + _profiles_response().with_record(_a_profile()).build(), + ) + + output = _read(_config(), SyncMode.full_refresh) + + assert len(output.records) == 1 + + @HttpMocker() + def test_given_region_is_number_when_read_then_cast_as_string(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_profile_request(_START_DATE), + _profiles_response().with_record(_a_profile().with_field(NestedPath(["attributes", "location", "region"]), 10)).build(), + ) + + output = _read(_config(), SyncMode.full_refresh) + + assert len(output.records) == 1 + assert isinstance(output.records[0].record.data["attributes"]["location"]["region"], str) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py new file mode 100644 index 0000000000000..893de10b45e4d --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py @@ -0,0 +1,256 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import urllib.parse +from datetime import datetime +from typing import Any, List, Mapping, Optional + +import pendulum +import pytest +import requests +from dateutil.relativedelta import relativedelta +from freezegun import freeze_time +from integration.config import KlaviyoConfigBuilder +from source_klaviyo.source import SourceKlaviyo + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.test.catalog_builder import CatalogBuilder, ConfiguredAirbyteStreamBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.state_builder import StateBuilder + + +_ANY_ATTEMPT_COUNT = 123 +API_KEY = "some_key" +START_DATE = pendulum.datetime(2020, 10, 10) +CONFIG = {"api_key": API_KEY, "start_date": START_DATE} +CONFIG_NO_DATE = {"api_key": API_KEY} + +EVENTS_STREAM_DEFAULT_START_DATE = "2012-01-01T00:00:00+00:00" +EVENTS_STREAM_CONFIG_START_DATE = "2021-11-08T00:00:00+00:00" +EVENTS_STREAM_STATE_DATE = (datetime.fromisoformat(EVENTS_STREAM_CONFIG_START_DATE) + relativedelta(years=1)).isoformat() +EVENTS_STREAM_TESTING_FREEZE_TIME = "2023-12-12 12:00:00" + + +def get_step_diff(provided_date: str) -> int: + """ + This function returns the difference in weeks between provided date and freeze time. + """ + provided_date = datetime.fromisoformat(provided_date).replace(tzinfo=None) + freeze_date = datetime.strptime(EVENTS_STREAM_TESTING_FREEZE_TIME, "%Y-%m-%d %H:%M:%S") + return (freeze_date - provided_date).days // 7 + + +def read_records(stream_name: str, config: Mapping[str, Any], states: Mapping[str, Any] = dict()) -> List[Mapping[str, Any]]: + state = StateBuilder() + for stream_name_key in states: + state.with_stream_state(stream_name_key, states[stream_name_key]) + source = SourceKlaviyo(CatalogBuilder().build(), config, state.build()) + output = read( + source, + config, + CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(stream_name)).build(), + ) + return [r.record.data for r in output.records] + + +def get_stream_by_name(stream_name: str, config: Mapping[str, Any], states: Mapping[str, Any] = dict()) -> Stream: + state = StateBuilder() + for stream_name_key in states: + state.with_stream_state(stream_name_key, states[stream_name_key]) + source = SourceKlaviyo(CatalogBuilder().build(), KlaviyoConfigBuilder().build(), state.build()) + matches_by_name = [stream_config for stream_config in source.streams(config) if stream_config.name == stream_name] + if not matches_by_name: + raise ValueError("Please provide a valid stream name.") + return matches_by_name[0] + + +def get_records(stream: Stream, sync_mode: Optional[SyncMode] = SyncMode.full_refresh) -> List[Mapping[str, Any]]: + records = [] + for stream_slice in stream.stream_slices(sync_mode=sync_mode): + for record in stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice): + records.append(dict(record)) + return records + + +@pytest.fixture(name="response") +def response_fixture(mocker): + return mocker.Mock(spec=requests.Response) + + +class TestSemiIncrementalKlaviyoStream: + @pytest.mark.parametrize( + ("start_date", "stream_state", "input_records", "expected_records"), + ( + ( + "2021-11-08T00:00:00Z", + "2022-11-07T00:00:00+00:00", + [ + {"attributes": {"updated": "2022-11-08T00:00:00+00:00"}}, + {"attributes": {"updated": "2023-11-08T00:00:00+00:00"}}, + {"attributes": {"updated": "2021-11-08T00:00:00+00:00"}}, + ], + [ + {"attributes": {"updated": "2022-11-08T00:00:00+00:00"}, "updated": "2022-11-08T00:00:00+00:00"}, + {"attributes": {"updated": "2023-11-08T00:00:00+00:00"}, "updated": "2023-11-08T00:00:00+00:00"}, + ], + ), + ( + "2021-11-09T00:00:00Z", + None, + [ + {"attributes": {"updated": "2022-11-08T00:00:00+00:00"}}, + {"attributes": {"updated": "2023-11-08T00:00:00+00:00"}}, + {"attributes": {"updated": "2021-11-08T00:00:00+00:00"}}, + ], + [ + {"attributes": {"updated": "2022-11-08T00:00:00+00:00"}, "updated": "2022-11-08T00:00:00+00:00"}, + {"attributes": {"updated": "2023-11-08T00:00:00+00:00"}, "updated": "2023-11-08T00:00:00+00:00"}, + ], + ), + ("2021-11-08T00:00:00Z", "2022-11-07T00:00:00+00:00", [], []), + ), + ) + def test_read_records(self, start_date, stream_state, input_records, expected_records, requests_mock): + state = {"metrics": {"updated": stream_state}} if stream_state else {} + requests_mock.register_uri("GET", "https://a.klaviyo.com/api/metrics", status_code=200, json={"data": input_records}) + records = read_records("metrics", CONFIG_NO_DATE | {"start_date": start_date}, state) + assert records == expected_records + + +class TestProfilesStream: + def test_read_records(self, requests_mock): + stream = get_stream_by_name("profiles", CONFIG) + json = { + "data": [ + { + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "attributes": {"email": "name@airbyte.io", "updated": "2023-03-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_complete"}, + }, + { + "type": "profile", + "id": "AAAA1A1AA1AA111AAAAAAA1AA1", + "attributes": {"email": "name2@airbyte.io", "updated": "2023-02-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_started"}, + }, + ], + } + requests_mock.register_uri("GET", "https://a.klaviyo.com/api/profiles", status_code=200, json=json) + + records = get_records(stream=stream) + assert records == [ + { + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "updated": "2023-03-10T20:36:36+00:00", + "attributes": {"email": "name@airbyte.io", "updated": "2023-03-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_complete"}, + }, + { + "type": "profile", + "id": "AAAA1A1AA1AA111AAAAAAA1AA1", + "updated": "2023-02-10T20:36:36+00:00", + "attributes": {"email": "name2@airbyte.io", "updated": "2023-02-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_started"}, + }, + ] + + +class TestGlobalExclusionsStream: + def test_read_records(self, requests_mock): + stream = get_stream_by_name("global_exclusions", CONFIG) + json = { + "data": [ + { + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "attributes": { + "updated": "2023-03-10T20:36:36+00:00", + "subscriptions": {"email": {"marketing": {"suppression": [{"reason": "SUPPRESSED"}]}}}, + }, + }, + { + "type": "profile", + "id": "AAAA1A1AA1AA111AAAAAAA1AA1", + "attributes": {"updated": "2023-02-10T20:36:36+00:00"}, + }, + ], + } + requests_mock.register_uri("GET", "https://a.klaviyo.com/api/profiles", status_code=200, json=json) + + records = get_records(stream=stream) + assert records == [ + { + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "attributes": { + "updated": "2023-03-10T20:36:36+00:00", + "subscriptions": {"email": {"marketing": {"suppressions": [{"reason": "SUPPRESSED"}]}}}, + }, + "updated": "2023-03-10T20:36:36+00:00", + } + ] + + +class TestCampaignsStream: + @freeze_time(pendulum.datetime(2020, 11, 10).isoformat()) + def test_read_records(self, requests_mock): + input_records = { + "sms": { + "true": {"attributes": {"name": "Some name 1", "archived": True, "updated_at": "2020-10-21T00:00:00+0000"}}, + "false": {"attributes": {"name": "Some name 1", "archived": False, "updated_at": "2020-10-20T00:00:00+0000"}}, + }, + "email": { + "true": {"attributes": {"name": "Some name 1", "archived": True, "updated_at": "2020-10-18T00:00:00+0000"}}, + "false": {"attributes": {"name": "Some name 1", "archived": False, "updated_at": "2020-10-23T00:00:00+0000"}}, + }, + } + + stream = get_stream_by_name("campaigns", CONFIG) + expected_records = [ + { + "attributes": {"archived": True, "name": "Some name 1", "updated_at": "2020-10-21T00:00:00+0000", "channel": "sms"}, + "updated_at": "2020-10-21T00:00:00+0000", + }, + { + "attributes": {"archived": False, "name": "Some name 1", "updated_at": "2020-10-20T00:00:00+0000", "channel": "sms"}, + "updated_at": "2020-10-20T00:00:00+0000", + }, + { + "attributes": {"archived": True, "name": "Some name 1", "updated_at": "2020-10-18T00:00:00+0000", "channel": "email"}, + "updated_at": "2020-10-18T00:00:00+0000", + }, + { + "attributes": {"archived": False, "name": "Some name 1", "updated_at": "2020-10-23T00:00:00+0000", "channel": "email"}, + "updated_at": "2020-10-23T00:00:00+0000", + }, + ] + + records = [] + base_url = "https://a.klaviyo.com/api/campaigns" + + for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): + query_params = { + "filter": f"and(greater-or-equal(updated_at,{stream_slice['start_time']}),less-or-equal(updated_at,{stream_slice['end_time']}),equals(messages.channel,'{stream_slice['campaign_type']}'),equals(archived,{stream_slice['archived']}))", + "sort": "updated_at", + } + encoded_query = urllib.parse.urlencode(query_params) + encoded_url = f"{base_url}?{encoded_query}" + requests_mock.register_uri( + "GET", + encoded_url, + status_code=200, + json={"data": input_records[stream_slice["campaign_type"]][stream_slice["archived"]]}, + complete_qs=True, + ) + + for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): + records.append(record) + + assert len(records) == len(expected_records) + for expected_record, record in zip(expected_records, records): + assert expected_record == dict(record) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py index cd8afe7c78a10..217d5a0979abc 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py @@ -12,7 +12,7 @@ import pendulum import requests -from isodate import Duration, parse_duration +from isodate import Duration from requests.exceptions import InvalidURL from airbyte_cdk.models import FailureType @@ -31,11 +31,11 @@ from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.transformations import AddFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition -from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import StreamSlice from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, ResponseAction -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException +from airbyte_cdk.sources.streams.http.exceptions import RequestBodyException from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS from .utils import ANALYTICS_FIELDS_V2, FIELDS_CHUNK_SIZE, transform_data diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py index 3ac0fae777714..a6382bdaf54e7 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import pytest -from requests import Response, Session +from requests import Response from requests.models import PreparedRequest from source_linkedin_ads.components import ( AnalyticsDatetimeBasedCursor, diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py index 177193dfe6d1c..82a4aaa3ebcd3 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py @@ -14,7 +14,7 @@ from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py index 32761a344de52..76fc2b7a6053e 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py @@ -79,7 +79,7 @@ def refresh_access_token(self) -> Tuple[str, int]: response.raise_for_status() response_json = response.json() return response_json[self.access_token_name], response_json[self.expires_in_name], response_json[self.server_name] - except Exception as e: + except Exception: try: e = Exception(response.json()["Message"]) except Exception: diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py index b457c6beede88..3c4eeb8548af0 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py @@ -23,7 +23,7 @@ from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec -from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata +from .exceptions import ErrorFetchingMetadata from .utils import FolderNotFoundException, MicrosoftSharePointRemoteFile, execute_query_with_retry, filter_http_urls @@ -279,7 +279,7 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo if not items_processed: raise AirbyteTracedException( - message=f"Drive is empty or does not exist.", + message="Drive is empty or does not exist.", failure_type=FailureType.config_error, ) @@ -335,7 +335,7 @@ def file_size(self, file: MicrosoftSharePointRemoteFile) -> int: response.raise_for_status() return int(response.headers["Content-Length"]) except KeyError: - raise ErrorFetchingMetadata(f"Size was expected in metadata response but was missing") + raise ErrorFetchingMetadata("Size was expected in metadata response but was missing") except Exception as e: raise ErrorFetchingMetadata(f"An error occurred while retrieving file size: {str(e)}") diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py index 683063cf490ad..059444a286514 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py @@ -108,7 +108,7 @@ def test_get_access_token(setup_client_class, has_refresh_token, token_response, if raises_exception: with pytest.raises(AirbyteTracedException) as exception: instance._get_access_token() - assert exception.value.message == f"Failed to acquire access token. Error: test_error. Error description: test_error_description." + assert exception.value.message == "Failed to acquire access token. Error: test_error. Error description: test_error_description." else: assert instance._get_access_token() == expected_result @@ -249,8 +249,8 @@ def test_get_file(mock_requests_head, mock_requests_get, mock_get_access_token, @patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.get_access_token") @patch("source_microsoft_sharepoint.stream_reader.requests.head") def test_get_file_size_error_fetching_metadata_for_missing_header(mock_requests_head, mock_get_access_token): - file_uri = f"https://my_favorite_sharepoint.sharepoint.com/Shared%20Documents/file.txt" - mock_file = Mock(download_url=f"https://example.com/file.txt", uri=file_uri) + file_uri = "https://my_favorite_sharepoint.sharepoint.com/Shared%20Documents/file.txt" + mock_file = Mock(download_url="https://example.com/file.txt", uri=file_uri) mock_logger = Mock() mock_get_access_token.return_value = "dummy_access_token" @@ -272,8 +272,8 @@ def test_get_file_size_error_fetching_metadata(mock_requests_head, mock_get_acce """ Test that the get_file method raises an ErrorFetchingMetadata exception when the requests.head call fails. """ - file_uri = f"https://my_favorite_sharepoint.sharepoint.com/Shared%20Documents/file.txt" - mock_file = Mock(download_url=f"https://example.com/file.txt", uri=file_uri) + file_uri = "https://my_favorite_sharepoint.sharepoint.com/Shared%20Documents/file.txt" + mock_file = Mock(download_url="https://example.com/file.txt", uri=file_uri) mock_logger = Mock() mock_get_access_token.return_value = "dummy_access_token" @@ -571,7 +571,7 @@ def test_get_shared_drive_object( def test_drives_property(auth_type, user_principal_name, has_refresh_token): with ( patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") as mock_execute_query, - patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client") as mock_one_drive_client, + patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client"), ): refresh_token = "dummy_refresh_token" if has_refresh_token else None # Setup for different authentication types @@ -632,12 +632,12 @@ def test_retrieve_files_from_accessible_drives(mocker, refresh_token, auth_type, reader._config = config # Mock the necessary methods - with patch.object(SourceMicrosoftSharePointStreamReader, "drives", return_value=[]) as mock_drives: + with patch.object(SourceMicrosoftSharePointStreamReader, "drives", return_value=[]): mocker.patch.object(reader, "_get_files_by_drive_name") mocker.patch.object(reader, "_get_shared_files_from_all_drives") # Call the method under test - files = list(reader.get_all_files()) + list(reader.get_all_files()) # Assert that only the desired methods were called assert reader._get_files_by_drive_name.called == ("_get_files_by_drive_name" in expected_methods_called) diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py index 15a3914b2cf6f..387afa7c36eb0 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta from http import HTTPStatus from unittest.mock import Mock, patch -from urllib.parse import parse_qs, urlparse import pytest from source_microsoft_sharepoint.utils import PlaceholderUrlBuilder, execute_query_with_retry, filter_http_urls @@ -52,7 +51,7 @@ def test_execute_query_with_retry(status_code, retry_after_header, expected_retr obj.execute_query = Mock(side_effect=MockException(status_code, {"Retry-After": retry_after_header})) with ( - patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, + patch("source_microsoft_sharepoint.utils.time.sleep"), patch("source_microsoft_sharepoint.utils.datetime") as mock_datetime, ): start_time = datetime(2021, 1, 1, 0, 0, 0) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py index 9216c0cf7eaad..7ad3ba1e3c871 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py @@ -238,7 +238,6 @@ def stream_slices(self) -> Iterable[StreamSlice]: sync_mode=SyncMode.full_refresh, cursor_field=None, stream_state=None ): empty_parent_slice = True - parent_partition = parent_stream_slice.partition if parent_stream_slice else {} for parent_record in parent_stream.read_records( sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py index 815e24e61fa66..3766067d6c1e1 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py @@ -2,12 +2,10 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # -import re from typing import Optional, Union import requests -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py index a47c2611854e7..3e619e23aaa1e 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py @@ -7,7 +7,7 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction +from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, ResponseAction from airbyte_protocol.models import FailureType from .base_errors_handler import MixpanelStreamErrorHandler diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index 9774037a6038d..f4c0e6d09b864 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -15,7 +15,6 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from airbyte_cdk.utils import AirbyteTracedException from .utils import get_url_to_mock, read_incremental, setup_response diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py index 3eeff0eac9fd2..4351e0d0e4e2a 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py @@ -2,14 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from datetime import date, timedelta +from datetime import date import pendulum from source_mixpanel.streams import Export def test_date_slices(): - now = pendulum.today(tz="US/Pacific").date() + pendulum.today(tz="US/Pacific").date() # test with stream_state stream_slices = Export( diff --git a/airbyte-integrations/connectors/source-my-hours/components.py b/airbyte-integrations/connectors/source-my-hours/components.py index 109b493ebd030..a7377e15d539b 100644 --- a/airbyte-integrations/connectors/source-my-hours/components.py +++ b/airbyte-integrations/connectors/source-my-hours/components.py @@ -48,7 +48,7 @@ def token(self) -> str: return self._access_token def _get_refresh_access_token_response(self): - url = f"https://api2.myhours.com/api/tokens/refresh" + url = "https://api2.myhours.com/api/tokens/refresh" headers = {"Content-Type": "application/json", "api-version": "1.0", self.auth_header: f"Bearer {self._access_token}"} data = { diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py index 0fcfbd6ff847d..b35fa9c017dc8 100755 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py @@ -174,7 +174,7 @@ def delete_schemas_with_prefix(conn, date_prefix): cursor.execute(drop_query) print(f"Database {schema[0]} has been dropped.") conn.commit() - except Error as error: + except Error: print(f"An error occurred in deleting schema: {e}") sys.exit(1) diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py index 9b880749910f8..d50fafdda5cac 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py @@ -9,7 +9,7 @@ import pytest import requests -from source_okta.components import CustomBearerAuthenticator, CustomOauth2Authenticator +from source_okta.components import CustomOauth2Authenticator from source_okta.source import SourceOkta from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/product_catalog.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/product_catalog.py index 9ad9a75538140..377363aa17ee9 100755 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/product_catalog.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/product_catalog.py @@ -110,6 +110,6 @@ def update_paypal_product(access_token, product_id, updates): product = update_paypal_product(access_token, args.product_id, operations) print("Updated product:", product) except json.JSONDecodeError: - print(f"Invalid JSON in update payload") + print("Invalid JSON in update payload") else: print("Invalid arguments") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py index 814bfd5b0c434..a1cc2f2af5fb4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py @@ -5,15 +5,11 @@ import base64 import logging from dataclasses import dataclass -from datetime import datetime, timedelta -from typing import Any, Iterable, Mapping, MutableMapping, Optional import backoff import requests from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator -from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester -from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index c71a7fb2a7823..66db0b515ec96 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py index 1bf0f5692aaae..a11de5fb42ab0 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py @@ -2,15 +2,12 @@ import logging import time -from unittest.mock import patch import pytest import requests import requests_mock from source_paypal_transaction.components import PayPalOauth2Authenticator -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException - @pytest.fixture def mock_authenticator(): diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py index bd2f9c89836a8..41339421e55c7 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py @@ -1,7 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # conftest.py -import json from datetime import datetime from unittest.mock import patch diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py index 1166076c79724..785b19b9cde4c 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py @@ -1,7 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import os -import re import pytest import requests diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py index 94045a7e7013a..878ac611ed770 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py @@ -10,7 +10,7 @@ import pendulum import requests -from airbyte_cdk import AirbyteTracedException, BackoffStrategy +from airbyte_cdk import BackoffStrategy from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import WaitTimeFromHeaderBackoffStrategy from airbyte_cdk.sources.streams import Stream @@ -285,7 +285,7 @@ def interpret_response(self, response: Optional[Union[requests.Response, Excepti return ErrorResolution( ResponseAction.RETRY, FailureType.transient_error, - f"Analytics API returns bad request error when under load. This error should be retried after a second. If this error message appears, it means the Analytics API did not recover or there might be a bigger issue so please contact the support team.", + "Analytics API returns bad request error when under load. This error should be retried after a second. If this error message appears, it means the Analytics API did not recover or there might be a bigger issue so please contact the support team.", ) return self._decorated.interpret_response(response) diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py index 6f2f2207cdb8b..fe4f2bfaaefeb 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py @@ -10,8 +10,6 @@ from requests import Response from source_pinterest.python_stream_auth import PinterestOauthAuthenticator -from airbyte_cdk.models import FailureType -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.utils.traced_exception import AirbyteTracedException diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index b1ecc6bfef44b..dd66c51ff2957 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -18,7 +18,6 @@ ) from source_pinterest.utils import get_analytics_columns -from airbyte_cdk import AirbyteTracedException from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction diff --git a/airbyte-integrations/connectors/source-railz/components.py b/airbyte-integrations/connectors/source-railz/components.py index d0d9d96083c53..10cfbb1fdb84b 100644 --- a/airbyte-integrations/connectors/source-railz/components.py +++ b/airbyte-integrations/connectors/source-railz/components.py @@ -5,15 +5,17 @@ import datetime import time from dataclasses import InitVar, dataclass -from typing import Any, Mapping, Union +from typing import Any, Iterable, Mapping, Optional, Union import requests from isodate import Duration, parse_duration +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice @dataclass @@ -80,3 +82,53 @@ def auth_header(self) -> str: def token(self) -> str: self.check_token() return f"Bearer {self._token}" + + +@dataclass +class NestedStateCartesianProductStreamSlicer(CartesianProductStreamSlicer): + """ + [Low-Code Custom Component] NestedStateCartesianProductStreamSlicer + https://github.com/airbytehq/airbyte/issues/22873 + + Some streams require support of nested state: + { + "accounting_transactions": { + "Business1": { + "dynamicsBusinessCentral": { + "postedDate": "2022-12-28T00:00:00.000Z" + } + }, + "Business2": { + "oracleNetsuite": { + "postedDate": "2022-12-28T00:00:00.000Z" + } + } + } + } + """ + + def __post_init__(self, parameters: Mapping[str, Any]): + self._cursor = {} + self._parameters = parameters + + def get_stream_state(self) -> Mapping[str, Any]: + return self._cursor + + def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: + connections_slicer, datetime_slicer = self.stream_slicers + for connection_slice in connections_slicer.stream_slices(sync_mode, stream_state): + businessName = connection_slice["connection"]["businessName"] + serviceName = connection_slice["connection"]["serviceName"] + datetime_slicer._cursor = None + for datetime_slice in datetime_slicer.stream_slices(sync_mode, stream_state.get(businessName, {}).get(serviceName, {})): + yield connection_slice | datetime_slice + + def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): + datetime_slicer = self.stream_slicers[1] + datetime_slicer.update_cursor(stream_slice, last_record) + if last_record: + businessName = stream_slice["connection"]["businessName"] + serviceName = stream_slice["connection"]["serviceName"] + self._cursor.setdefault(businessName, {}).setdefault(serviceName, {}).update(datetime_slicer.get_stream_state()) + else: + self._cursor = stream_slice diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py index 7957a3c0d9068..ab6aef53aab3c 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py @@ -4,12 +4,10 @@ from dataclasses import dataclass -from datetime import datetime -from typing import Any, List, Mapping, Optional, Union +from typing import Any, Mapping, Optional from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor -from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState @dataclass diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py index cebf1c6765490..167ea368753d9 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py @@ -10,7 +10,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest -from .config import ACCESS_TOKEN, DATE_TIME_FORMAT +from .config import DATE_TIME_FORMAT def get_stream_request(stream_name: str) -> RequestBuilder: diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_bundle_selections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_bundle_selections.py index 4bfa88ffc5ea9..44f5f941abdeb 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_bundle_selections.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_bundle_selections.py @@ -3,15 +3,13 @@ # -from unittest import TestCase - import freezegun from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response -from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental +from ..utils import StreamTestCase, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "bundle_selections" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py index b6055873bc816..e1b63d04278c1 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py @@ -2,7 +2,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # -from unittest import TestCase import freezegun @@ -10,7 +9,7 @@ from ..config import NOW from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response -from ..utils import StreamTestCase, config, read_full_refresh +from ..utils import StreamTestCase, read_full_refresh _STREAM_NAME = "collections" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py index 430a27d2a0ce8..a7876d21181a6 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py @@ -3,15 +3,13 @@ # -from unittest import TestCase - import freezegun from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response -from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental +from ..utils import StreamTestCase, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "discounts" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py index 4093fba40a020..2355797d1a9ea 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py @@ -3,15 +3,13 @@ # -from unittest import TestCase - import freezegun from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response -from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental +from ..utils import StreamTestCase, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "events" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py index 64255813f5033..8f8482165f09a 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py @@ -3,15 +3,13 @@ # -from unittest import TestCase - import freezegun from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response -from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental +from ..utils import StreamTestCase, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "onetimes" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py index 0991a86f1d9bf..29eccd3563cb4 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py @@ -3,7 +3,6 @@ # import json -from unittest import TestCase import freezegun diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py index 475f8d95ee1d0..6c597c029278a 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py @@ -4,7 +4,6 @@ from abc import ABC -from lib2to3.pgen2.literals import test from typing import Any, List, Mapping, Optional from unittest import TestCase diff --git a/airbyte-integrations/connectors/source-rss/source_rss/components.py b/airbyte-integrations/connectors/source-rss/source_rss/components.py index 4379d281e143f..cbc2abb58af46 100644 --- a/airbyte-integrations/connectors/source-rss/source_rss/components.py +++ b/airbyte-integrations/connectors/source-rss/source_rss/components.py @@ -3,22 +3,16 @@ # import datetime -import logging from calendar import timegm -from dataclasses import dataclass from datetime import datetime -from typing import Any, Iterable, List, Mapping, Optional +from typing import Any, List, Mapping import feedparser import pytz import requests from dateutil.parser import parse -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor -from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor -from airbyte_cdk.sources.declarative.types import StreamSlice -from airbyte_cdk.sources.streams.core import Stream class CustomExtractor(RecordExtractor): diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source.py index 43da43b656e2e..c2ed4cec0635f 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source.py @@ -39,10 +39,10 @@ class Config: order=2, ) role_arn: Optional[str] = Field( - title=f"AWS Role ARN", + title="AWS Role ARN", default=None, description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations " - f"requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", always_show=True, order=7, ) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py index cdc66ab06fc70..7a8fb25e7bc07 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py @@ -34,10 +34,10 @@ def documentation_url(cls) -> AnyUrl: ) role_arn: Optional[str] = Field( - title=f"AWS Role ARN", + title="AWS Role ARN", default=None, description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations " - f"requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", order=6, ) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py index 9c6051c8dd16f..17bc50101df3d 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py @@ -6,7 +6,7 @@ import time from datetime import datetime from io import IOBase -from os import getenv, makedirs, path +from os import getenv from typing import Dict, Iterable, List, Optional, Set, cast import boto3.session diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_launch.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_launch.py index 0f43624f29d08..35b920de7c36b 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_launch.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/test_launch.py @@ -2,7 +2,6 @@ from __future__ import annotations -from pathlib import Path from unittest.mock import MagicMock, patch import pytest diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py index 0653f3a21ea4a..3b3c8c007118b 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py @@ -15,7 +15,7 @@ from airbyte_cdk.utils import AirbyteTracedException from .exceptions import TypeSalesforceException -from .rate_limiting import SalesforceErrorHandler, default_backoff_handler +from .rate_limiting import SalesforceErrorHandler from .utils import filter_streams_by_criteria diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 2de1867674cb6..5b8fbd94a0f85 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -11,7 +11,6 @@ import pendulum import requests # type: ignore[import] -from pendulum import DateTime # type: ignore[attr-defined] from requests import exceptions from airbyte_cdk import ( @@ -42,7 +41,6 @@ from airbyte_cdk.sources.declarative.retrievers import AsyncRetriever from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import IsoMillisConcurrentStreamStateConverter from airbyte_cdk.sources.streams.core import CheckpointMixin, Stream, StreamData diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index cd370fba21f45..681f15ad4f84b 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -22,7 +22,6 @@ CSV_FIELD_SIZE_LIMIT, BulkIncrementalSalesforceStream, BulkSalesforceStream, - BulkSalesforceSubStream, IncrementalRestSalesforceStream, RestSalesforceStream, ) @@ -554,7 +553,7 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api source.streams.return_value = [generate_stream("Account", stream_config_date_format, stream_api, state=state, legacy=False)] # using legacy state to configure HTTP requests - stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api, state=state, legacy=True) + generate_stream("Account", stream_config_date_format, stream_api, state=state, legacy=True) job_id_1 = "fake_job_1" requests_mock.register_uri( diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py index cd380be8ee2c0..b3751bb2a54a0 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py @@ -13,7 +13,7 @@ from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from integration.utils import create_base_url, given_authentication, given_stream +from integration.utils import create_base_url, given_authentication _CLIENT_ID = "a_client_id" diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py index 76af9b7eaf382..ccb735a6ab57b 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py @@ -3,7 +3,7 @@ import json from airbyte_cdk.test.mock_http import HttpResponse -from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, find_template +from airbyte_cdk.test.mock_http.response_builder import find_template class JobCreateResponseBuilder: diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py index b2f335f594e80..d5399b310de7f 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py @@ -1,6 +1,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from unittest import TestCase import freezegun diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py index ffa0852de7999..31ab35ec03796 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py @@ -41,7 +41,7 @@ def wrapper(self, *args, **kwargs) -> Any: sleep(self._job_backoff_time) except ShopifyBulkExceptions.BulkJobCreationFailedConcurrentError: if self._concurrent_attempt == self._concurrent_max_retry: - message = f"The BULK Job couldn't be created at this time, since another job is running." + message = "The BULK Job couldn't be created at this time, since another job is running." LOGGER.error(message) raise ShopifyBulkExceptions.BulkJobConcurrentError(message) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py index fc5ead602a7cf..8816aa268bac6 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py @@ -795,7 +795,7 @@ def emit_slice_message(self, slice_start: datetime, slice_end: datetime) -> None if self.job_manager._supports_checkpointing: checkpointing_message = f" The BULK checkpoint after `{self.job_manager.job_checkpoint_interval}` lines." else: - checkpointing_message = f" The BULK checkpointing is not supported." + checkpointing_message = " The BULK checkpointing is not supported." self.logger.info(slice_message + checkpointing_message) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py index cf586b159ae17..0ed33cc9c75a1 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py @@ -92,7 +92,7 @@ def test_bulk_query_cancel() -> None: Query( name="test_root", arguments=[ - Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), + Argument(name="query", value="\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), ], fields=[Field(name="edges", fields=[Field(name="node", fields=["test_field1", "test_field2"])])], ), @@ -138,7 +138,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, Query( name="customers", arguments=[ - Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), + Argument(name="query", value="\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), Argument(name="sortKey", value="UPDATED_AT"), ], fields=[ @@ -196,7 +196,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, Query( name="products", arguments=[ - Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), + Argument(name="query", value="\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), Argument(name="sortKey", value="UPDATED_AT"), ], fields=[ @@ -295,7 +295,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, name="inventoryLevels", arguments=[ Argument( - name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\"" + name="query", value="\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\"" ), ], fields=[ diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py index ad8d1e31fb379..61e30d14ff669 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py @@ -4,8 +4,6 @@ from typing import Any, Dict, List, Optional from unittest import TestCase -import pytest - from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py index 5101fb1585f27..1877f3ee96923 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from decimal import Decimal import pytest from source_shopify.transform import DataTypeEnforcer diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py index 24ee67a632a01..216d86fb5ed98 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -8,7 +8,7 @@ import requests from source_shopify.run import run from source_shopify.source import ConnectionCheckTest, SourceShopify -from source_shopify.streams.streams import BalanceTransactions, DiscountCodes, FulfillmentOrders, PriceRules +from source_shopify.streams.streams import BalanceTransactions, DiscountCodes, PriceRules from source_shopify.utils import ShopifyNonRetryableErrors @@ -82,7 +82,7 @@ def test_privileges_validation(requests_mock, fetch_transactions_user_id, basic_ ) basic_config["fetch_transactions_user_id"] = fetch_transactions_user_id # mock the get_shop_id method - with patch.object(ConnectionCheckTest, "get_shop_id", return_value=123) as mock: + with patch.object(ConnectionCheckTest, "get_shop_id", return_value=123): source = SourceShopify() streams = source.streams(basic_config) assert [stream.name for stream in streams] == expected diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py index d0068d1f4aa9e..5e1e0db07e1fb 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock -import pendulum import pytest from source_slack import SourceSlack from source_slack.components.channel_members_extractor import ChannelMembersExtractor diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py index 4ec3b24c517a0..31ab978637cca 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py @@ -8,7 +8,7 @@ import pytest from requests import Response from source_slack import SourceSlack -from source_slack.streams import Channels, JoinChannelsStream, Threads +from source_slack.streams import Channels, Threads from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py index cc3bce0820be7..ce8718aa7683f 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py @@ -2,7 +2,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # -import logging from typing import Optional, Union import requests @@ -22,7 +21,7 @@ "oauth_not_supported": "Please use a different authentication method.", } -DOCS_URL = f"https://docs.airbyte.com/integrations/sources/stripe" +DOCS_URL = "https://docs.airbyte.com/integrations/sources/stripe" DOCUMENTATION_MESSAGE = f"Please visit {DOCS_URL} to learn more. " diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index 7002a36f5a4a7..18b0886b1ec85 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -29,7 +29,6 @@ IncrementalStripeStream, ParentIncrementalStripeSubStream, SetupAttempts, - StripeLazySubStream, StripeStream, StripeSubStream, UpdatedCursorIncrementalStripeLazySubStream, diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/components.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/components.py index 6732dcec83a09..88ae87e058edc 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/components.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/components.py @@ -2,12 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dataclasses import dataclass from typing import Iterable from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import StreamSlice class SurveyIdPartitionRouter(SubstreamPartitionRouter): diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py index 8c96e04e6453b..fc0090b74fdf1 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py @@ -9,7 +9,6 @@ from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.sources import Source -from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py index ece17f12fc701..fe67d1a2bdc7f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py @@ -1,10 +1,10 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -from dataclasses import InitVar, dataclass +from dataclasses import dataclass from typing import Any, Mapping, Optional from airbyte_cdk.sources.declarative.transformations import RecordTransformation -from airbyte_cdk.sources.declarative.types import Config, FieldPointer, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState @dataclass diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py index dba9ba324d682..9a0200ffd4a60 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py @@ -12,7 +12,7 @@ def mock_advertisers_slices(http_mocker: HttpMocker, config: dict): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", + url="https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", query_params={"secret": config["credentials"]["secret"], "app_id": config["credentials"]["app_id"]}, ), HttpResponse(body=json.dumps(find_template(ADVERTISERS_FILE, __file__)), status_code=200), diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_advertisers_audience_reports.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_advertisers_audience_reports.py index c666b389e8091..7d808d2d6d62b 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_advertisers_audience_reports.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_advertisers_audience_reports.py @@ -49,7 +49,7 @@ def _mock_response(self, http_mocker: HttpMocker, include_deleted: bool = False) ) http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py index ff52836086f6e..e9af99004039f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py @@ -30,7 +30,7 @@ def test_basic_read(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/file/music/get/?page_size=100&advertiser_id=872746382648", + url="https://business-api.tiktok.com/open_api/v1.3/file/music/get/?page_size=100&advertiser_id=872746382648", ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), ) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py index 49ef79b46ef67..5ce2071d1c83d 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py @@ -30,7 +30,7 @@ def test_basic_read(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/creative/portfolio/list/?page_size=100&advertiser_id=872746382648", + url="https://business-api.tiktok.com/open_api/v1.3/creative/portfolio/list/?page_size=100&advertiser_id=872746382648", ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), ) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py index 2a26bb85fe902..2e5dff6ffabbc 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py @@ -130,7 +130,7 @@ def mock_response(self, http_mocker: HttpMocker, include_deleted=False): query_params["filtering"] = '[{"field_name": "ad_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), @@ -139,7 +139,7 @@ def mock_response(self, http_mocker: HttpMocker, include_deleted=False): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), @@ -285,7 +285,7 @@ def test_basic_read(self, http_mocker: HttpMocker): } http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), @@ -293,7 +293,7 @@ def test_basic_read(self, http_mocker: HttpMocker): query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), @@ -310,7 +310,7 @@ def test_read_with_state(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params={ "service_type": "AUCTION", "report_type": "BASIC", @@ -328,7 +328,7 @@ def test_read_with_state(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params={ "service_type": "AUCTION", "report_type": "BASIC", @@ -374,7 +374,7 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): } http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), @@ -382,7 +382,7 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), @@ -456,7 +456,7 @@ def state(self): def mock_response(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params={ "service_type": "AUCTION", "report_type": "BASIC", @@ -474,7 +474,7 @@ def mock_response(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params={ "service_type": "AUCTION", "report_type": "BASIC", @@ -593,7 +593,7 @@ def mock_response(self, http_mocker: HttpMocker, include_deleted: bool = False): query_params["filtering"] = '[{"field_name": "campaign_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), @@ -602,7 +602,7 @@ def mock_response(self, http_mocker: HttpMocker, include_deleted: bool = False): query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] http_mocker.get( HttpRequest( - url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", + url="https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", query_params=query_params, ), HttpResponse(body=json.dumps(EMPTY_LIST_RESPONSE), status_code=200), diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py index cd1ef342b2886..30143e023a173 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py @@ -10,9 +10,7 @@ ) from source_tiktok_marketing.components.transformations import TransformEmptyMetrics -from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig -from airbyte_cdk.sources.declarative.types import StreamSlice @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py index 0ea135cec9f47..fc845c46dbf29 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json from unittest.mock import MagicMock import pytest diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py new file mode 100644 index 0000000000000..b154c29ba6ff8 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from unittest.mock import Mock + +import pytest +from source_typeform.components import FormIdPartitionRouter + +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig + + +# test cases as a list of tuples (form_ids, parent_stream_configs, expected_slices) +test_cases = [ + ( + # test form ids present in config + ["form_id_1", "form_id_2"], + [{"stream": Mock(read_records=Mock(return_value=[{"id": "form_id_3"}, {"id": "form_id_4"}]))}], + [{"form_id": "form_id_1"}, {"form_id": "form_id_2"}], + ), + ( + # test no form ids in config + [], + [ + {"stream": Mock(read_records=Mock(return_value=[{"id": "form_id_3"}, {"id": "form_id_4"}]))}, + {"stream": Mock(read_records=Mock(return_value=[{"id": "form_id_5"}, {"id": "form_id_6"}]))}, + ], + [{"form_id": "form_id_3"}, {"form_id": "form_id_4"}, {"form_id": "form_id_5"}, {"form_id": "form_id_6"}], + ), +] + + +@pytest.mark.parametrize("form_ids, parent_stream_configs, expected_slices", test_cases) +def test_stream_slices(form_ids, parent_stream_configs, expected_slices): + stream_configs = [] + + for parent_stream_config in parent_stream_configs: + stream_config = ParentStreamConfig( + stream=parent_stream_config["stream"], parent_key=None, partition_field=None, config=None, parameters=None + ) + stream_configs.append(stream_config) + if not stream_configs: + stream_configs = [None] + + router = FormIdPartitionRouter(config={"form_ids": form_ids}, parent_stream_configs=stream_configs, parameters=None) + slices = list(router.stream_slices()) + + assert slices == expected_slices diff --git a/airbyte-integrations/connectors/source-xero/components.py b/airbyte-integrations/connectors/source-xero/components.py index 87e228e0f21b4..72ec56076733d 100644 --- a/airbyte-integrations/connectors/source-xero/components.py +++ b/airbyte-integrations/connectors/source-xero/components.py @@ -5,7 +5,7 @@ import re from dataclasses import InitVar, dataclass from datetime import date, datetime, time, timedelta, timezone -from typing import Any, Dict, List, Mapping, Union +from typing import Any, List, Mapping, Union import dpath.util import requests diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py index 28e6de3dd4fca..261a78ef834f2 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py @@ -10,7 +10,6 @@ import requests from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor -from airbyte_cdk.sources.declarative.types import Record @dataclass diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py index caab6dfc3cf90..df1da58cc1bc5 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py @@ -8,7 +8,7 @@ from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState @dataclass diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py index cdd05d55f4427..38e42274d1192 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py @@ -1,6 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import base64 from typing import Any, Dict from pendulum.datetime import DateTime diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py index e672521e1501a..485249079c692 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py @@ -34,7 +34,7 @@ def get_authenticator(config): return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"]) @HttpMocker() - def test_given_incoming_state_semi_incremental_groups_does_not_emit_earlier_record(self, http_mocker): + def test_no_state_semi_incremental_groups_does_not_emit_earlier_record(self, http_mocker): """ Perform a semi-incremental sync where records that came before the current state are not included in the set of records emitted diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py index c5e2dda402cdf..efb6b9351767b 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py @@ -5,7 +5,7 @@ import freezegun import pendulum -from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, SyncMode +from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_cdk.test.state_builder import StateBuilder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/groups_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/groups_request_builder.py index f4dc2d140848c..952ce14f763f4 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/groups_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/groups_request_builder.py @@ -1,8 +1,5 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -import calendar - -import pendulum from .base_request_builder import ZendeskSupportBaseRequestBuilder from .request_authenticators.authenticator import Authenticator diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/tickets_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/tickets_request_builder.py index e61938f6268f4..bb22bdbbe8970 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/tickets_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/tickets_request_builder.py @@ -1,8 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import calendar - -import pendulum from .base_request_builder import ZendeskSupportBaseRequestBuilder from .request_authenticators.authenticator import Authenticator diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/ticket_metrics_response_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/ticket_metrics_response_builder.py index 1bbe5a7b59305..a05965acb1376 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/ticket_metrics_response_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/ticket_metrics_response_builder.py @@ -7,7 +7,6 @@ HttpResponse, HttpResponseBuilder, NestedPath, - RecordBuilder, find_template, ) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 8482c40c45f75..05a94186960d1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -1094,7 +1094,7 @@ def test_get_implemented_stream(self, state, expected_implemented_stream): ) def test_stream_slices(self, sync_mode, state, expected_implemented_stream): stream = get_stream_instance(TicketMetrics, STREAM_ARGS) - slices = list(stream.stream_slices(sync_mode=sync_mode, stream_state=state)) + list(stream.stream_slices(sync_mode=sync_mode, stream_state=state)) assert isinstance(stream.implemented_stream, expected_implemented_stream) @@ -1130,13 +1130,13 @@ class TestStatefulTicketMetrics: ) def test_stream_slices(self, requests_mock, stream_state, response, expected_slices): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) - requests_mock.get(f"https://sandbox.zendesk.com/api/v2/incremental/tickets/cursor.json", json=response) + requests_mock.get("https://sandbox.zendesk.com/api/v2/incremental/tickets/cursor.json", json=response) assert list(stream.stream_slices(sync_mode=SyncMode.incremental, stream_state=stream_state)) == expected_slices def test_read_with_error(self, requests_mock): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) requests_mock.get( - f"https://sandbox.zendesk.com/api/v2/tickets/13/metrics", json={"error": "RecordNotFound", "description": "Not found"} + "https://sandbox.zendesk.com/api/v2/tickets/13/metrics", json={"error": "RecordNotFound", "description": "Not found"} ) records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"ticket_id": "13"})) diff --git a/tools/bin/identify-dormant-workflows.py b/tools/bin/identify-dormant-workflows.py index 40a62d426af3d..bdad69552d756 100644 --- a/tools/bin/identify-dormant-workflows.py +++ b/tools/bin/identify-dormant-workflows.py @@ -3,7 +3,6 @@ # import argparse -import json import os import re import subprocess @@ -80,8 +79,6 @@ def main(): repo = g.get_repo(repo_name) workflows = repo.get_workflows() - runs_to_delete = [] - for workflow in workflows: runs = workflow.get_runs() for run in runs: @@ -102,7 +99,7 @@ def main(): client = WebClient(slack_token) try: - response = client.chat_postMessage(channel=SLACK_CHANNEL_FOR_NOTIFICATIONS, text=message) + client.chat_postMessage(channel=SLACK_CHANNEL_FOR_NOTIFICATIONS, text=message) except SlackApiError as e: print(e, "\n\n") raise Exception("Error calling the Slack API") diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index 93529595f7066..db538630e3e51 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -4,7 +4,6 @@ import argparse import json -import os """ diff --git a/tools/bin/record_obfuscator.py b/tools/bin/record_obfuscator.py index 031010549fb1e..81845b9b761f8 100755 --- a/tools/bin/record_obfuscator.py +++ b/tools/bin/record_obfuscator.py @@ -53,7 +53,7 @@ def obfuscate(value: Any) -> Any: line = line.strip() try: data = json.loads(line) - except Exception as exc: + except Exception: # We don't expect invalid json so if we see it, it will go to stderr sys.stderr.write(f"{line}\n") diff --git a/tools/schema_generator/schema_generator/infer_schemas.py b/tools/schema_generator/schema_generator/infer_schemas.py index 4323955e39073..54de2ad0506ec 100644 --- a/tools/schema_generator/schema_generator/infer_schemas.py +++ b/tools/schema_generator/schema_generator/infer_schemas.py @@ -26,7 +26,6 @@ import os import sys -import genson.schema.strategies as strategies from genson import SchemaBuilder from genson.schema.strategies.object import Object