diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml index 7c4568934..e77cf5346 100644 --- a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml +++ b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml @@ -41,6 +41,9 @@ singer-sdk = { version="~=0.39.1", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] +[tool.pytest.ini_options] +addopts = '--durations=10' + [tool.mypy] python_version = "3.12" warn_unused_configs = true diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index 6556a2f1b..98851d925 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -51,6 +51,9 @@ singer-sdk = { version="~=0.39.1", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] +[tool.pytest.ini_options] +addopts = '--durations=10' + [tool.mypy] python_version = "3.12" warn_unused_configs = true diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py index df3f9f754..74c8927e9 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py @@ -50,6 +50,16 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type == default="https://api.mysample.com", description="The url for the API service", ), + {%- if cookiecutter.stream_type in ("GraphQL", "REST") %} + th.Property( + "user_agent", + th.StringType, + description=( + "A custom User-Agent header to send with each request. Default is " + "'/'" + ), + ), + {%- endif %} ).to_dict() {%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 9cf626117..5aa268f7c 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -43,6 +43,13 @@ singer-sdk = { version="~=0.39.1", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] +[tool.pytest.ini_options] +addopts = '--durations=10' + +[tool.mypy] +python_version = "3.12" +warn_unused_configs = true + [tool.ruff] src = ["{{cookiecutter.library_name}}"] target-version = "py38" diff --git a/poetry.lock b/poetry.lock index ace6f7f76..b5e2b63bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -43,13 +43,13 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "babel" -version = "2.15.0" +version = "2.16.0" description = "Internationalization utilities" optional = true python-versions = ">=3.8" files = [ - {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, - {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, + {file = "babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b"}, + {file = "babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"}, ] [package.extras] @@ -129,17 +129,17 @@ lxml = ["lxml"] [[package]] name = "boto3" -version = "1.34.156" +version = "1.34.157" description = "The AWS SDK for Python" optional = true python-versions = ">=3.8" files = [ - {file = "boto3-1.34.156-py3-none-any.whl", hash = "sha256:cbbd453270b8ce94ef9da60dfbb6f9ceeb3eeee226b635aa9ec44b1def98cc96"}, - {file = "boto3-1.34.156.tar.gz", hash = "sha256:b33e9a8f8be80d3053b8418836a7c1900410b23a30c7cb040927d601a1082e68"}, + {file = "boto3-1.34.157-py3-none-any.whl", hash = "sha256:3cc357156df5482154a016f138d1953061a181b4c594f8b6302c9d6c024bd950"}, + {file = "boto3-1.34.157.tar.gz", hash = "sha256:7ef19ed38cba9863b58430fb4a66a72a5c250304f234bd1c16b860f9bf25677b"}, ] [package.dependencies] -botocore = ">=1.34.156,<1.35.0" +botocore = ">=1.34.157,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -148,13 +148,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.156" +version = "1.34.157" description = "Low-level, data-driven core of boto 3." optional = true python-versions = ">=3.8" files = [ - {file = "botocore-1.34.156-py3-none-any.whl", hash = "sha256:c48f8c8996216dfdeeb0aa6d3c0f2c7ae25234766434a2ea3e57bdc08494bdda"}, - {file = "botocore-1.34.156.tar.gz", hash = "sha256:5d1478c41ab9681e660b3322432fe09c4055759c317984b7b8d3af9557ff769a"}, + {file = "botocore-1.34.157-py3-none-any.whl", hash = "sha256:c6cba6de8eb86ca4d2f934e009b37adbe1e7fdcfa52fbab74783f4c30676e07d"}, + {file = "botocore-1.34.157.tar.gz", hash = "sha256:5628a36cec123cdc8c1158d05a7b06aa5e53649ad73796c50ef3fb51199785fb"}, ] [package.dependencies] @@ -520,22 +520,23 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "deptry" -version = "0.19.0" +version = "0.19.1" description = "A command line utility to check for unused, missing and transitive dependencies in a Python project." optional = false python-versions = ">=3.8" files = [ - {file = "deptry-0.19.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:074bfb613c1789e7489c735159356a8e7f260b0cf85193c6cc5887abcdabe5cc"}, - {file = "deptry-0.19.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:8fdfe2531fa773e5849b46d8d0ca851341aeeba3dc285b1a3f560a2a468676ba"}, - {file = "deptry-0.19.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:248c34d78f83da111379a43e37c78d5e049661735e43ab5934307b2ba265431d"}, - {file = "deptry-0.19.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b18c45b52c2822fd1186e73e22274ceec85384ed0616d0454ae874724ac0846"}, - {file = "deptry-0.19.0-cp38-abi3-win_amd64.whl", hash = "sha256:79028cbc885ff8cd0a11fc0954bb0b552bf656fe6df73084df7014cbd902516c"}, - {file = "deptry-0.19.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4fb158d0e4747e8dc21b2160f9a11540c01ede318cf558d702d485cb89119214"}, - {file = "deptry-0.19.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:bd80e70a2b306732d9c35019480a32452c00f74cbbe81b4a9791917fceeb2a0d"}, - {file = "deptry-0.19.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25f3b0e46b9936b3e9bb08aa9aa2f88a56d7d3bc14f0180844da4997670f21ab"}, - {file = "deptry-0.19.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbd13b8619d4fcfa0a06f752de5b92d552bdf71cfed41b2cf22cd9a16e28f4f2"}, - {file = "deptry-0.19.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fe61febc0ba979b20678d232daa4c6df31949f63d5dcaa5c363a5f1ffac9b887"}, - {file = "deptry-0.19.0.tar.gz", hash = "sha256:df5899d63a4e607bc9b2a091483b8e07ea98e021f2872defb1fd44573ae8c9a7"}, + {file = "deptry-0.19.1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3a20ef0dd1c737fb05553d1b9c2fa9f185d0c9d3d881d255334cef401ffdc599"}, + {file = "deptry-0.19.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:2c6b2df353e5113fd2f787c2f7e694657548d388929e988e8644bd178e19fc5c"}, + {file = "deptry-0.19.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a407bab3486e3844f93d702f1a381942873b2a46056c693b5634bbde219bb056"}, + {file = "deptry-0.19.1-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43f33789b97b47313609e92b62fabf8a71bba0d35a7476806da5d3d152e32345"}, + {file = "deptry-0.19.1-cp38-abi3-win_amd64.whl", hash = "sha256:0bad85a77b31360d0f52383b14783fdae4a201b597c0158fe10e91a779c67079"}, + {file = "deptry-0.19.1-cp38-abi3-win_arm64.whl", hash = "sha256:c59142d9dca8873325692fbb7aa1d2902fde87020dcc8102f75120ba95515172"}, + {file = "deptry-0.19.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a1abc119f9c8536b8ab1ee2122d4130665f33225d00d8615256ce354eb2c11ba"}, + {file = "deptry-0.19.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7344c6cea032b549d86e156aa1e679fb94cd44deb7e93f25cb6d9c0ded5ea06f"}, + {file = "deptry-0.19.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff7d8954265c48ea334fdd508339c51d3fba05e2d4a8be47712c69d1c8d35c94"}, + {file = "deptry-0.19.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023073247e5dac21254bf7b600ca2e2b71560652d2dfbe11535445ee912ca059"}, + {file = "deptry-0.19.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:af8a0a9c42f8f92dfbc048e724fa89b9131f032f7e245812260560c214395abf"}, + {file = "deptry-0.19.1.tar.gz", hash = "sha256:1c12fea1d2301f42c7035c5636e4b9421457fde256fe7a241245662d20b4c841"}, ] [package.dependencies] @@ -641,13 +642,13 @@ test = ["pytest (>=6)"] [[package]] name = "faker" -version = "26.2.0" +version = "26.3.0" description = "Faker is a Python package that generates fake data for you." optional = true python-versions = ">=3.8" files = [ - {file = "Faker-26.2.0-py3-none-any.whl", hash = "sha256:7b123090774deff5f2cd3eb92a84dcbbf1e163f30a6d07321b7852c11bfe6a75"}, - {file = "Faker-26.2.0.tar.gz", hash = "sha256:81768de19012147521140f0d8bf5353e501ac42c1065d25e0cac455d3615c0a7"}, + {file = "Faker-26.3.0-py3-none-any.whl", hash = "sha256:97fe1e7e953dd640ca2cd4dfac4db7c4d2432dd1b7a244a3313517707f3b54e9"}, + {file = "Faker-26.3.0.tar.gz", hash = "sha256:7c10ebdf74aaa0cc4fe6ec6db5a71e8598ec33503524bd4b5f4494785a5670dd"}, ] [package.dependencies] @@ -1492,20 +1493,6 @@ compat = ["pytest-benchmark (>=4.0.0,<4.1.0)", "pytest-xdist (>=2.0.0,<2.1.0)"] lint = ["mypy (>=1.3.0,<1.4.0)", "ruff (>=0.3.3,<0.4.0)"] test = ["pytest (>=7.0,<8.0)", "pytest-cov (>=4.0.0,<4.1.0)"] -[[package]] -name = "pytest-durations" -version = "1.2.0" -description = "Pytest plugin reporting fixtures and test functions execution time." -optional = true -python-versions = ">=3.6.2" -files = [ - {file = "pytest-durations-1.2.0.tar.gz", hash = "sha256:75793f7c2c393a947de4a92cc205e8dcb3d7fcde492628926cca97eb8e87077d"}, - {file = "pytest_durations-1.2.0-py3-none-any.whl", hash = "sha256:210c649d989fdf8e864b7f614966ca2c8be5b58a5224d60089a43618c146d7fb"}, -] - -[package.dependencies] -pytest = ">=4.6" - [[package]] name = "pytest-snapshot" version = "0.9.0" @@ -2403,13 +2390,13 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.20240724" +version = "6.0.12.20240808" description = "Typing stubs for PyYAML" optional = false python-versions = ">=3.8" files = [ - {file = "types-PyYAML-6.0.12.20240724.tar.gz", hash = "sha256:cf7b31ae67e0c5b2919c703d2affc415485099d3fe6666a6912f040fd05cb67f"}, - {file = "types_PyYAML-6.0.12.20240724-py3-none-any.whl", hash = "sha256:e5becec598f3aa3a2ddf671de4a75fa1c6856fbf73b2840286c9d50fae2d5d48"}, + {file = "types-PyYAML-6.0.12.20240808.tar.gz", hash = "sha256:b8f76ddbd7f65440a8bda5526a9607e4c7a322dc2f8e1a8c405644f9a6f4b9af"}, + {file = "types_PyYAML-6.0.12.20240808-py3-none-any.whl", hash = "sha256:deda34c5c655265fc517b546c902aa6eed2ef8d3e921e4765fe606fe2afe8d35"}, ] [[package]] @@ -2519,9 +2506,9 @@ faker = ["faker"] jwt = ["PyJWT", "cryptography"] parquet = ["numpy", "numpy", "pyarrow"] s3 = ["fs-s3fs"] -testing = ["pytest", "pytest-durations"] +testing = ["pytest"] [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "a9608352516ffcc098a93ce4e3375b11989c328c7520cad28ab89c458c1dbc3b" +content-hash = "8d0665d7e5397609e616976d470dca9863a925a12f513c0b78439f055aeb664a" diff --git a/pyproject.toml b/pyproject.toml index 030f94b96..68faa36ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,6 @@ pyarrow = { version = ">=13", optional = true } # Testing dependencies installed as optional 'testing' extras pytest = {version=">=7.2.1", optional = true} -pytest-durations = {version = ">=1.2.0", optional = true} # installed as optional 'faker' extra faker = {version = ">=22.5,<27.0", optional = true} @@ -110,7 +109,6 @@ docs = [ s3 = ["fs-s3fs"] testing = [ "pytest", - "pytest-durations" ] parquet = ["numpy", "pyarrow"] faker = ["faker"] @@ -143,7 +141,7 @@ types-PyYAML = ">=6.0.12" pytest-codspeed = ">=2.2.0" [tool.pytest.ini_options] -addopts = '--ignore=singer_sdk/helpers/_simpleeval.py -m "not external"' +addopts = '--durations=10 --ignore=singer_sdk/helpers/_simpleeval.py -m "not external"' filterwarnings = [ "error", "ignore:Could not configure external gitlab tests:UserWarning", @@ -251,7 +249,6 @@ DEP002 = [ "sphinx-reredirects", # Plugins "fs-s3fs", - "pytest-durations", ] [tool.mypy] diff --git a/samples/sample_tap_dummy_json/tap_dummyjson/client.py b/samples/sample_tap_dummy_json/tap_dummyjson/client.py index c946675f3..3f0ba2a91 100644 --- a/samples/sample_tap_dummy_json/tap_dummyjson/client.py +++ b/samples/sample_tap_dummy_json/tap_dummyjson/client.py @@ -28,10 +28,6 @@ def authenticator(self): password=self.config["password"], ) - @property - def http_headers(self): - return {"User-Agent": "tap-dummyjson"} - def get_new_paginator(self): return BaseOffsetPaginator(start_value=0, page_size=PAGE_SIZE) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index f48222640..b6a74a976 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -5,6 +5,7 @@ import logging import typing as t import warnings +from collections import UserString from contextlib import contextmanager from datetime import datetime from functools import lru_cache @@ -22,6 +23,86 @@ from sqlalchemy.engine.reflection import Inspector +class FullyQualifiedName(UserString): + """A fully qualified table name. + + This class provides a simple way to represent a fully qualified table name + as a single object. The string representation of this object is the fully + qualified table name, with the parts separated by periods. + + The parts of the fully qualified table name are: + - database + - schema + - table + + The database and schema are optional. If only the table name is provided, + the string representation of the object will be the table name alone. + + Example: + ``` + table_name = FullyQualifiedName("my_table", "my_schema", "my_db") + print(table_name) # my_db.my_schema.my_table + ``` + """ + + def __init__( + self, + *, + table: str = "", + schema: str | None = None, + database: str | None = None, + delimiter: str = ".", + ) -> None: + """Initialize the fully qualified table name. + + Args: + table: The name of the table. + schema: The name of the schema. Defaults to None. + database: The name of the database. Defaults to None. + delimiter: The delimiter to use between parts. Defaults to '.'. + + Raises: + ValueError: If the fully qualified name could not be generated. + """ + self.table = table + self.schema = schema + self.database = database + self.delimiter = delimiter + + parts = [] + if self.database: + parts.append(self.prepare_part(self.database)) + if self.schema: + parts.append(self.prepare_part(self.schema)) + if self.table: + parts.append(self.prepare_part(self.table)) + + if not parts: + raise ValueError( + "Could not generate fully qualified name: " + + ":".join( + [ + self.database or "(unknown-db)", + self.schema or "(unknown-schema)", + self.table or "(unknown-table-name)", + ], + ), + ) + + super().__init__(self.delimiter.join(parts)) + + def prepare_part(self, part: str) -> str: # noqa: PLR6301 + """Prepare a part of the fully qualified name. + + Args: + part: The part to prepare. + + Returns: + The prepared part. + """ + return part + + class SQLConnector: # noqa: PLR0904 """Base class for SQLAlchemy-based connectors. @@ -244,7 +325,7 @@ def get_fully_qualified_name( schema_name: str | None = None, db_name: str | None = None, delimiter: str = ".", - ) -> str: + ) -> FullyQualifiedName: """Concatenates a fully qualified name from the parts. Args: @@ -253,34 +334,15 @@ def get_fully_qualified_name( db_name: The name of the database. Defaults to None. delimiter: Generally: '.' for SQL names and '-' for Singer names. - Raises: - ValueError: If all 3 name parts not supplied. - Returns: The fully qualified name as a string. """ - parts = [] - - if db_name: - parts.append(db_name) - if schema_name: - parts.append(schema_name) - if table_name: - parts.append(table_name) - - if not parts: - raise ValueError( - "Could not generate fully qualified name: " - + ":".join( - [ - db_name or "(unknown-db)", - schema_name or "(unknown-schema)", - table_name or "(unknown-table-name)", - ], - ), - ) - - return delimiter.join(parts) + return FullyQualifiedName( + table=table_name, # type: ignore[arg-type] + schema=schema_name, + database=db_name, + delimiter=delimiter, + ) @property def _dialect(self) -> sa.engine.Dialect: @@ -429,12 +491,7 @@ def discover_catalog_entry( `CatalogEntry` object for the given table or a view """ # Initialize unique stream name - unique_stream_id = self.get_fully_qualified_name( - db_name=None, - schema_name=schema_name, - table_name=table_name, - delimiter="-", - ) + unique_stream_id = f"{schema_name}-{table_name}" # Detect key properties possible_primary_keys: list[list[str]] = [] @@ -528,7 +585,7 @@ def discover_catalog_entries(self) -> list[dict]: def parse_full_table_name( # noqa: PLR6301 self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, ) -> tuple[str | None, str | None, str]: """Parse a fully qualified table name into its parts. @@ -547,6 +604,13 @@ def parse_full_table_name( # noqa: PLR6301 A three part tuple (db_name, schema_name, table_name) with any unspecified or unused parts returned as None. """ + if isinstance(full_table_name, FullyQualifiedName): + return ( + full_table_name.database, + full_table_name.schema, + full_table_name.table, + ) + db_name: str | None = None schema_name: str | None = None @@ -560,7 +624,7 @@ def parse_full_table_name( # noqa: PLR6301 return db_name, schema_name, table_name - def table_exists(self, full_table_name: str) -> bool: + def table_exists(self, full_table_name: str | FullyQualifiedName) -> bool: """Determine if the target table already exists. Args: @@ -587,7 +651,7 @@ def schema_exists(self, schema_name: str) -> bool: def get_table_columns( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, column_names: list[str] | None = None, ) -> dict[str, sa.Column]: """Return a list of table columns. @@ -618,7 +682,7 @@ def get_table_columns( def get_table( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, column_names: list[str] | None = None, ) -> sa.Table: """Return a table object. @@ -643,7 +707,9 @@ def get_table( schema=schema_name, ) - def column_exists(self, full_table_name: str, column_name: str) -> bool: + def column_exists( + self, full_table_name: str | FullyQualifiedName, column_name: str + ) -> bool: """Determine if the target table already exists. Args: @@ -666,7 +732,7 @@ def create_schema(self, schema_name: str) -> None: def create_empty_table( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, schema: dict, primary_keys: t.Sequence[str] | None = None, partition_keys: list[str] | None = None, @@ -715,7 +781,7 @@ def create_empty_table( def _create_empty_column( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, column_name: str, sql_type: sa.types.TypeEngine, ) -> None: @@ -753,7 +819,7 @@ def prepare_schema(self, schema_name: str) -> None: def prepare_table( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, schema: dict, primary_keys: t.Sequence[str], partition_keys: list[str] | None = None, @@ -797,7 +863,7 @@ def prepare_table( def prepare_column( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, column_name: str, sql_type: sa.types.TypeEngine, ) -> None: @@ -822,7 +888,9 @@ def prepare_column( sql_type=sql_type, ) - def rename_column(self, full_table_name: str, old_name: str, new_name: str) -> None: + def rename_column( + self, full_table_name: str | FullyQualifiedName, old_name: str, new_name: str + ) -> None: """Rename the provided columns. Args: @@ -951,7 +1019,7 @@ def _get_type_sort_key( def _get_column_type( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, column_name: str, ) -> sa.types.TypeEngine: """Get the SQL type of the declared column. @@ -976,7 +1044,7 @@ def _get_column_type( def get_column_add_ddl( self, - table_name: str, + table_name: str | FullyQualifiedName, column_name: str, column_type: sa.types.TypeEngine, ) -> sa.DDL: @@ -1009,7 +1077,7 @@ def get_column_add_ddl( @staticmethod def get_column_rename_ddl( - table_name: str, + table_name: str | FullyQualifiedName, column_name: str, new_column_name: str, ) -> sa.DDL: @@ -1037,7 +1105,7 @@ def get_column_rename_ddl( @staticmethod def get_column_alter_ddl( - table_name: str, + table_name: str | FullyQualifiedName, column_name: str, column_type: sa.types.TypeEngine, ) -> sa.DDL: @@ -1096,7 +1164,7 @@ def update_collation( def _adapt_column_type( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, column_name: str, sql_type: sa.types.TypeEngine, ) -> None: @@ -1187,7 +1255,7 @@ def deserialize_json(self, json_str: str) -> object: # noqa: PLR6301 def delete_old_versions( self, *, - full_table_name: str, + full_table_name: str | FullyQualifiedName, version_column_name: str, current_version: int, ) -> None: diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 33a741614..0f7695ef0 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -21,6 +21,7 @@ if t.TYPE_CHECKING: from sqlalchemy.sql import Executable + from singer_sdk.connectors.sql import FullyQualifiedName from singer_sdk.target_base import Target _C = t.TypeVar("_C", bound=SQLConnector) @@ -109,7 +110,7 @@ def database_name(self) -> str | None: # Assumes single-DB target context. @property - def full_table_name(self) -> str: + def full_table_name(self) -> FullyQualifiedName: """Return the fully qualified table name. Returns: @@ -122,7 +123,7 @@ def full_table_name(self) -> str: ) @property - def full_schema_name(self) -> str: + def full_schema_name(self) -> FullyQualifiedName: """Return the fully qualified schema name. Returns: @@ -269,7 +270,7 @@ def process_batch(self, context: dict) -> None: def generate_insert_statement( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, schema: dict, ) -> str | Executable: """Generate an insert statement for the given records. @@ -297,7 +298,7 @@ def generate_insert_statement( def bulk_insert_records( self, - full_table_name: str, + full_table_name: str | FullyQualifiedName, schema: dict, records: t.Iterable[dict[str, t.Any]], ) -> int | None: diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index 559389e8c..76e2abf5d 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -6,6 +6,7 @@ import copy import logging import typing as t +from functools import cached_property from http import HTTPStatus from urllib.parse import urlparse from warnings import warn @@ -100,7 +101,7 @@ def __init__( super().__init__(name=name, schema=schema, tap=tap) if path: self.path = path - self._http_headers: dict = {} + self._http_headers: dict = {"User-Agent": self.user_agent} self._requests_session = requests.Session() self._compiled_jsonpath = None self._next_page_token_compiled_jsonpath = None @@ -150,6 +151,18 @@ def requests_session(self) -> requests.Session: self._requests_session = requests.Session() return self._requests_session + @cached_property + def user_agent(self) -> str: + """Get the user agent string for the stream. + + Returns: + The user agent string. + """ + return self.config.get( + "user_agent", + f"{self.tap_name}/{self._tap.plugin_version}", + ) + def validate_response(self, response: requests.Response) -> None: """Validate HTTP response. @@ -553,10 +566,7 @@ def http_headers(self) -> dict: Returns: Dictionary of HTTP headers to use as a base for every request. """ - result = self._http_headers - if "user_agent" in self.config: - result["User-Agent"] = self.config.get("user_agent") - return result + return self._http_headers @property def timeout(self) -> int: diff --git a/singer_sdk/streams/sql.py b/singer_sdk/streams/sql.py index 954159885..2877a505b 100644 --- a/singer_sdk/streams/sql.py +++ b/singer_sdk/streams/sql.py @@ -14,6 +14,7 @@ from singer_sdk.streams.core import REPLICATION_INCREMENTAL, Stream if t.TYPE_CHECKING: + from singer_sdk.connectors.sql import FullyQualifiedName from singer_sdk.helpers.types import Context from singer_sdk.tap_base import Tap @@ -124,7 +125,7 @@ def primary_keys(self, new_value: t.Sequence[str]) -> None: self._singer_catalog_entry.metadata.root.table_key_properties = new_value @property - def fully_qualified_name(self) -> str: + def fully_qualified_name(self) -> FullyQualifiedName: """Generate the fully qualified version of the table name. Raises: diff --git a/tests/core/test_connector_sql.py b/tests/core/test_connector_sql.py index 10ee0c0f4..c8390f33d 100644 --- a/tests/core/test_connector_sql.py +++ b/tests/core/test_connector_sql.py @@ -7,9 +7,11 @@ import pytest import sqlalchemy as sa from sqlalchemy.dialects import registry, sqlite +from sqlalchemy.engine.default import DefaultDialect from samples.sample_duckdb import DuckDBConnector from singer_sdk.connectors import SQLConnector +from singer_sdk.connectors.sql import FullyQualifiedName from singer_sdk.exceptions import ConfigValidationError if t.TYPE_CHECKING: @@ -355,3 +357,38 @@ def create_engine(self) -> Engine: connector = CustomConnector(config={"sqlalchemy_url": "myrdbms:///"}) connector.create_engine() + + +def test_fully_qualified_name(): + fqn = FullyQualifiedName(table="my_table") + assert fqn == "my_table" + + fqn = FullyQualifiedName(schema="my_schema", table="my_table") + assert fqn == "my_schema.my_table" + + fqn = FullyQualifiedName( + database="my_catalog", + schema="my_schema", + table="my_table", + ) + assert fqn == "my_catalog.my_schema.my_table" + + +def test_fully_qualified_name_with_quoting(): + class QuotedFullyQualifiedName(FullyQualifiedName): + def __init__(self, *, dialect: sa.engine.Dialect, **kwargs: t.Any): + self.dialect = dialect + super().__init__(**kwargs) + + def prepare_part(self, part: str) -> str: + return self.dialect.identifier_preparer.quote(part) + + dialect = DefaultDialect() + + fqn = QuotedFullyQualifiedName(table="order", schema="public", dialect=dialect) + assert fqn == 'public."order"' + + +def test_fully_qualified_name_empty_error(): + with pytest.raises(ValueError, match="Could not generate fully qualified name"): + FullyQualifiedName()