diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ece6e65..4f36603 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,23 +13,33 @@ jobs: fail-fast: false matrix: include: + # dbt-core now supports Python 3.12 + - { python: "3.12", os: "ubuntu-latest", dbt: "1.7" } + - { python: "3.12", os: "ubuntu-latest", dbt: "1.8" } # Latest dbts' test matrix on 3.11 (well supported py version) on ubuntu - { python: "3.11", os: "ubuntu-latest", dbt: "1.4" } - { python: "3.11", os: "ubuntu-latest", dbt: "1.5" } + - { python: "3.11", os: "ubuntu-latest", dbt: "1.6" } + - { python: "3.11", os: "ubuntu-latest", dbt: "1.7" } + - { python: "3.11", os: "ubuntu-latest", dbt: "1.8" } # full test matrix on 3.10 (well supported py version) on ubuntu - - { python: "3.10", os: "ubuntu-latest", dbt: "1.0" } - { python: "3.10", os: "ubuntu-latest", dbt: "1.1" } - { python: "3.10", os: "ubuntu-latest", dbt: "1.2" } - { python: "3.10", os: "ubuntu-latest", dbt: "1.3" } - { python: "3.10", os: "ubuntu-latest", dbt: "1.4" } - { python: "3.10", os: "ubuntu-latest", dbt: "1.5" } + - { python: "3.10", os: "ubuntu-latest", dbt: "1.6" } + - { python: "3.10", os: "ubuntu-latest", dbt: "1.7" } + - { python: "3.10", os: "ubuntu-latest", dbt: "1.8" } # full test matrix on 3.9 (well supported py version) on ubuntu - - { python: "3.9", os: "ubuntu-latest", dbt: "1.0" } - { python: "3.9", os: "ubuntu-latest", dbt: "1.1" } - { python: "3.9", os: "ubuntu-latest", dbt: "1.2" } - { python: "3.9", os: "ubuntu-latest", dbt: "1.3" } - { python: "3.9", os: "ubuntu-latest", dbt: "1.4" } - { python: "3.9", os: "ubuntu-latest", dbt: "1.5" } + - { python: "3.9", os: "ubuntu-latest", dbt: "1.6" } + - { python: "3.9", os: "ubuntu-latest", dbt: "1.7" } + - { python: "3.9", os: "ubuntu-latest", dbt: "1.8" } # stability test on 3.8 (min python ver) with latest dbt - { python: "3.8", os: "ubuntu-latest", dbt: "1.5" } # stability test on 3.10 with latest dbt on macos (cross platform) diff --git a/Makefile b/Makefile index 74afaf5..12f7417 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ # Makes it easy to create virtual environments for different versions of dbt ADAPTERS = sqlite duckdb -everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python .venv-dbt13/bin/python .venv-dbt14/bin/python .venv-dbt15/bin/python +everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python .venv-dbt13/bin/python .venv-dbt14/bin/python \ + .venv-dbt15/bin/python .venv-dbt16/bin/python .venv-dbt17/bin/python .venv-dbt18/bin/python .PHONY: everything .venv-dbt10/bin/python: @@ -53,8 +54,39 @@ everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python .venv-dbt15/bin/pip install "dbt-$$adapter>=1.4.0,<1.6.0" "dbt-core>=1.5.0,<1.6.0"; \ done +.venv-dbt16/bin/python: + python -m venv .venv-dbt16 + .venv-dbt16/bin/pip install --upgrade wheel setuptools pip + .venv-dbt16/bin/pip install pytest WebTest . + for adapter in $(ADAPTERS); do \ + .venv-dbt16/bin/pip install "dbt-$$adapter>=1.4.0,<1.7.0"; \ + .venv-dbt16/bin/pip install "dbt-core>=1.6.0,<1.7.0"; \ + done + +.venv-dbt17/bin/python: + python -m venv .venv-dbt17 + .venv-dbt17/bin/pip install --upgrade wheel setuptools pip + .venv-dbt17/bin/pip install pytest WebTest . + for adapter in $(ADAPTERS); do \ + .venv-dbt17/bin/pip install "dbt-$$adapter>=1.4.0,<1.8.0"; \ + .venv-dbt17/bin/pip install "dbt-core>=1.7.0,<1.8.0"; \ + done + +# SQLITE adapter is not supported in dbt 1.8+ yet: https://github.com/codeforkjeff/dbt-sqlite/issues +ADAPTERS = duckdb + +.venv-dbt18/bin/python: + python -m venv .venv-dbt18 + .venv-dbt18/bin/pip install --upgrade wheel setuptools pip + .venv-dbt18/bin/pip install pytest WebTest . + for adapter in $(ADAPTERS); do \ + .venv-dbt18/bin/pip install "dbt-core>=1.8.0,<1.9.0"; \ + .venv-dbt18/bin/pip install "dbt-$$adapter"; \ + done + .venv-dbt18/bin/pip install --force-reinstall dbt-adapters dbt-common; + clean: - rm -rf .venv-dbt10 .venv-dbt11 .venv-dbt12 .venv-dbt13 .venv-dbt14 .venv-dbt15 + rm -rf .venv-dbt10 .venv-dbt11 .venv-dbt12 .venv-dbt13 .venv-dbt14 .venv-dbt15 .venv-dbt16 .venv-dbt17 .venv-dbt18 .PHONY: clean test-dbt1.0: .venv-dbt10/bin/python @@ -81,5 +113,17 @@ test-dbt1.5: .venv-dbt15/bin/python .venv-dbt15/bin/python -m pytest tests/test_main.py .PHONY: test-dbt1.5 -test: test-dbt1.0 test-dbt1.1 test-dbt1.2 test-dbt1.3 test-dbt1.4 test-dbt1.5 +test-dbt1.6: .venv-dbt16/bin/python + .venv-dbt16/bin/python -m pytest tests/test_main.py +.PHONY: test-dbt1.6 + +test-dbt1.7: .venv-dbt17/bin/python + .venv-dbt17/bin/python -m pytest tests/test_main.py +.PHONY: test-dbt1.7 + +test-dbt1.8: .venv-dbt18/bin/python + .venv-dbt18/bin/python -m pytest tests/test_main.py +.PHONY: test-dbt1.8 + +test: test-dbt1.0 test-dbt1.1 test-dbt1.2 test-dbt1.3 test-dbt1.4 test-dbt1.5 test-dbt1.6 test-dbt1.7 test-dbt1.8 .PHONY: test diff --git a/pyproject.toml b/pyproject.toml index 38d52bc..e614fc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ classifiers = ["Development Status :: 3 - Alpha"] Changelog = "https://github.com/z3z1ma/dbt-core-interface/releases" [tool.poetry.dependencies] -python = ">=3.8,<3.12" +python = ">=3.8,<3.13" [tool.poetry.dev-dependencies] Pygments = ">=2.10.0" diff --git a/src/dbt_core_interface/project.py b/src/dbt_core_interface/project.py index 43e92be..441fdcb 100644 --- a/src/dbt_core_interface/project.py +++ b/src/dbt_core_interface/project.py @@ -59,6 +59,7 @@ from io import BytesIO from json import dumps as json_dumps from json import loads as json_lds +from multiprocessing import get_context from pathlib import Path from tempfile import NamedTemporaryFile from traceback import format_exc, print_exc @@ -88,7 +89,12 @@ # We maintain the smallest possible surface area of dbt imports from dbt.adapters.factory import get_adapter_class_by_name -from dbt.clients.system import make_directory +try: + # dbt >= 1.8 + from dbt_common.clients.system import make_directory +except ImportError: + # dbt < 1.8 + from dbt.clients.system import make_directory from dbt.config.runtime import RuntimeConfig from dbt.flags import set_from_args from dbt.node_types import NodeType @@ -112,9 +118,16 @@ # These imports are only used for type checking from agate import Table # type: ignore # No stubs for agate from dbt.adapters.base import BaseAdapter, BaseRelation # type: ignore - from dbt.contracts.connection import AdapterResponse + try: + # dbt >= 1.8 + from dbt.adapters.contracts.connection import AdapterResponse + from dbt_common.semver import VersionSpecifier + + except ImportError: + # dbt < 1.8 + from dbt.contracts.connection import AdapterResponse + from dbt.semver import VersionSpecifier from dbt.contracts.results import ExecutionResult, RunExecutionResult - from dbt.semver import VersionSpecifier from dbt.task.runnable import ManifestTask try: @@ -128,9 +141,15 @@ from agate import Table, Number, Text, Column try: - from dbt.clients.agate_helper import Integer + # dbt >= 1.8 + from dbt_common.clients.agate_helper import Integer except ImportError: - from dbt.clients.agate_helper import Number as Integer + try: + # dbt < 1.8 and older Agate version + from dbt.clients.agate_helper import Integer + except ImportError: + # dbt < 1.8 and newer Agate version + from dbt.clients.agate_helper import Number as Integer # dbt-core-interface is designed for non-standard use. There is no # reason to track usage of this package. disable_tracking() @@ -266,8 +285,10 @@ class DbtConfiguration: use_experimental_parser: bool = False static_parser: bool = False partial_parse: bool = False - # A required attribute for dbt, not used by our interface + # required attributes for dbt, not used by our interface dependencies: List[str] = field(default_factory=list) + REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES: bool = field(default_factory=bool) + which: str = "blah" def __post_init__(self) -> None: """Post init hook to set single_threaded and remove target if not provided.""" @@ -474,7 +495,12 @@ def initialize_adapter(self) -> None: LOGGER.debug(f"Failed to cleanup adapter connections: {e}") # The adapter.setter verifies connection, resets TTL, and updates adapter ref on config # this is thread safe by virtue of the adapter_mutex on the adapter.setter - self.adapter = self.get_adapter_cls()(self.config) + if (__dbt_major_version__, __dbt_minor_version__) < (1, 8): + self.adapter = self.get_adapter_cls()(self.config) + else: + # from dbt 1.8 adapter decoupling onwwards, + # instantiating an Adapter requires a multiprocessing context. + self.adapter = self.get_adapter_cls()(self.config, get_context("spawn")) @property def adapter(self) -> "BaseAdapter": diff --git a/tests/test_main.py b/tests/test_main.py index be59e5a..ea39d09 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -21,8 +21,8 @@ def test_list(): from dbt_core_interface.project import DbtProject project = DbtProject( - project_dir="demo_sqlite", - profiles_dir="demo_sqlite", + project_dir="demo_duckdb", + profiles_dir="demo_duckdb", target="dev", ) nodes = project.list("*") @@ -58,23 +58,23 @@ def test_server(): client = TestApp(app.default) SIMULATED_CLIENTS = 50 # noqa: N806 - DUCKDB_PROJECTS = ( # noqa: N806 - [ - "j_shop_1_duckdb", - "j_shop_2_duckdb", - "h_niceserver_1_duckdb", - "h_niceserver_2_duckdb", - ] - if (__dbt_major_version__, __dbt_minor_version__) < (1, 4) - else [] # DuckDB adapter is not supported in dbt 1.4+ yet - ) - SQLITE_PROJECTS = [ # noqa: N806 - "j_shop_1_sqlite", - "j_shop_2_sqlite", - "j_shop_3_sqlite", - "j_shop_4_sqlite", - "h_niceserver_1_sqlite", + DUCKDB_PROJECTS = [ # noqa: N806 + "j_shop_1_duckdb", + "j_shop_2_duckdb", + "h_niceserver_1_duckdb", + "h_niceserver_2_duckdb", ] + SQLITE_PROJECTS = ( + [ # noqa: N806 + "j_shop_1_sqlite", + "j_shop_2_sqlite", + "j_shop_3_sqlite", + "j_shop_4_sqlite", + "h_niceserver_1_sqlite", + ] + if (__dbt_major_version__, __dbt_minor_version__) < (1, 8) + else [] + ) # SQLITE adapter is not supported in dbt 1.8+ yet: https://github.com/codeforkjeff/dbt-sqlite/issues PROJECTS = DUCKDB_PROJECTS + SQLITE_PROJECTS # noqa: N806 for proj in SQLITE_PROJECTS: