Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add dbt build operators #795

Merged
merged 2 commits into from
Jan 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cosmos/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from cosmos.log import get_logger
from cosmos.operators.lazy_load import MissingPackage
from cosmos.operators.local import (
DbtBuildLocalOperator,
DbtDepsLocalOperator,
DbtLSLocalOperator,
DbtRunLocalOperator,
Expand Down Expand Up @@ -97,6 +98,7 @@
"DbtRunLocalOperator",
"DbtSeedLocalOperator",
"DbtTestLocalOperator",
"DbtBuildLocalOperator",
"DbtDepsLocalOperator",
"DbtSnapshotLocalOperator",
"DbtDag",
Expand All @@ -106,12 +108,14 @@
"DbtRunDockerOperator",
"DbtSeedDockerOperator",
"DbtTestDockerOperator",
"DbtBuildDockerOperator",
"DbtSnapshotDockerOperator",
"DbtLSKubernetesOperator",
"DbtRunOperationKubernetesOperator",
"DbtRunKubernetesOperator",
"DbtSeedKubernetesOperator",
"DbtTestKubernetesOperator",
"DbtBuildKubernetesOperator",
"DbtSnapshotKubernetesOperator",
"ExecutionMode",
"LoadMode",
Expand Down
2 changes: 2 additions & 0 deletions cosmos/operators/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from .local import DbtBuildLocalOperator as DbtBuildOperator
from .local import DbtDepsLocalOperator as DbtDepsOperator
from .local import DbtDocsAzureStorageLocalOperator as DbtDocsAzureStorageOperator
from .local import DbtDocsLocalOperator as DbtDocsOperator
Expand All @@ -16,6 +17,7 @@
"DbtSnapshotOperator",
"DbtRunOperator",
"DbtTestOperator",
"DbtBuildOperator",
"DbtRunOperationOperator",
"DbtDepsOperator",
"DbtDocsOperator",
Expand Down
7 changes: 7 additions & 0 deletions cosmos/operators/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,13 @@ def build_cmd(
return dbt_cmd, env


class DbtBuildMixin:
"""Mixin for dbt build command."""

base_cmd = ["build"]
ui_color = "#8194E0"


class DbtLSMixin:
"""
Executes a dbt core ls command.
Expand Down
7 changes: 7 additions & 0 deletions cosmos/operators/docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from cosmos.log import get_logger
from cosmos.operators.base import (
AbstractDbtBaseOperator,
DbtBuildMixin,
DbtRunMixin,
DbtSeedMixin,
DbtSnapshotMixin,
Expand Down Expand Up @@ -66,6 +67,12 @@ def execute(self, context: Context) -> None:
self.build_and_run_cmd(context=context)


class DbtBuildDockerOperator(DbtBuildMixin, DbtDockerBaseOperator):
"""
Executes a dbt core build command.
"""


class DbtLSDockerOperator(DbtLSMixin, DbtDockerBaseOperator):
"""
Executes a dbt core ls command.
Expand Down
7 changes: 7 additions & 0 deletions cosmos/operators/kubernetes.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from cosmos.config import ProfileConfig
from cosmos.operators.base import (
AbstractDbtBaseOperator,
DbtBuildMixin,
DbtRunMixin,
DbtSeedMixin,
DbtSnapshotMixin,
Expand Down Expand Up @@ -100,6 +101,12 @@ def execute(self, context: Context) -> None:
self.build_and_run_cmd(context=context)


class DbtBuildKubernetesOperator(DbtBuildMixin, DbtKubernetesBaseOperator):
"""
Executes a dbt core build command.
"""


class DbtLSKubernetesOperator(DbtLSMixin, DbtKubernetesBaseOperator):
"""
Executes a dbt core ls command.
Expand Down
7 changes: 7 additions & 0 deletions cosmos/operators/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from cosmos.log import get_logger
from cosmos.operators.base import (
AbstractDbtBaseOperator,
DbtBuildMixin,
DbtRunMixin,
DbtSeedMixin,
DbtSnapshotMixin,
Expand Down Expand Up @@ -384,6 +385,12 @@ def on_kill(self) -> None:
self.subprocess_hook.send_sigterm()


class DbtBuildLocalOperator(DbtBuildMixin, DbtLocalBaseOperator):
"""
Executes a dbt core build command.
"""


class DbtLSLocalOperator(DbtLSMixin, DbtLocalBaseOperator):
"""
Executes a dbt core ls command.
Expand Down
8 changes: 8 additions & 0 deletions cosmos/operators/virtualenv.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from cosmos.log import get_logger
from cosmos.operators.local import (
DbtBuildLocalOperator,
DbtDocsLocalOperator,
DbtLocalBaseOperator,
DbtLSLocalOperator,
Expand Down Expand Up @@ -98,6 +99,13 @@ def execute(self, context: Context) -> None:
logger.info(output)


class DbtBuildVirtualenvOperator(DbtVirtualenvBaseOperator, DbtBuildLocalOperator):
"""
Executes a dbt core build command within a Python Virtual Environment, that is created before running the dbt command
and deleted just after.
"""


class DbtLSVirtualenvOperator(DbtVirtualenvBaseOperator, DbtLSLocalOperator):
"""
Executes a dbt core ls command within a Python Virtual Environment, that is created before running the dbt command
Expand Down
2 changes: 2 additions & 0 deletions tests/operators/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from cosmos.operators.base import (
AbstractDbtBaseOperator,
DbtBuildMixin,
DbtLSMixin,
DbtSeedMixin,
DbtRunOperationMixin,
Expand All @@ -26,6 +27,7 @@ def test_dbt_base_operator_is_abstract():
("ls", DbtLSMixin),
("seed", DbtSeedMixin),
("run", DbtRunMixin),
("build", DbtBuildMixin),
],
)
def test_dbt_mixin_base_cmd(dbt_command, dbt_operator_class):
Expand Down
2 changes: 2 additions & 0 deletions tests/operators/test_docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from pendulum import datetime

from cosmos.operators.docker import (
DbtBuildDockerOperator,
DbtDockerBaseOperator,
DbtLSDockerOperator,
DbtRunDockerOperator,
Expand Down Expand Up @@ -84,6 +85,7 @@ def test_dbt_docker_operator_get_env(p_context_to_airflow_vars: MagicMock) -> No
"ls": DbtLSDockerOperator(**base_kwargs),
"run": DbtRunDockerOperator(**base_kwargs),
"test": DbtTestDockerOperator(**base_kwargs),
"build": DbtBuildDockerOperator(**base_kwargs),
"seed": DbtSeedDockerOperator(**base_kwargs),
}

Expand Down
2 changes: 2 additions & 0 deletions tests/operators/test_kubernetes.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from pendulum import datetime

from cosmos.operators.kubernetes import (
DbtBuildKubernetesOperator,
DbtKubernetesBaseOperator,
DbtLSKubernetesOperator,
DbtRunKubernetesOperator,
Expand Down Expand Up @@ -94,6 +95,7 @@ def test_dbt_kubernetes_operator_get_env(p_context_to_airflow_vars: MagicMock) -
"ls": DbtLSKubernetesOperator(**base_kwargs),
"run": DbtRunKubernetesOperator(**base_kwargs),
"test": DbtTestKubernetesOperator(**base_kwargs),
"build": DbtBuildKubernetesOperator(**base_kwargs),
"seed": DbtSeedKubernetesOperator(**base_kwargs),
}

Expand Down
2 changes: 2 additions & 0 deletions tests/operators/test_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
DbtSnapshotLocalOperator,
DbtRunLocalOperator,
DbtTestLocalOperator,
DbtBuildLocalOperator,
DbtDocsLocalOperator,
DbtDocsS3LocalOperator,
DbtDocsAzureStorageLocalOperator,
Expand Down Expand Up @@ -380,6 +381,7 @@ def test_operator_execute_with_flags(mock_build_and_run_cmd, operator_class, kwa
DbtLSLocalOperator,
DbtSnapshotLocalOperator,
DbtTestLocalOperator,
DbtBuildLocalOperator,
DbtDocsLocalOperator,
DbtDocsS3LocalOperator,
DbtDocsAzureStorageLocalOperator,
Expand Down
Loading