Skip to content

Commit

Permalink
vdk-impala: support also pydantic 1.0
Browse files Browse the repository at this point in the history
Make sure vdk-impala works with pydantic 1.0 as this is still used and
some libraries do not support the new version.

In hindsight I probably should have pinned to 1.0 only . But I did not
and it's good idea to support the new version otherwise at some point in
the future it will cause conflicts and problems as other library
adopted.
  • Loading branch information
antoniivanov committed Jul 7, 2023
1 parent 0beefa0 commit 70b2f4e
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 14 deletions.
7 changes: 0 additions & 7 deletions projects/vdk-plugins/vdk-impala/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@

# testing requirements
click
docker-compose
impyla
pyarrow
pydantic
pytest-docker
tabulate
vdk-core
vdk-lineage-model
vdk-test-utils
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,18 @@
from typing import List

from pydantic import BaseModel
from pydantic import field_validator
from pydantic import FieldValidationInfo
from vdk.api.job_input import IJobInput
from vdk.plugin.impala.templates.template_arguments_validator import (
TemplateArgumentsValidator,
)

try:
from pydantic import field_validator, FieldValidationInfo
except ImportError:
from pydantic import validator

FieldValidationInfo = None


class LoadVersionedParams(BaseModel):
target_schema: str
Expand All @@ -25,11 +30,10 @@ class LoadVersionedParams(BaseModel):
active_to_column: str = "active_to"
active_to_max_value: str = "9999-12-31"

@field_validator("tracked_columns")
def tracked_columns_subset_of_value_columns(
cls, tracked_columns: List[str], values: FieldValidationInfo, **kwargs
@staticmethod
def _validate_tracked_columns_subset_of_value_columns(
tracked_columns: List[str], value_columns: List[str]
):
value_columns = values.data["value_columns"]
if type(value_columns) == list and not tracked_columns:
raise ValueError("The list must contain at least one column")
if type(value_columns) == list == type(value_columns) and not set(
Expand All @@ -38,7 +42,30 @@ def tracked_columns_subset_of_value_columns(
raise ValueError(
"All elements in the list must be also present in `value_columns`"
)
return tracked_columns

if FieldValidationInfo is not None:

@field_validator("tracked_columns")
def tracked_columns_subset_of_value_columns(
cls, tracked_columns: List[str], values: FieldValidationInfo, **kwargs
):
value_columns = values.data["value_columns"]
LoadVersionedParams._validate_tracked_columns_subset_of_value_columns(
tracked_columns, value_columns
)
return tracked_columns

else:

@validator("tracked_columns", allow_reuse=True)
def tracked_columns_subset_of_value_columns(
cls, tracked_columns, values, **kwargs
):
value_columns = values.get("value_columns")
LoadVersionedParams._validate_tracked_columns_subset_of_value_columns(
tracked_columns, value_columns
)
return tracked_columns


class LoadVersioned(TemplateArgumentsValidator):
Expand Down

0 comments on commit 70b2f4e

Please sign in to comment.