vdk-impala: fix pydantic to 1.x

A week ago pydantic released 2.0 version which is a breaking version . https://docs.pydantic.dev/dev-v2/migration Fortunately pydantic has also released a tool bump-pydantic which automatically upgraded the code. That's very useful. Also in tests use `cli_assert(not res.exception, res)` instead of assert to print more details about the job failure.
vmware · Jul 5, 2023 · e153219 · e153219
1 parent 21ecea2
commit e153219
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 21 deletions.
diff --git a/...mpala/src/vdk/plugin/impala/templates/load/dimension/scd1/00-dimension-scd1-definition.py b/...mpala/src/vdk/plugin/impala/templates/load/dimension/scd1/00-dimension-scd1-definition.py
@@ -15,8 +15,8 @@ class SlowlyChangingDimensionTypeOverwriteParams(BaseModel):
     target_table: str
     source_schema: str
     source_view: str
-    check: Optional[Callable[[str], bool]]
-    staging_schema: Optional[str]
+    check: Optional[Callable[[str], bool]] = None
+    staging_schema: Optional[str] = None
 
 
 class SlowlyChangingDimensionTypeOverwrite(TemplateArgumentsValidator):

diff --git a/...dk-impala/src/vdk/plugin/impala/templates/load/fact/insert/00-fact-snapshot-definition.py b/...dk-impala/src/vdk/plugin/impala/templates/load/fact/insert/00-fact-snapshot-definition.py
@@ -15,8 +15,8 @@ class FactDailySnapshotParams(BaseModel):
     target_table: str
     source_schema: str
     source_view: str
-    check: Optional[Callable[[str], bool]]
-    staging_schema: Optional[str]
+    check: Optional[Callable[[str], bool]] = None
+    staging_schema: Optional[str] = None
 
 
 class FactDailySnapshot(TemplateArgumentsValidator):

diff --git a/...-impala/src/vdk/plugin/impala/templates/load/fact/snapshot/00-fact-snapshot-definition.py b/...-impala/src/vdk/plugin/impala/templates/load/fact/snapshot/00-fact-snapshot-definition.py
@@ -16,8 +16,8 @@ class FactDailySnapshotParams(BaseModel):
     source_schema: str
     source_view: str
     last_arrival_ts: str
-    check: Optional[Callable[[str], bool]]
-    staging_schema: Optional[str]
+    check: Optional[Callable[[str], bool]] = None
+    staging_schema: Optional[str] = None
 
 
 class FactDailySnapshot(TemplateArgumentsValidator):

diff --git a/...gins/vdk-impala/src/vdk/plugin/impala/templates/load/versioned/00-versioned-definition.py b/...gins/vdk-impala/src/vdk/plugin/impala/templates/load/versioned/00-versioned-definition.py
@@ -3,7 +3,7 @@
 from typing import List
 
 from pydantic import BaseModel
-from pydantic import validator
+from pydantic import field_validator
 from vdk.api.job_input import IJobInput
 from vdk.plugin.impala.templates.template_arguments_validator import (
     TemplateArgumentsValidator,
@@ -24,7 +24,7 @@ class LoadVersionedParams(BaseModel):
     active_to_column: str = "active_to"
     active_to_max_value: str = "9999-12-31"
 
-    @validator("tracked_columns", allow_reuse=True)
+    @field_validator("tracked_columns")
     def passwords_match(cls, tracked_columns, values, **kwargs):
         value_columns = values.get("value_columns")
         if type(value_columns) == list and not tracked_columns:

diff --git a/projects/vdk-plugins/vdk-impala/tests/functional/template_regression_test.py b/projects/vdk-plugins/vdk-impala/tests/functional/template_regression_test.py
@@ -13,6 +13,7 @@
 import pytest
 from vdk.internal.core import errors
 from vdk.plugin.impala import impala_plugin
+from vdk.plugin.test_utils.util_funcs import cli_assert
 from vdk.plugin.test_utils.util_funcs import CliEntryBasedTestRunner
 from vdk.plugin.test_utils.util_funcs import get_test_job_path
 
@@ -51,7 +52,7 @@ def test_load_dimension_scd1(self) -> None:
                 "target_table": target_table,
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{source_view}")
         assert actual_rs.output and expected_rs.output
@@ -71,7 +72,7 @@ def test_load_dimension_scd1_partitioned(self) -> None:
                 "target_table": target_table,
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{source_view}")
@@ -127,7 +128,7 @@ def test_load_dimension_scd1_checks_positive(self) -> None:
             },
         )
 
-        assert not res.exception
+        cli_assert(not res.exception, res)
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{source_view}")
         assert actual_rs.output and expected_rs.output
@@ -180,7 +181,7 @@ def test_load_dimension_scd2(self) -> None:
                 "id_column": "id",
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
@@ -259,7 +260,7 @@ def test_load_versioned(self) -> None:
                 "updated_at_column": "updated_at",
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
@@ -308,7 +309,7 @@ def test_load_versioned_partitioned(self) -> None:
                 "updated_at_column": "updated_at",
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
@@ -434,7 +435,7 @@ def test_load_fact_snapshot(self) -> None:
                 "last_arrival_ts": "updated_at",
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
@@ -466,7 +467,7 @@ def test_load_fact_snapshot_empty_source(self) -> None:
             },
         )
         # Expecting data job not to finish due to empty source.
-        assert not res.exception
+        cli_assert(not res.exception, res)
         assert res.exit_code == 0
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
@@ -498,7 +499,7 @@ def test_load_fact_snapshot_partition(self) -> None:
                 "last_arrival_ts": "updated_at",
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
@@ -560,7 +561,7 @@ def test_load_fact_snapshot_checks_positive(self) -> None:
             },
         )
 
-        assert not res.exception
+        cli_assert(not res.exception, res)
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
         assert actual_rs.output and expected_rs.output
@@ -724,7 +725,7 @@ def test_insert(self) -> None:
                 "expect_table": expect_table,
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
@@ -754,7 +755,7 @@ def test_insert_partition(self) -> None:
                 "expect_table": expect_table,
             },
         )
-        assert not res.exception
+        cli_assert(not res.exception, res)
 
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
@@ -788,7 +789,7 @@ def test_insert_checks_positive(self) -> None:
             },
         )
 
-        assert not res.exception
+        cli_assert(not res.exception, res)
         actual_rs = self._run_query(f"SELECT * FROM {test_schema}.{target_table}")
         expected_rs = self._run_query(f"SELECT * FROM {test_schema}.{expect_table}")
         assert actual_rs.output and expected_rs.output