Skip to content

Commit

Permalink
capturing recon error
Browse files Browse the repository at this point in the history
  • Loading branch information
nkvuong committed May 23, 2024
1 parent 1b96b25 commit faa035e
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 7 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
-- viz type=table, name=Migration status, search_by=table_name,upgraded_status, columns=table_name,object_type,table_format,location,upgraded_status,upgraded_to_table_name,schema_matches,column_comparison,data_matches,source_row_count,target_row_count,source_missing_count,target_missing_count,view_text,storage_properties,is_partitioned
-- viz type=table, name=Migration status, search_by=table_name,upgraded_status, columns=table_name,object_type,table_format,location,upgraded_status,upgraded_to_table_name,schema_matches,column_comparison,data_matches,source_row_count,target_row_count,source_missing_count,target_missing_count,reconciliation_error,view_text,storage_properties,is_partitioned
-- widget title=Migration status, row=3, col=0, size_x=8, size_y=16
SELECT
concat_ws('.', tables.`catalog`, tables.`database`, tables.name) AS table_name,
Expand All @@ -17,6 +17,7 @@ SELECT
reconciliation_results.target_row_count,
reconciliation_results.source_missing_count,
reconciliation_results.target_missing_count,
reconciliation_results.error_message as reconciliation_error,
tables.view_text,
tables.storage_properties,
tables.is_partitioned
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ SELECT
data_comparison_result.target_row_count,
data_comparison_result.source_missing_count,
data_comparison_result.target_missing_count,
schema_comparison_result.data AS column_comparison
schema_comparison_result.data AS column_comparison,
error_message
FROM
flattened
18 changes: 13 additions & 5 deletions src/databricks/labs/ucx/recon/migration_recon.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ class ReconResult:
dst_catalog: str
dst_schema: str
dst_table: str
schema_matches: bool
data_matches: bool
schema_comparison: str
data_comparison: str
schema_matches: bool = False
data_matches: bool = False
schema_comparison: str | None = None
data_comparison: str | None = None
error_message: str | None = None


class MigrationRecon(CrawlerBase[ReconResult]):
Expand Down Expand Up @@ -108,7 +109,14 @@ def _recon(
data_comparison = self._data_comparator.compare_data(source, target, compare_rows)
except DatabricksError as e:
logger.warning(f"Error while comparing {source.fqn_escaped} and {target.fqn_escaped}: {e}")
return None
return ReconResult(

Check warning on line 112 in src/databricks/labs/ucx/recon/migration_recon.py

View check run for this annotation

Codecov / codecov/patch

src/databricks/labs/ucx/recon/migration_recon.py#L111-L112

Added lines #L111 - L112 were not covered by tests
source.schema,
source.table,
target.catalog,
target.schema,
target.table,
error_message=str(e),
)
recon_results = ReconResult(
source.schema,
source.table,
Expand Down

0 comments on commit faa035e

Please sign in to comment.