Skip to content

Commit

Permalink
Merge pull request #43 from Infostrux-Solutions/fix-alter-column-add-…
Browse files Browse the repository at this point in the history
…reorg-step

(fix): Add REORG step after ALTER COLUMN
  • Loading branch information
haleemur-infostrux authored Oct 2, 2024
2 parents a222411 + 2fd14dc commit 32513ca
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 9 deletions.
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ ignore = [
"ANN102", # missing-type-cls
"COM812", # missing-trailing-comma
"ISC001", # single-line-implicit-string-concatenation
"S608", # Possible SQL injection vector through string-based query construction
]
exclude = ["target_db2/ibm_db_sa"]
select = ["ALL"]
Expand Down
26 changes: 24 additions & 2 deletions target_db2/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,30 @@ def _adapt_column_type(
column_name=self.quote(column_name),
column_type=compatible_sql_type,
)
table_name_split = full_table_name.split(".", 1)
table_name = table_name_split[-1]
schema_clause = (
f"AND tabschema = '{table_name_split[0]}'"
if len(table_name_split) == 2 # noqa: PLR2004
else ""
)
check_reorg_stmt = sa.text(
f"SELECT true FROM SYSIBMADM.ADMINTABINFO WHERE REORG_PENDING "
f"{schema_clause} AND lower(tabname) = '{table_name}';"
)
reorg_table_stmt = sa.text(
f"CALL SYSPROC.ADMIN_CMD ('REORG TABLE { self.quote(full_table_name) }')"
)

with self._engine.connect() as conn, conn.begin():
conn.execute(alter_column_ddl)
_msg = f"Executed: {alter_column_ddl}"
self.logger.info(_msg)
resp = conn.execute(check_reorg_stmt).scalar()
if resp:
conn.execute(reorg_table_stmt)
_msg = f"Executed: {reorg_table_stmt}"
self.logger.info(_msg)

def _create_empty_column(
self,
Expand Down Expand Up @@ -584,7 +606,7 @@ def merge_upsert_from_table(
WHEN NOT MATCHED THEN
INSERT ({', '.join(final_columns)})
VALUES ({', '.join(load_columns)});
""").strip() # noqa: S608
""").strip()

return sa.text(merge_query)

Expand Down Expand Up @@ -639,7 +661,7 @@ def generate_insert_statement(
INSERT INTO {self.connector.quote(full_table_name)}
({", ".join(column_identifiers)})
VALUES ({", ".join([f":{name}" for name in property_names])})
""", # noqa: S608
""",
)
return statement.rstrip()

Expand Down
22 changes: 15 additions & 7 deletions tests/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
import pytest
from singer_sdk.helpers._compat import importlib_resources
from singer_sdk.testing import get_target_test_class
from singer_sdk.testing.suites import TestSuite
from singer_sdk.testing.suites import TestSuite as TS # noqa: N817
from singer_sdk.testing.templates import TargetFileTestTemplate
from sqlalchemy import (
DECIMAL,
Column,
Integer,
MetaData,
Expand Down Expand Up @@ -122,7 +123,7 @@ class TargetSchemaHasNumericMultipleOf(TargetFileTestTemplateCustomPath):
name = "schema_has_numeric_multipleof"


custom_tests = TestSuite(
custom_tests = TS(
kind="target",
tests=[TargetSchemaChangeToNumericMultipleOf, TargetSchemaHasNumericMultipleOf],
)
Expand Down Expand Up @@ -180,24 +181,31 @@ def test_alter_column() -> None:
"properties": {
"_id": {"type": ["integer"]},
"email": {"type": ["string"], "maxLength": 10},
"val": {"type": ["number"], "multipleOf": 0.1},
}
},
primary_keys=["_id"],
)
connector._adapt_column_type( # noqa: SLF001
"test_alter_column", "email", String(20)
)
connector._adapt_column_type( # noqa: SLF001
"test_alter_column", "val", DECIMAL(10, 3)
)
with connector._engine.connect() as conn, conn.begin(): # noqa: SLF001
resp = conn.execute(
text("""
SELECT typename, length
SELECT colname, typename, length, scale
FROM syscat.columns
WHERE tabname = 'TEST_ALTER_COLUMN'
AND tabschema = 'DB2INST1'
and colname = 'EMAIL'
and colname IN ('EMAIL', 'VAL')
""")
)
_type = resp.fetchone()
assert _type[0] == "VARCHAR"
assert _type[1] == 20
_types = resp.fetchall()
for _type in _types:
if _type[0] == "EMAIL":
assert _type[1:3] == ("VARCHAR", 20)
if _type[0] == "VAL":
assert _type[1:4] == ("DECIMAL", 10, 3)
conn.execute(text("drop table test_alter_column"))
4 changes: 4 additions & 0 deletions tests/testdata/schema_change_to_numeric_multipleof.singer
Original file line number Diff line number Diff line change
@@ -1,2 +1,6 @@
{"type":"SCHEMA","stream":"test_schema_change_numeric_mulipleof","schema":{"properties":{"id":{"type":["integer"]},"val1":{"type":["number","null"]}},"type":"object","required":["id"]},"key_properties":["id"]}
{"type": "RECORD","stream":"test_schema_change_numeric_mulipleof","record": {"id":1,"val1":10.00}}
{"type": "RECORD","stream":"test_schema_change_numeric_mulipleof","record": {"id":2,"val1":20.00}}
{"type": "RECORD","stream":"test_schema_change_numeric_mulipleof","record": {"id":3,"val1":30.00}}
{"type":"SCHEMA","stream":"test_schema_change_numeric_mulipleof","schema":{"properties":{"id":{"type":["integer"]},"val1":{"type":["number","null"], "multipleOf": 0.01}},"type":"object","required":["id"]},"key_properties":["id"]}
{"type": "RECORD","stream":"test_schema_change_numeric_mulipleof","record": {"id":1,"val1":10.25}}

0 comments on commit 32513ca

Please sign in to comment.