From fa4abc2c15a24e5130dc8c91388dcdb9501cfc7a Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Fri, 10 May 2024 19:47:18 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A7=A3=E5=86=B3clickhouse=E8=A1=A8=E5=90=8D?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=86=99=E6=95=8F=E6=84=9F=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20(#2637)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 解决clickhouse表名大小写敏感问题 解决clickhouse表名大小写敏感问题 * 单元测试 --- sql/engines/clickhouse.py | 30 ++++++++++++++++++------------ sql/engines/tests.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/sql/engines/clickhouse.py b/sql/engines/clickhouse.py index 69dfd78772..c09b25889f 100644 --- a/sql/engines/clickhouse.py +++ b/sql/engines/clickhouse.py @@ -255,7 +255,7 @@ def execute_check(self, db_name=None, sql=""): for statement in sql_list: statement = statement.rstrip(";") # 禁用语句 - if re.match(r"^select|^show", statement.lower()): + if re.match(r"^select|^show", statement, re.M | re.IGNORECASE): result = ReviewResult( id=line, errlevel=2, @@ -273,11 +273,13 @@ def execute_check(self, db_name=None, sql=""): sql=statement, ) # alter语句 - elif re.match(r"^alter", statement.lower()): + elif re.match(r"^alter", statement, re.M | re.IGNORECASE): # alter table语句 - if re.match(r"^alter\s+table\s+(.+?)\s+", statement.lower()): + if re.match( + r"^alter\s+table\s+(.+?)\s+", statement, re.M | re.IGNORECASE + ): table_name = re.match( - r"^alter\s+table\s+(.+?)\s+", statement.lower(), re.M + r"^alter\s+table\s+(.+?)\s+", statement, re.M | re.IGNORECASE ).group(1) if "." not in table_name: table_name = f"{db_name}.{table_name}" @@ -298,7 +300,8 @@ def execute_check(self, db_name=None, sql=""): # delete与update语句，实际是alter语句的变种 if re.match( r"^alter\s+table\s+(.+?)\s+(delete|update)\s+", - statement.lower(), + statement, + re.M | re.IGNORECASE, ): if not table_engine.endswith("MergeTree"): result = ReviewResult( @@ -328,16 +331,18 @@ def execute_check(self, db_name=None, sql=""): else: result = self.explain_check(check_result, db_name, line, statement) # truncate语句 - elif re.match(r"^truncate\s+table\s+(.+?)(\s|$)", statement.lower()): + elif re.match( + r"^truncate\s+table\s+(.+?)(\s|$)", statement, re.M | re.IGNORECASE + ): table_name = re.match( - r"^truncate\s+table\s+(.+?)(\s|$)", statement.lower(), re.M + r"^truncate\s+table\s+(.+?)(\s|$)", statement, re.M | re.IGNORECASE ).group(1) if "." not in table_name: table_name = f"{db_name}.{table_name}" table_engine = self.get_table_engine(table_name)["engine"] table_exist = self.get_table_engine(table_name)["status"] if table_exist == 1: - if table_engine in ("View", "File,", "URL", "Buffer", "Null"): + if table_engine in ("View", "File", "URL", "Buffer", "Null"): result = ReviewResult( id=line, errlevel=2, @@ -358,15 +363,16 @@ def execute_check(self, db_name=None, sql=""): sql=statement, ) # insert语句，explain无法正确判断，暂时只做表存在性检查与简单关键字匹配 - elif re.match(r"^insert", statement.lower()): + elif re.match(r"^insert", statement, re.M | re.IGNORECASE): if re.match( r"^insert\s+into\s+([a-zA-Z_][0-9a-zA-Z_.]+)([\w\W]*?)(values|format|select)(\s+|\()", - statement.lower(), + statement, + re.M | re.IGNORECASE, ): table_name = re.match( r"^insert\s+into\s+([a-zA-Z_][0-9a-zA-Z_.]+)([\w\W]*?)(values|format|select)(\s+|\()", - statement.lower(), - re.M, + statement, + re.M | re.IGNORECASE, ).group(1) if "." not in table_name: table_name = f"{db_name}.{table_name}" diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 74654ac01b..5556829833 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -2106,13 +2106,43 @@ def test_execute_check_alter_sql(self, mock_query): mock_query.return_value = result new_engine = ClickHouseEngine(instance=self.ins1) table_engine = new_engine.get_table_engine(table_name) - alter_sql = "alter table default.tb_test add column remark String" + alter_sql = "alter table tb_test add column remark String" check_result = new_engine.execute_check(db_name="some_db", sql=alter_sql) self.assertEqual( check_result.rows[0].errormessage, "ALTER TABLE仅支持*MergeTree，Merge以及Distributed等引擎表！", ) + @patch.object(ClickHouseEngine, "query") + def test_execute_check_truncate_sql(self, mock_query): + table_name = "default.tb_test" + result = ResultSet() + result.rows = [("File",)] + mock_query.return_value = result + new_engine = ClickHouseEngine(instance=self.ins1) + table_engine = new_engine.get_table_engine(table_name) + alter_sql = "truncate table tb_test" + check_result = new_engine.execute_check(db_name="some_db", sql=alter_sql) + self.assertEqual( + check_result.rows[0].errormessage, + "TRUNCATE不支持View,File,URL,Buffer和Null表引擎！", + ) + + @patch.object(ClickHouseEngine, "query") + def test_execute_check_insert_sql(self, mock_query): + table_name = "default.tb_test" + result = ResultSet() + result.rows = [("Log",)] + mock_query.return_value = result + new_engine = ClickHouseEngine(instance=self.ins1) + table_engine = new_engine.get_table_engine(table_name) + alter_sql = "insert into tb_test(name) values('nick');" + check_result = new_engine.execute_check(db_name="some_db", sql=alter_sql) + self.assertEqual( + check_result.rows[0].errlevel, + 0, + ) + def test_filter_sql_with_delimiter(self): new_engine = ClickHouseEngine(instance=self.ins1) sql_without_limit = "select user from usertable;"