diff --git a/tests/integration_tests/_utils/start_tidb_cluster_impl b/tests/integration_tests/_utils/start_tidb_cluster_impl index 4bd83ae67a9..7c306179836 100755 --- a/tests/integration_tests/_utils/start_tidb_cluster_impl +++ b/tests/integration_tests/_utils/start_tidb_cluster_impl @@ -190,7 +190,6 @@ if [[ "$tidb_config" != "" ]]; then else cat - >"$OUT_DIR/tidb-config.toml" <>>>>>" diff --git a/tests/integration_tests/debezium/sql/data_types.sql b/tests/integration_tests/debezium/sql/data_types.sql new file mode 100644 index 00000000000..48452db04fe --- /dev/null +++ b/tests/integration_tests/debezium/sql/data_types.sql @@ -0,0 +1,405 @@ +SET sql_mode='strict_trans_tables'; +SET time_zone='+06:00'; + +/* +---------------------------------------------------------------------- +-- DATE type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_date( + col DATE, + pk INT PRIMARY KEY +); + +INSERT INTO t_date VALUES ('2023-11-16', 1); +INSERT INTO t_date VALUES ('1000-01-01', 2); +INSERT INTO t_date VALUES ('9999-12-31', 3); + +SET sql_mode=''; +INSERT INTO t_date VALUES (/* Zero dates */ '0000-00-00', 4); +INSERT INTO t_date VALUES (/* Invalid dates */ '2009-11-31', 5); +SET sql_mode='strict_trans_tables'; + +/* +---------------------------------------------------------------------- +-- DATETIME type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_datetime( + col DATETIME, + col_0 DATETIME(0), + col_1 DATETIME(1), + col_2 DATETIME(2), + col_3 DATETIME(3), + col_4 DATETIME(4), + col_5 DATETIME(5), + col_6 DATETIME(6), + col_z DATETIME DEFAULT 0, + -- TODO: + -- col_default_current_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, + pk INT PRIMARY KEY +); + +INSERT INTO t_datetime VALUES ( + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + NULL, + -- '2023-11-16 12:34:56.123456', + 1 +); + +INSERT INTO t_datetime VALUES ( + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + NULL, + -- '2023-11-16 12:34:56', + 2 +); + +SET time_zone='+04:00'; + +INSERT INTO t_datetime VALUES ( + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + NULL, + -- '2023-11-16 12:34:56.123456', + 3 +); + +SET time_zone='+06:00'; + +INSERT INTO t_datetime VALUES ( + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + -- NULL, + 4 +); + +/* +---------------------------------------------------------------------- +-- TIME type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_time( + col TIME, + col_0 TIME(0), + col_1 TIME(1), + col_5 TIME(5), + col_6 TIME(6), + pk INT PRIMARY KEY +); + +INSERT INTO t_time VALUES ( + '00:00:00', + '00:00:00', + '00:00:00', + '00:00:00', + '00:00:00', + 1 +); + +INSERT INTO t_time VALUES ( + '00:00:00.123456', + '00:00:00.123456', + '00:00:00.123456', + '00:00:00.123456', + '00:00:00.123456', + 2 +); + +INSERT INTO t_time VALUES ( + '-10:23:45.123456', + '-10:23:45.123456', + '-10:23:45.123456', + '-10:23:45.123456', + '-10:23:45.123456', + 3 +); + +/* + +Commented out because Debezium produce wrong result: + +"col":-3020399000000, +"col_0":-3020399000000, +"col_1":-3020400147483, +"col_5":-3020399048576, +"col_6":-3020399048576, + +INSERT INTO t_time VALUES ( + '-838:59:59.000000', + '-838:59:59.000000', + '-838:59:59.000000', + '-838:59:59.000000', + '-838:59:59.000000', + 3 +); + +*/ + +INSERT INTO t_time VALUES ( + '838:59:59.000000', + '838:59:59.000000', + '838:59:59.000000', + '838:59:59.000000', + '838:59:59.000000', + 4 +); + +INSERT INTO t_time VALUES ( + '123:44:56.123456', + '123:44:56.123456', + '123:44:56.123456', + '123:44:56.123456', + '123:44:56.123456', + 5 +); + +/* +---------------------------------------------------------------------- +-- TIMESTAMP type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_timestamp( + col TIMESTAMP, + col_0 TIMESTAMP(0), + col_1 TIMESTAMP(1), + col_5 TIMESTAMP(5), + col_6 TIMESTAMP(6), + col_z TIMESTAMP DEFAULT 0, + pk INT PRIMARY KEY +); + +INSERT INTO t_timestamp VALUES ( + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + NULL, + 1 +); + +INSERT INTO t_timestamp VALUES ( + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + '2023-11-16 12:34:56', + NULL, + 2 +); + +SET time_zone='+04:00'; + +INSERT INTO t_timestamp VALUES ( + '2023-11-16 12:34:56.123456', + FROM_UNIXTIME(1), + FROM_UNIXTIME(1470762668), + '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', + NULL, + 3 +); + +SET time_zone='+06:00'; + +/* +---------------------------------------------------------------------- +-- YEAR type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_year( + col YEAR, + col_4 YEAR(4), + pk INT PRIMARY KEY +); + +INSERT INTO t_year VALUES (1901, 1901, 1); + +/* +---------------------------------------------------------------------- +-- BIT type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_bit( + col_1 BIT(1), + col_5 BIT(5), + col_6 BIT(6), + col_60 BIT(60), + pk INT PRIMARY KEY +); + +INSERT INTO t_bit VALUES (0, 16, 16, 16, 1); +INSERT INTO t_bit VALUES (1, 1, 1, 1, 2); + +/* +---------------------------------------------------------------------- +-- VARCHAR type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_varchar( + col VARCHAR(64), + col_utf8_bin VARCHAR(64) + CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + /* + + Note: There is a bug of Debezium that it does not recognize + VARCHAR(..) CHARACTER SET BINARY as binary type + when the table is created AFTER the connector. + + col_bin VARCHAR(64) + CHARACTER SET BINARY, + */ + + pk INT PRIMARY KEY +); + +INSERT INTO t_varchar VALUES ('abc', 'abc', /* 'abc' , */ 1); +INSERT INTO t_varchar VALUES ('def', 'def', /* 0xAABBCC , */ 2); + +/* +---------------------------------------------------------------------- +-- BINARY / VARBINARY type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_binary( + col_1 BINARY(64), + col_2 VARBINARY(64), + pk INT PRIMARY KEY +); + +INSERT INTO t_binary VALUES ('abc', 'abc', 1); +INSERT INTO t_binary VALUES ('def', 'def', 2); + +/* +---------------------------------------------------------------------- +-- BLOB / TEXT type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_blob( + col_b BLOB, + col_t TEXT, + pk INT PRIMARY KEY +); + +INSERT INTO t_blob VALUES ('abc', 'abc', 1); + +/* +---------------------------------------------------------------------- +-- BOOL type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_bool( + /* + + We do not support BOOL type. + Debezium supports BOOL type only when the table is created AFTER the connector. + + col_bool BOOL, + + */ + + col_tinyint_1 TINYINT(1), + col_tinyint_1_u TINYINT(1) UNSIGNED, + col_tinyint_2 TINYINT(2), + pk INT PRIMARY KEY +); + +INSERT INTO t_bool VALUES(/* true, */ 10, 10, 10, 1); +INSERT INTO t_bool VALUES(/* false, */ 10, 10, 10, 2); + + +/* +---------------------------------------------------------------------- +-- FLOAT / DOUBLE type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_float( + col_f FLOAT, + col_d DOUBLE, + pk INT PRIMARY KEY +); + +INSERT INTO t_float VALUES (12345.12345, 12345.12345, 1); + +/* +---------------------------------------------------------------------- +-- DECIMAL / NUMERIC type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_decimal( + col_d DECIMAL(10, 5), + col_n NUMERIC(10, 5), + pk INT PRIMARY KEY +); + +INSERT INTO t_decimal VALUES (12345.12345, 12345.12345, 1); + +/* +---------------------------------------------------------------------- +-- JSON type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_json( + col JSON, + pk INT PRIMARY KEY +); + +INSERT INTO t_json VALUES ('["foo"]', 1); + +/* +---------------------------------------------------------------------- +-- ENUM / SET type +---------------------------------------------------------------------- +*/ + +CREATE TABLE t_enum( + col_e ENUM('a', 'b', 'c'), + col_s SET('a', 'b', 'c'), + pk INT PRIMARY KEY +); + +INSERT INTO t_enum VALUES ('a', 'c', 1); + +SET sql_mode=''; +INSERT INTO t_enum VALUES ('d', 'e', 2); +SET sql_mode='strict_trans_tables'; diff --git a/tests/integration_tests/debezium/sql/debezium/LICENSE.txt b/tests/integration_tests/debezium/sql/debezium/LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tests/integration_tests/debezium/sql/debezium/README.txt b/tests/integration_tests/debezium/sql/debezium/README.txt new file mode 100644 index 00000000000..6979227b498 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/README.txt @@ -0,0 +1,3 @@ +Test files in this directory is ported from: + +https://github.com/debezium/debezium/tree/main/debezium-connector-mysql/src/test/resources/ddl diff --git a/tests/integration_tests/debezium/sql/debezium/binary_column_test.sql b/tests/integration_tests/debezium/sql/debezium/binary_column_test.sql new file mode 100644 index 00000000000..245452e0e27 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/binary_column_test.sql @@ -0,0 +1,14 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: binary_column_test +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE dbz_254_binary_column_test ( + id INT AUTO_INCREMENT NOT NULL, + file_uuid BINARY(16), + PRIMARY KEY (id) +) DEFAULT CHARSET=utf8; + +INSERT INTO dbz_254_binary_column_test VALUES (default, unhex(replace('651aed08-390f-4893-b2f1-36923e7b7400','-',''))); +INSERT INTO dbz_254_binary_column_test VALUES (default, unhex(replace('651aed08-390f-4893-b2f1-36923e7b74ab','-',''))); +INSERT INTO dbz_254_binary_column_test VALUES (default, unhex(replace('651aed08-390f-4893-b2f1-36923e7b74','-',''))); +INSERT INTO dbz_254_binary_column_test VALUES (default, unhex(00)); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/binary_mode_test.sql b/tests/integration_tests/debezium/sql/debezium/binary_mode_test.sql new file mode 100644 index 00000000000..8ad20ccad17 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/binary_mode_test.sql @@ -0,0 +1,31 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: binary_column_test +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE dbz_1814_binary_mode_test ( + id INT AUTO_INCREMENT NOT NULL, + blob_col BLOB NOT NULL, + tinyblob_col TINYBLOB NOT NULL, + mediumblob_col MEDIUMBLOB NOT NULL, + longblob_col LONGBLOB NOT NULL, + binary_col BINARY(3) NOT NULL, + varbinary_col varbinary(20) NOT NULL, + PRIMARY KEY (id) +) DEFAULT CHARSET=utf8; + +INSERT INTO dbz_1814_binary_mode_test ( + id, + blob_col, + tinyblob_col, + mediumblob_col, + longblob_col, + binary_col, + varbinary_col ) +VALUES ( + default, + X'010203', + X'010203', + X'010203', + X'010203', + X'010203', + X'010203' ); diff --git a/tests/integration_tests/debezium/sql/debezium/connector_test.sql b/tests/integration_tests/debezium/sql/debezium/connector_test.sql new file mode 100644 index 00000000000..8ae03647b01 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/connector_test.sql @@ -0,0 +1,73 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: connector_test +-- ---------------------------------------------------------------------------------------------------------------- + +-- Create and populate our products using a single insert with many rows +CREATE TABLE products ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description VARCHAR(512), + weight FLOAT +); +ALTER TABLE products AUTO_INCREMENT = 101; + +INSERT INTO products +VALUES (default,"scooter","Small 2-wheel scooter",3.14), + (default,"car battery","12V car battery",8.1), + (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8), + (default,"hammer","12oz carpenter's hammer",0.75), + (default,"hammer","14oz carpenter's hammer",0.875), + (default,"hammer","16oz carpenter's hammer",1.0), + (default,"rocks","box of assorted rocks",5.3), + (default,"jacket","water resistent black wind breaker",0.1), + (default,"spare tire","24 inch spare tire",22.2); + +-- Create and populate the products on hand using multiple inserts +CREATE TABLE products_on_hand ( + product_id INTEGER NOT NULL PRIMARY KEY, + quantity INTEGER NOT NULL, + FOREIGN KEY (product_id) REFERENCES products(id) +); + +INSERT INTO products_on_hand VALUES (101,3); +INSERT INTO products_on_hand VALUES (102,8); +INSERT INTO products_on_hand VALUES (103,18); +INSERT INTO products_on_hand VALUES (104,4); +INSERT INTO products_on_hand VALUES (105,5); +INSERT INTO products_on_hand VALUES (106,0); +INSERT INTO products_on_hand VALUES (107,44); +INSERT INTO products_on_hand VALUES (108,2); +INSERT INTO products_on_hand VALUES (109,5); + +-- Create some customers ... +CREATE TABLE customers ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + first_name VARCHAR(255) NOT NULL, + last_name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE KEY +) AUTO_INCREMENT=1001; + + +INSERT INTO customers +VALUES (default,"Sally","Thomas","sally.thomas@acme.com"), + (default,"George","Bailey","gbailey@foobar.com"), + (default,"Edward","Walker","ed@walker.com"), + (default,"Anne","Kretchmar","annek@noanswer.org"); + +-- Create some very simple orders +CREATE TABLE orders ( + order_number INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + order_date DATE NOT NULL, + purchaser INTEGER NOT NULL, + quantity INTEGER NOT NULL, + product_id INTEGER NOT NULL, + FOREIGN KEY order_customer (purchaser) REFERENCES customers(id), + FOREIGN KEY ordered_product (product_id) REFERENCES products(id) +) AUTO_INCREMENT = 10001; + +INSERT INTO orders +VALUES (default, '2016-01-16', 1001, 1, 102), + (default, '2016-01-17', 1002, 2, 105), + (default, '2016-02-18', 1004, 3, 109), + (default, '2016-02-19', 1002, 2, 106), + (default, '16-02-21', 1003, 1, 107); diff --git a/tests/integration_tests/debezium/sql/debezium/connector_test_ro.sql b/tests/integration_tests/debezium/sql/debezium/connector_test_ro.sql new file mode 100644 index 00000000000..22ab5d1dda6 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/connector_test_ro.sql @@ -0,0 +1,87 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: connector_test_ro +-- ---------------------------------------------------------------------------------------------------------------- + +-- Create and populate our products using a single insert with many rows +CREATE TABLE Products ( + PRIMARY KEY (id), + id INTEGER NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL, + description VARCHAR(512), + weight FLOAT +); +ALTER TABLE Products AUTO_INCREMENT = 101; + +INSERT INTO Products +VALUES (default,"scooter","Small 2-wheel scooter",3.14), + (default,"car battery","12V car battery",8.1), + (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8), + (default,"hammer","12oz carpenter's hammer",0.75), + (default,"hammer2","14oz carpenter's hammer",8.75E-1), + (default,"hammer3","16oz carpenter's hammer",1.0), + (default,"rocks","box of assorted rocks",5.3), + (default,"jacket","water resistent black wind breaker",0.1), + (default,"spare tire","24 inch spare tire",22.2); + +-- Create and populate the products on hand using multiple inserts +CREATE TABLE products_on_hand ( + product_id INTEGER NOT NULL PRIMARY KEY, + quantity INTEGER NOT NULL, + FOREIGN KEY (product_id) REFERENCES Products(id) +); + +INSERT INTO products_on_hand VALUES (101,3); +INSERT INTO products_on_hand VALUES (102,8); +INSERT INTO products_on_hand VALUES (103,18); +INSERT INTO products_on_hand VALUES (104,4); +INSERT INTO products_on_hand VALUES (105,5); +INSERT INTO products_on_hand VALUES (106,0); +INSERT INTO products_on_hand VALUES (107,44); +INSERT INTO products_on_hand VALUES (108,2); +INSERT INTO products_on_hand VALUES (109,5); + +-- Create some customers ... +CREATE TABLE customers ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + first_name VARCHAR(255) NOT NULL, + last_name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE KEY +) AUTO_INCREMENT=1001; + + +INSERT INTO customers +VALUES (default,"Sally","Thomas","sally.thomas@acme.com"), + (default,"George","Bailey","gbailey@foobar.com"), + (default,"Edward","Walker","ed@walker.com"), + (default,"Anne","Kretchmar","annek@noanswer.org"); + +-- Create some very simple orders +CREATE TABLE orders ( + order_number INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + order_date DATE NOT NULL, + purchaser INTEGER NOT NULL, + quantity INTEGER NOT NULL, + product_id INTEGER NOT NULL, + FOREIGN KEY order_customer (purchaser) REFERENCES customers(id), + FOREIGN KEY ordered_product (product_id) REFERENCES Products(id) +) AUTO_INCREMENT = 10001; + +INSERT INTO orders +VALUES (default, '2016-01-16', 1001, 1, 102), + (default, '2016-01-17', 1002, 2, 105), + (default, '2016-02-18', 1004, 3, 109), + (default, '2016-02-19', 1002, 2, 106), + (default, '2016-02-21', 1003, 1, 107); + + +-- DBZ-342 handle TIME values that exceed the value range of java.sql.Time +CREATE TABLE dbz_342_timetest ( + c1 TIME(2) PRIMARY KEY, + c2 TIME(0), + c3 TIME(3), + c4 TIME(3), + c5 TIME(6) +); +INSERT INTO dbz_342_timetest VALUES ('517:51:04.777', '-13:14:50', '-733:00:00.0011', '-1:59:59.0011', '-838:59:58.999999'); + +CREATE DATABASE IF NOT EXISTS emptydb; diff --git a/tests/integration_tests/debezium/sql/debezium/datetime_key_test.sql b/tests/integration_tests/debezium/sql/debezium/datetime_key_test.sql new file mode 100644 index 00000000000..08740721ba3 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/datetime_key_test.sql @@ -0,0 +1,14 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: datetime_key_test +-- ---------------------------------------------------------------------------------------------------------------- + +SET sql_mode=''; +CREATE TABLE dbz_1194_datetime_key_test ( + id INT AUTO_INCREMENT NOT NULL, + dtval DATETIME NOT NULL, + dval DATE NOT NULL, + tval TIME NOT NULL, + PRIMARY KEY (id, dtval, dval, tval) +) DEFAULT CHARSET=utf8; + +INSERT INTO dbz_1194_datetime_key_test VALUES (default, '0000-00-00 00:00:00', '0000-00-00', '00:00:00'); diff --git a/tests/integration_tests/debezium/sql/debezium/db_default_charset.sql b/tests/integration_tests/debezium/sql/debezium/db_default_charset.sql new file mode 100644 index 00000000000..764da8e67e5 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/db_default_charset.sql @@ -0,0 +1,5 @@ +CREATE TABLE DATA ( + MESSAGE TEXT, + PK INT PRIMARY KEY +); +INSERT INTO DATA VALUES ('Žluťoučký', 1); diff --git a/tests/integration_tests/debezium/sql/debezium/db_default_charset_noutf.sql b/tests/integration_tests/debezium/sql/debezium/db_default_charset_noutf.sql new file mode 100644 index 00000000000..46610614847 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/db_default_charset_noutf.sql @@ -0,0 +1,6 @@ +CREATE TABLE DATA ( + MESSAGE TEXT, + FLAG TINYINT(1), + PK INT PRIMARY KEY +); +INSERT INTO DATA VALUES ('Žluťoučký', 1, 1); diff --git a/tests/integration_tests/debezium/sql/debezium/decimal_column_test.sql b/tests/integration_tests/debezium/sql/debezium/decimal_column_test.sql new file mode 100644 index 00000000000..64cd7742ad5 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/decimal_column_test.sql @@ -0,0 +1,14 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: decimal_column_test +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE dbz_751_decimal_column_test ( + id INT AUTO_INCREMENT NOT NULL, + rating1 DECIMAL, + rating2 DECIMAL(8, 4), + rating3 DECIMAL(7), + rating4 DECIMAL(6, 0), + PRIMARY KEY (id) +) DEFAULT CHARSET=utf8; + +INSERT INTO dbz_751_decimal_column_test VALUES (default, 123, 123.4567, 234.5, 345.6); diff --git a/tests/integration_tests/debezium/sql/debezium/enum_column_test.sql b/tests/integration_tests/debezium/sql/debezium/enum_column_test.sql new file mode 100644 index 00000000000..a8db5dfe308 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/enum_column_test.sql @@ -0,0 +1,16 @@ +CREATE TABLE `test_stations_10` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `name` varchar(500) COLLATE utf8_unicode_ci NOT NULL, + `type` enum('station', 'post_office') COLLATE utf8_unicode_ci NOT NULL DEFAULT 'station', + `created` datetime DEFAULT CURRENT_TIMESTAMP, + `modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +); + +INSERT INTO test_stations_10 (`name`, `type`) values ( 'ha Tinh 7', 'station' ); + +ALTER TABLE `test_stations_10` + MODIFY COLUMN `type` ENUM('station', 'post_office', 'plane', 'ahihi_dongok', 'now', 'test', 'a,b', 'c,\'d', 'g,''h') + CHARACTER SET 'utf8' COLLATE 'utf8_unicode_ci' NOT NULL DEFAULT 'station'; + +INSERT INTO test_stations_10 ( `name`, `type` ) values ( 'Ha Tinh 1', 'now' ); diff --git a/tests/integration_tests/debezium/sql/debezium/multitable_dbz_871.sql b/tests/integration_tests/debezium/sql/debezium/multitable_dbz_871.sql new file mode 100644 index 00000000000..679bb0169dc --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/multitable_dbz_871.sql @@ -0,0 +1,10 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: multitable_statement +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE t1 (ID INT PRIMARY KEY); +CREATE TABLE t2 (ID INT PRIMARY KEY); +CREATE TABLE t3 (ID INT PRIMARY KEY); +CREATE TABLE t4 (ID INT PRIMARY KEY); + +DROP TABLE t1,t2,t3,t4; \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/mysql_dbz_6533.sql b/tests/integration_tests/debezium/sql/debezium/mysql_dbz_6533.sql new file mode 100644 index 00000000000..f51396ebe7f --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql_dbz_6533.sql @@ -0,0 +1,18 @@ +CREATE TABLE tablename_suffix ( + PRIMARY KEY (id), + id INTEGER NOT NULL AUTO_INCREMENT +); + +CREATE TABLE tablename ( + PRIMARY KEY (id), + id INTEGER NOT NULL AUTO_INCREMENT +); + +CREATE TABLE another ( + PRIMARY KEY (id), + id INTEGER NOT NULL AUTO_INCREMENT +); + +INSERT INTO tablename_suffix VALUES (default); +INSERT INTO tablename VALUES (default); +INSERT INTO another VALUES (default); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/nationalized_character_test.sql b/tests/integration_tests/debezium/sql/debezium/nationalized_character_test.sql new file mode 100644 index 00000000000..48cf6071c43 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/nationalized_character_test.sql @@ -0,0 +1,8 @@ +CREATE TABLE `NC_TEST` ( + `id` INT NOT NULL AUTO_INCREMENT, + `nc1` nchar default null, + `nc2` nchar(5) default null, + `nc3` nvarchar(25) default null, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; +INSERT INTO NC_TEST(nc1,nc2,nc3) VALUES ('a', '123', 'hello'); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/numeric_column_test.sql b/tests/integration_tests/debezium/sql/debezium/numeric_column_test.sql new file mode 100644 index 00000000000..6b60b39b928 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/numeric_column_test.sql @@ -0,0 +1,14 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: numeric_column_test +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE dbz_751_numeric_column_test ( + id INT AUTO_INCREMENT NOT NULL, + rating1 NUMERIC, + rating2 NUMERIC(8, 4), + rating3 NUMERIC(7), + rating4 NUMERIC(6, 0), + PRIMARY KEY (id) +) DEFAULT CHARSET=utf8; + +INSERT INTO dbz_751_numeric_column_test VALUES (default, 123, 123.4567, 234.5, 345.6); diff --git a/tests/integration_tests/debezium/sql/debezium/readbinlog_test.sql b/tests/integration_tests/debezium/sql/debezium/readbinlog_test.sql new file mode 100644 index 00000000000..7d6da0e272f --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/readbinlog_test.sql @@ -0,0 +1,23 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: readbinlog_test +-- Database needs to be populated to break dependency between MetadataIT and MySqlConnectorIT.shouldValidateAcceptableConfiguration run order +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE person ( + name VARCHAR(255) primary key, + birthdate DATE NULL, + age INTEGER NULL DEFAULT 10, + salary DECIMAL(5,2), bitStr BIT(18) +); +CREATE TABLE product ( + id INT NOT NULL AUTO_INCREMENT, + createdByDate DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + modifiedDate DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY(id) +); +CREATE TABLE purchased ( + purchaser VARCHAR(255) NOT NULL, + productId INT NOT NULL, + purchaseDate DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(productId,purchaser) +); diff --git a/tests/integration_tests/debezium/sql/debezium/real_test.sql b/tests/integration_tests/debezium/sql/debezium/real_test.sql new file mode 100644 index 00000000000..7fcf64f78ff --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/real_test.sql @@ -0,0 +1,6 @@ +CREATE TABLE `REAL_TEST` ( + `id` INT NOT NULL AUTO_INCREMENT, + `r1` real default 1.25, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; +INSERT INTO REAL_TEST(r1) VALUE (2.36); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/regression_test.sql b/tests/integration_tests/debezium/sql/debezium/regression_test.sql new file mode 100644 index 00000000000..5d396f4d44d --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/regression_test.sql @@ -0,0 +1,132 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: regression_test +-- ---------------------------------------------------------------------------------------------------------------- +-- The integration test for this database expects to scan all of the binlog events associated with this database +-- without error or problems. The integration test does not modify any records in this database, so this script +-- must contain all operations to these tables. + +-- DBZ-61 handle binary value recorded as hex string value +CREATE TABLE t1464075356413_testtable6 ( + pk_column int auto_increment NOT NULL, + varbinary_col varbinary(20) NOT NULL, + PRIMARY KEY(pk_column) +); +INSERT INTO t1464075356413_testtable6 (pk_column, varbinary_col) +VALUES(default, 0x4D7953514C); + +-- DBZ-84 Handle TINYINT +CREATE TABLE dbz84_integer_types_table ( + -- The column lengths are used for display purposes, and do not affect the range of values + colTinyIntA tinyint NOT NULL DEFAULT 100, + colTinyIntB tinyint(1) NOT NULL DEFAULT 101, + colTinyIntC tinyint(2) UNSIGNED NOT NULL DEFAULT 102, + colTinyIntD tinyint(3) UNSIGNED NOT NULL DEFAULT 103, + colSmallIntA smallint NOT NULL DEFAULT 200, + colSmallIntB smallint(1) NOT NULL DEFAULT 201, + colSmallIntC smallint(2) NOT NULL DEFAULT 201, + colSmallIntD smallint(3) NOT NULL DEFAULT 201, + colMediumIntA mediumint NOT NULL DEFAULT 300, + colMediumIntB mediumint(1) NOT NULL DEFAULT 301, + colMediumIntC mediumint(2) NOT NULL DEFAULT 302, + colMediumIntD mediumint(3) NOT NULL DEFAULT 303, + colIntA int NOT NULL DEFAULT 400, + colIntB int(1) NOT NULL DEFAULT 401, + colIntC int(2) NOT NULL DEFAULT 402, + colIntD int(3) NOT NULL DEFAULT 403, + colBigIntA bigint NOT NULL DEFAULT 500, + colBigIntB bigint(1) NOT NULL DEFAULT 501, + colBigIntC bigint(2) NOT NULL DEFAULT 502, + colBigIntD bigint(3) NOT NULL DEFAULT 503, + PK INT PRIMARY KEY +); +INSERT INTO dbz84_integer_types_table +VALUES(127,-128,128,255, default,201,202,203, default,301,302,303, default,401,402,403, default,501,502,503, 1); + +-- DBZ-85 handle fractional part of seconds +CREATE TABLE dbz_85_fractest ( + c1 DATE, + c2 TIME(2), + c3 DATETIME(2), + c4 TIMESTAMP(2), + PK INT PRIMARY KEY +); +INSERT INTO dbz_85_fractest VALUES ('2014-09-08', '17:51:04.777', '2014-09-08 17:51:04.777', '2014-09-08 17:51:04.777', 1); + +-- DBZ-100 handle enum and set +CREATE TABLE dbz_100_enumsettest ( + c1 enUM('a','b','c'), + c2 Set('a','b','c'), + PK INT PRIMARY KEY +); +INSERT INTO dbz_100_enumsettest VALUES ('a', 'a,b,c', 1); +INSERT INTO dbz_100_enumsettest VALUES ('b', 'b,a', 2); +INSERT INTO dbz_100_enumsettest VALUES ('c', 'a', 3); + +-- DBZ-102 handle character sets +-- Use session variables to dictate the character sets used by the client running these commands so +-- the literal value is interpretted correctly... +set character_set_client=utf8; +set character_set_connection=utf8; +CREATE TABLE dbz_102_charsettest ( + id INT(11) NOT NULL AUTO_INCREMENT, + text VARCHAR(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2001 DEFAULT CHARSET=utf8; +INSERT INTO dbz_102_charsettest VALUES (default, "产品"); + +-- DBZ-123 handle bit values, including bit field literals +CREATE TABLE dbz_123_bitvaluetest ( + c1 BIT, + c2 BIT(2), + c3 BIT(8) NOT NULL, + c4 BIT(64), + PK INT PRIMARY KEY +); +INSERT INTO dbz_123_bitvaluetest VALUES (1,2,64,23989979, 1); +INSERT INTO dbz_123_bitvaluetest VALUES (b'1',b'10',b'01000000',b'1011011100000111011011011', 2); + +-- DBZ-104 handle create table like ... +DROP DATABASE IF EXISTS connector_test; +CREATE DATABASE connector_test; +CREATE TABLE connector_test.customers ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + first_name VARCHAR(255) NOT NULL, + last_name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE KEY +) AUTO_INCREMENT=1001; +INSERT INTO connector_test.customers +VALUES (default,"Sally","Thomas","sally.thomas@acme.com"), + (default,"George","Bailey","gbailey@foobar.com"), + (default,"Edward","Walker","ed@walker.com"), + (default,"Anne","Kretchmar","annek@noanswer.org"); + +-- DBZ-147 handle decimal value +CREATE TABLE dbz_147_decimalvalues ( + pk_column int auto_increment NOT NULL, + decimal_value decimal(7,2) NOT NULL, + PRIMARY KEY(pk_column) +); +INSERT INTO dbz_147_decimalvalues (pk_column, decimal_value) +VALUES(default, 12345.67); + +-- DBZ-195 handle numeric values +CREATE TABLE dbz_195_numvalues ( + id int auto_increment NOT NULL, + `search_version_read` int(11) NOT NULL DEFAULT '0', -- (11) is the display width + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=4972 DEFAULT CHARSET=utf8; + +INSERT INTO dbz_195_numvalues VALUES (default,0); +INSERT INTO dbz_195_numvalues VALUES (default,-2147483648); +INSERT INTO dbz_195_numvalues VALUES (default,2147483647); + +-- DBZ-342 handle TIME values that exceed the value range of java.sql.Time +CREATE TABLE dbz_342_timetest ( + c1 TIME(2), + c2 TIME(0), + c3 TIME(3), + c4 TIME(3), + c5 TIME(6), + PK INT PRIMARY KEY +); +INSERT INTO dbz_342_timetest VALUES ('517:51:04.777', '-13:14:50', '-733:00:00.0011', '-1:59:59.0011', '-838:59:58.999999', 1); diff --git a/tests/integration_tests/debezium/sql/debezium/skip_messages_test.sql b/tests/integration_tests/debezium/sql/debezium/skip_messages_test.sql new file mode 100644 index 00000000000..b2260b71be6 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/skip_messages_test.sql @@ -0,0 +1,6 @@ +CREATE TABLE `debezium_test` ( + id INT NOT NULL PRIMARY KEY, + black INT, + white INT +) ENGINE=InnoDB AUTO_INCREMENT=15851 DEFAULT CHARSET=utf8; +INSERT INTO `debezium_test`(id, black, white) VALUES (0,0,0); diff --git a/tests/integration_tests/debezium/sql/debezium/strategy_test.sql b/tests/integration_tests/debezium/sql/debezium/strategy_test.sql new file mode 100644 index 00000000000..a7ba9e50f51 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/strategy_test.sql @@ -0,0 +1,39 @@ +CREATE TABLE `dbz4180` ( + id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + a NUMERIC(10, 2) NOT NULL DEFAULT 1.23, + b DECIMAL(10, 3) NOT NULL DEFAULT 2.321, + c VARCHAR(255) NULL DEFAULT 'default mysql strategy', + d INT NULL DEFAULT '100', + e DATETIME NULL DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; +INSERT INTO `dbz4180`(a, b, c, d) +VALUES (1.33, -2.111 , 'topic strategy', 99); + +CREATE TABLE `dbz_4180_00` ( + id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + a NUMERIC(10, 2) NOT NULL DEFAULT 1.23, + b DECIMAL(10, 3) NOT NULL DEFAULT 2.321, + c VARCHAR(255) NULL DEFAULT 'shard 0', + d INT NULL DEFAULT '100', + e DATETIME NULL DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + +CREATE TABLE `dbz_4180_01` ( + id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + a NUMERIC(10, 2) NOT NULL DEFAULT 1.23, + b DECIMAL(10, 3) NOT NULL DEFAULT 2.321, + c VARCHAR(255) NULL DEFAULT 'shard 1', + d INT NULL DEFAULT '100', + e DATETIME NULL DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + +CREATE TABLE `dbz5743中文` ( + id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + a NUMERIC(10, 2) NOT NULL DEFAULT 1.23, + b DECIMAL(10, 3) NOT NULL DEFAULT 2.321, + c VARCHAR(255) NULL DEFAULT 'default mysql strategy', + d INT NULL DEFAULT '100', + e DATETIME NULL DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; +INSERT INTO `dbz5743中文`(a, b, c, d) +VALUES (1.33, -2.111 , 'topic strategy', 99); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/table_column_comment_test.sql b/tests/integration_tests/debezium/sql/debezium/table_column_comment_test.sql new file mode 100644 index 00000000000..fc7b80aac2a --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/table_column_comment_test.sql @@ -0,0 +1,12 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: table_column_comment_test +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE dbz_4000_comment_test ( + id INT AUTO_INCREMENT NOT NULL COMMENT 'pk', + name VARCHAR(255) NOT NULL COMMENT 'this is name column', + value BIGINT NULL COMMENT 'the value is bigint type', + PRIMARY KEY (id) +) DEFAULT CHARSET=utf8 COMMENT='table for dbz-4000'; + +INSERT INTO dbz_4000_comment_test VALUES (default, 'DBZ-4000', 4000); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/timestamp_column_test.sql b/tests/integration_tests/debezium/sql/debezium/timestamp_column_test.sql new file mode 100644 index 00000000000..c80182cb59f --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/timestamp_column_test.sql @@ -0,0 +1,15 @@ +CREATE TABLE t_user_black_list ( + `id` int(10) unsigned NOT NULL, + `data` varchar(20), + `create_time` datetime, + `update_time` datetime, + PRIMARY KEY (`id`) +); + +ALTER TABLE t_user_black_list + MODIFY COLUMN `update_time` datetime(0) NOT NULL + DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'update_time' AFTER create_time; + +INSERT INTO t_user_black_list (`id`,`create_time`,`data`) VALUES (1, CURRENT_TIMESTAMP(), 'test'); + +UPDATE t_user_black_list SET `data` = 'test2' WHERE `id` = 1; \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/tinyint_test.sql b/tests/integration_tests/debezium/sql/debezium/tinyint_test.sql new file mode 100644 index 00000000000..341dcc8f25b --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/tinyint_test.sql @@ -0,0 +1,8 @@ +CREATE TABLE `DBZ5236` ( + id int(11) not null primary key auto_increment, + ti1 tinyint(3) unsigned NOT NULL DEFAULT '0', + ti2 tinyint(1) unsigned NOT NULL DEFAULT '0', + ti3 tinyint(1) NOT NULL DEFAULT '1' +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + +INSERT INTO DBZ5236 VALUES (DEFAULT, 1, 1, 0); diff --git a/tests/integration_tests/debezium/sql/debezium/topic_name_sanitization_test.sql b/tests/integration_tests/debezium/sql/debezium/topic_name_sanitization_test.sql new file mode 100644 index 00000000000..878cb7cb2a2 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/topic_name_sanitization_test.sql @@ -0,0 +1,19 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: topic-name.sanitization-it +-- ---------------------------------------------------------------------------------------------------------------- + +CREATE TABLE `dbz_878_some|test@data` ( + id INT, + some_col VARCHAR(255), + PRIMARY KEY (id) + ) DEFAULT CHARSET=utf8; + +INSERT INTO `dbz_878_some|test@data` VALUES (1, 'some text'); + +CREATE TABLE `DBZ.1834` ( + id INT, + some_col VARCHAR(255), + PRIMARY KEY (id) + ) DEFAULT CHARSET=utf8; + +INSERT INTO `DBZ.1834` VALUES (1, 'some text'); diff --git a/tests/integration_tests/debezium/sql/debezium/unsigned_integer_test.sql b/tests/integration_tests/debezium/sql/debezium/unsigned_integer_test.sql new file mode 100644 index 00000000000..1ee023a636d --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/unsigned_integer_test.sql @@ -0,0 +1,92 @@ +-- ---------------------------------------------------------------------------------------------------------------- +-- DATABASE: unsigned_integer_test +-- ---------------------------------------------------------------------------------------------------------------- +-- The integration test for this database expects to scan all of the binlog events associated with this database +-- without error or problems. The integration test does not modify any records in this database, so this script +-- must contain all operations to these tables. +-- +-- This relies upon MySQL 5.7's Geometries datatypes. + +-- DBZ-228 handle unsigned TINYINT UNSIGNED +CREATE TABLE dbz_228_tinyint_unsigned ( + id int auto_increment NOT NULL, + c1 TINYINT(3) UNSIGNED ZEROFILL NOT NULL, + c2 TINYINT(3) UNSIGNED NOT NULL, + c3 TINYINT(3) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO dbz_228_tinyint_unsigned VALUES (default, 255, 255, 127); +INSERT INTO dbz_228_tinyint_unsigned VALUES (default, 155, 155, -100); +INSERT INTO dbz_228_tinyint_unsigned VALUES (default, 0, 0, -128); + + +-- DBZ-228 handle unsigned SMALLINT UNSIGNED +CREATE TABLE dbz_228_smallint_unsigned ( + id int auto_increment NOT NULL, + c1 SMALLINT UNSIGNED ZEROFILL NOT NULL, + c2 SMALLINT UNSIGNED NOT NULL, + c3 SMALLINT NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO dbz_228_smallint_unsigned VALUES (default, 65535, 65535, 32767); +INSERT INTO dbz_228_smallint_unsigned VALUES (default, 45535, 45535, -12767); +INSERT INTO dbz_228_smallint_unsigned VALUES (default, 0, 0, -32768); + + +-- DBZ-228 handle unsigned MEDIUMINT UNSIGNED +CREATE TABLE dbz_228_mediumint_unsigned ( + id int auto_increment NOT NULL, + c1 MEDIUMINT UNSIGNED ZEROFILL NOT NULL, + c2 MEDIUMINT UNSIGNED NOT NULL, + c3 MEDIUMINT NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO dbz_228_mediumint_unsigned VALUES (default, 16777215, 16777215, 8388607); +INSERT INTO dbz_228_mediumint_unsigned VALUES (default, 10777215, 10777215, -6388607); +INSERT INTO dbz_228_mediumint_unsigned VALUES (default, 0, 0, -8388608); + +-- DBZ-228 handle unsigned INT UNSIGNED +CREATE TABLE dbz_228_int_unsigned ( + id int auto_increment NOT NULL, + c1 int(11) UNSIGNED ZEROFILL NOT NULL, + c2 int(11) UNSIGNED NOT NULL, + c3 int(11) NOT NULL, + c4 int(5) UNSIGNED ZEROFILL NOT NULL, + c5 int(5) UNSIGNED NOT NULL , + c6 int(5) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO dbz_228_int_unsigned VALUES (default, 4294967295, 4294967295, 2147483647, 4294967295, 4294967295, 2147483647); +INSERT INTO dbz_228_int_unsigned VALUES (default, 3294967295, 3294967295, -1147483647, 3294967295, 3294967295, -1147483647); +INSERT INTO dbz_228_int_unsigned VALUES (default, 0, 0, -2147483648, 0, 0, -2147483648); + + +-- DBZ-228 handle unsigned BIGINT UNSIGNED +CREATE TABLE dbz_228_bigint_unsigned ( + id int auto_increment NOT NULL, + c1 BIGINT UNSIGNED ZEROFILL NOT NULL, + c2 BIGINT UNSIGNED NOT NULL, + c3 BIGINT NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO dbz_228_bigint_unsigned VALUES (default, 18446744073709551615, 18446744073709551615, 9223372036854775807); +INSERT INTO dbz_228_bigint_unsigned VALUES (default, 14446744073709551615, 14446744073709551615, -1223372036854775807); +INSERT INTO dbz_228_bigint_unsigned VALUES (default, 0, 0, -9223372036854775808); + +-- DBZ-1185 handle SERIAL type alias +CREATE TABLE dbz_1185_serial ( + id SERIAL NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO dbz_1185_serial VALUES (10); +INSERT INTO dbz_1185_serial VALUES (default); +INSERT INTO dbz_1185_serial VALUES (18446744073709551615); + +-- DBZ-1185 handle SERIAL default value +CREATE TABLE dbz_1185_serial_default_value ( + id SMALLINT UNSIGNED SERIAL DEFAULT VALUE NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +INSERT INTO dbz_1185_serial_default_value VALUES (10); +INSERT INTO dbz_1185_serial_default_value VALUES (default); +INSERT INTO dbz_1185_serial_default_value VALUES (1000); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/dml.sql b/tests/integration_tests/debezium/sql/dml.sql new file mode 100644 index 00000000000..f8fc3f5644c --- /dev/null +++ b/tests/integration_tests/debezium/sql/dml.sql @@ -0,0 +1,24 @@ +CREATE TABLE foo( + PK INT PRIMARY KEY, + COL INT +); + +INSERT INTO foo VALUES (1, 1); + +INSERT INTO foo VALUES (2, 2); + +INSERT INTO foo VALUES (3, 3); + +/* Update PK */ +UPDATE foo SET PK = 5, COL = 5 WHERE COL = 3; + +/* Update Multiple Rows */ +UPDATE foo SET COL = 4; + +/* Update Single Row */ +UPDATE foo SET COL = 1 WHERE PK = 5; + +/* Update No Rows */ +UPDATE foo SET COL = 1 WHERE PK = 100; + +DELETE FROM foo WHERE PK = 3; diff --git a/tests/integration_tests/debezium/src/db_helper.go b/tests/integration_tests/debezium/src/db_helper.go new file mode 100644 index 00000000000..c5f1265491c --- /dev/null +++ b/tests/integration_tests/debezium/src/db_helper.go @@ -0,0 +1,80 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "database/sql" + "strings" + + "go.uber.org/zap" +) + +type DBHelper struct { + db *sql.DB + kind Kind +} + +func NewDBHelper(kind Kind) *DBHelper { + return &DBHelper{ + db: nil, + kind: kind, + } +} + +func (h *DBHelper) MustOpen(connStringPattern string, dbName string) { + connString := strings.Replace(connStringPattern, "{db}", dbName, -1) + db, err := sql.Open("mysql", connString) + if err != nil { + logger.Panic( + "Failed to open db", + zap.String("kind", string(h.kind)), + zap.String("conn", connString), + zap.Error(err)) + } + err = db.Ping() + if err != nil { + logger.Panic( + "Failed to open db", + zap.String("kind", string(h.kind)), + zap.String("conn", connString), + zap.Error(err)) + } + h.db = db +} + +func (h *DBHelper) MustExec(query string) { + _, err := h.db.Exec(query) + if err != nil { + logger.Panic( + "Failed to execute query", + zap.String("kind", string(h.kind)), + zap.String("query", query), + zap.Error(err)) + } +} + +func (h *DBHelper) Exec(query string) error { + _, err := h.db.Exec(query) + return err +} + +func (h *DBHelper) MustClose() { + err := h.db.Close() + if err != nil { + logger.Panic( + "Failed to close connection", + zap.String("kind", string(h.kind)), + zap.Error(err)) + } +} diff --git a/tests/integration_tests/debezium/src/logger.go b/tests/integration_tests/debezium/src/logger.go new file mode 100644 index 00000000000..5ed7f9387c1 --- /dev/null +++ b/tests/integration_tests/debezium/src/logger.go @@ -0,0 +1,25 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + prettyconsole "github.com/thessem/zap-prettyconsole" + "go.uber.org/zap" +) + +var logger *zap.Logger + +func init() { + logger = prettyconsole.NewLogger(zap.DebugLevel) +} diff --git a/tests/integration_tests/debezium/src/main.go b/tests/integration_tests/debezium/src/main.go new file mode 100644 index 00000000000..d9abea1ca57 --- /dev/null +++ b/tests/integration_tests/debezium/src/main.go @@ -0,0 +1,105 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" + _ "github.com/pingcap/tidb/pkg/types/parser_driver" + "github.com/segmentio/kafka-go" + "go.uber.org/zap" +) + +var ( + dbName *string + testCaseDir *string + kafkaAddr *string +) + +var ( + dbMySQL *DBHelper + dbTiDB *DBHelper + + readerDebezium *kafka.Reader + readerTiCDC *kafka.Reader +) + +type Kind string + +const ( + KindMySQL Kind = "mysql" + KindTiDB Kind = "tidb" +) + +func prepareDBConn(kind Kind, connString string) *DBHelper { + db := NewDBHelper(kind) + db.MustOpen(connString, "") + return db +} + +func prepareKafkaConn(topic string) *kafka.Reader { + r := kafka.NewReader(kafka.ReaderConfig{ + Brokers: []string{*kafkaAddr}, + Topic: topic, + MaxBytes: 10e6, // 10MB + RetentionTime: time.Hour, + + // Config below ensures we will not read history messages. + GroupID: uuid.New().String(), + StartOffset: kafka.LastOffset, + }) + return r +} + +func buildDefaultDBConnStr(port int) string { + return fmt.Sprintf("root@tcp(127.0.0.1:%d)/{db}?allowNativePasswords=true", port) +} + +func main() { + dbConnMySQL := flag.String("db.mysql", buildDefaultDBConnStr(3310), "The connection string to connect to a MySQL instance") + dbConnTiDB := flag.String("db.tidb", buildDefaultDBConnStr(4000), "The connection string to connect to a TiDB instance") + kafkaAddr = flag.String("cdc.kafka", "127.0.0.1:9092", "") + topicDebezium := flag.String("cdc.topic.debezium", "output_debezium", "") + topicTiCDC := flag.String("cdc.topic.ticdc", "output_ticdc", "") + dbName = flag.String("db", "test", "The database to test with") + testCaseDir = flag.String("dir", "./sql", "The directory of SQL test cases") + + flag.Parse() + + logger.Info("Info", + zap.String("db.mysql", *dbConnMySQL), + zap.String("db.tidb", *dbConnTiDB), + zap.String("cdc.mysql", fmt.Sprintf("kafka://%s/%s", *kafkaAddr, *topicDebezium)), + zap.String("cdc.tidb", fmt.Sprintf("kafka://%s/%s", *kafkaAddr, *topicTiCDC)), + ) + + readerDebezium = prepareKafkaConn(*topicDebezium) + defer readerDebezium.Close() + readerTiCDC = prepareKafkaConn(*topicTiCDC) + defer readerTiCDC.Close() + + dbMySQL = prepareDBConn(KindMySQL, *dbConnMySQL) + defer dbMySQL.MustClose() + dbTiDB = prepareDBConn(KindTiDB, *dbConnTiDB) + defer dbTiDB.MustClose() + + if !runAllTestCases(*testCaseDir) { + os.Exit(1) + } +} diff --git a/tests/integration_tests/debezium/src/test_cases.go b/tests/integration_tests/debezium/src/test_cases.go new file mode 100644 index 00000000000..6d2fb9b416c --- /dev/null +++ b/tests/integration_tests/debezium/src/test_cases.go @@ -0,0 +1,310 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/alecthomas/chroma/quick" + "github.com/fatih/color" + "github.com/google/go-cmp/cmp" + "github.com/pingcap/tidb/pkg/parser" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/segmentio/kafka-go" + "go.uber.org/zap" +) + +var ( + nFailed = 0 + nPassed = 0 +) + +func parseSQLText(data string) (res []ast.StmtNode, warns []error, err error) { + p := parser.New() + statements, warns, err := p.Parse(data, "utf8mb4", "utf8mb4_bin") + return statements, warns, err +} + +func readAndParseSQLText(sqlFilePath string) []ast.StmtNode { + data, err := os.ReadFile(sqlFilePath) + if err != nil { + logger.Panic("Failed to read test case file", + zap.String("case", sqlFilePath), + zap.Error(err)) + } + statements, warns, err := parseSQLText(string(data)) + if warns != nil { + logger.Warn("Meet warnings when parsing SQL", + zap.String("case", sqlFilePath), + zap.Any("warnings", warns)) + } + if err != nil { + logger.Panic("Failed to parse SQL", + zap.String("case", sqlFilePath), + zap.Error(err)) + } + return statements +} + +func runAllTestCases(dir string) bool { + var files []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + if !strings.HasSuffix(info.Name(), ".sql") { + return nil + } + files = append(files, path) + return nil + }) + if err != nil { + logger.Panic("Failed to read test case directory", zap.String("dir", dir), zap.Error(err)) + } + + for _, path := range files { + logger.Info("Run", zap.String("case", path)) + runTestCase(path) + } + + if nFailed > 0 { + logger.Error( + "Test finished with error", + zap.Int("passed", nPassed), + zap.Int("failed", nFailed)) + } else { + logger.Info( + "All tests pass", + zap.Int("passed", nPassed), + zap.Int("failed", nFailed)) + } + + return nFailed == 0 +} + +func resetDB(db *DBHelper) { + db.MustExec("drop database if exists `" + *dbName + "`;") + db.MustExec("create database `" + *dbName + "`;") + db.MustExec("use `" + *dbName + "`;") +} + +func runTestCase(testCasePath string) bool { + resetDB(dbMySQL) + resetDB(dbTiDB) + + statementKindsToWaitCDCRecord := map[string]bool{ + "Delete": true, + "Insert": true, + "Replace": true, + "Update": true, + } + + hasError := false + stmtAsts := readAndParseSQLText(testCasePath) + for _, stmt := range stmtAsts { + query := strings.TrimSpace(stmt.Text()) + + waitCDCRows := false + statementKind := ast.GetStmtLabel(stmt) + if v, ok := statementKindsToWaitCDCRecord[statementKind]; v && ok { + waitCDCRows = true + } + + if runSingleQuery(query, waitCDCRows) { + nPassed++ + } else { + nFailed++ + hasError = true + } + } + + return hasError +} + +func fetchNextCDCRecord(reader *kafka.Reader, kind Kind, timeout time.Duration) (map[string]any, error) { + for { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + m, err := reader.FetchMessage(ctx) + cancel() + + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return nil, nil + } + return nil, fmt.Errorf("Failed to read CDC record of %s: %w", kind, err) + } + + if len(m.Value) == 0 { + continue + } + + var obj map[string]any + err = json.Unmarshal(m.Value, &obj) + if err != nil { + return nil, fmt.Errorf("Failed to parse CDC record of %s (msg=%s): %w", kind, m.Value, err) + } + + // Ignore DDL events in the Debezium's output + if kind == KindMySQL { + schema, ok := obj["schema"] + if !ok { + return nil, fmt.Errorf("Unexpected CDC record of %s: schema field not exist in %s", kind, m.Value) + } + if schema.(map[string]any)["name"] == "io.debezium.connector.mysql.SchemaChangeValue" { + continue + } + } + + return obj, nil + } +} + +func fetchAllCDCRecords(reader *kafka.Reader, kind Kind) []map[string]any { + var records []map[string]any + for { + waitTimeout := time.Millisecond * 1000 + if len(records) == 0 { + // Wait a bit longer for the first record + if kind == KindMySQL { + waitTimeout = 10 * time.Second + } else if kind == KindTiDB { + waitTimeout = 20 * time.Second + } + } + + obj, err := fetchNextCDCRecord(reader, kind, waitTimeout) + if err != nil { + logger.Error( + "Received error when fetching CDC record", + zap.Error(err), + zap.String("kind", string(kind))) + break + } + if obj == nil { + // No more records + break + } + + records = append(records, obj) + } + + return records +} + +var ignoredRecordPaths = map[string]bool{ + `{map[string]any}["schema"]`: true, + `{map[string]any}["payload"].(map[string]any)["source"]`: true, + `{map[string]any}["payload"].(map[string]any)["ts_ms"]`: true, +} + +var headingColor = color.New(color.FgHiWhite, color.Bold) + +func printObj(obj any) { + v, _ := json.MarshalIndent(obj, "", " ") + quick.Highlight(os.Stdout, string(v), "json", "terminal16m", "vs") + fmt.Println() +} + +func runSingleQuery(query string, waitCDCRows bool) bool { + { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + dbMySQL.MustExec(query) + wg.Done() + }() + go func() { + dbTiDB.MustExec(query) + wg.Done() + }() + wg.Wait() + } + + if !waitCDCRows { + return true + } + + testCasePassed := true + onError := func(err error) { + fmt.Println("==========================================") + logger.Error("Test failed", zap.Error(err)) + headingColor.Print("\nSQL:\n\n") + quick.Highlight(os.Stdout, query, "sql", "terminal16m", "vs") + fmt.Println() + testCasePassed = false + } + + var objsDebezium []map[string]any + var objsTiCDC []map[string]any + { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + objsDebezium = fetchAllCDCRecords(readerDebezium, KindMySQL) + wg.Done() + }() + go func() { + objsTiCDC = fetchAllCDCRecords(readerTiCDC, KindTiDB) + wg.Done() + }() + wg.Wait() + } + + if len(objsDebezium) != len(objsTiCDC) { + onError(fmt.Errorf( + "Mismatch CDC rows: Got %d rows from Debezium and %d rows from TiCDC", + len(objsDebezium), + len(objsTiCDC))) + headingColor.Print("\nDebezium output:\n\n") + for _, obj := range objsDebezium { + printObj(obj) + } + headingColor.Print("\nTiCDC output:\n\n") + for _, obj := range objsTiCDC { + printObj(obj) + } + return testCasePassed + } + + cmpOption := cmp.FilterPath( + func(p cmp.Path) bool { + path := p.GoString() + _, shouldIgnore := ignoredRecordPaths[path] + return shouldIgnore + }, + cmp.Ignore(), + ) + + for i := 0; i < len(objsDebezium); i++ { + objDebezium := objsDebezium[i] + objTiCDC := objsTiCDC[i] + if diff := cmp.Diff(objDebezium, objTiCDC, cmpOption); diff != "" { + onError(fmt.Errorf("Found mismatch CDC record (output row #%d)", i+1)) + headingColor.Print("\nCDC Result Diff (-debezium +ticdc):\n\n") + quick.Highlight(os.Stdout, diff, "diff", "terminal16m", "murphy") + fmt.Println() + continue + } + } + + return testCasePassed +} diff --git a/tests/integration_tests/run_group.sh b/tests/integration_tests/run_group.sh index c527db9b1a2..a572f9182ad 100755 --- a/tests/integration_tests/run_group.sh +++ b/tests/integration_tests/run_group.sh @@ -14,7 +14,7 @@ mysql_only="bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_sui mysql_only_http="http_api http_api_tls api_v2" mysql_only_consistent_replicate="consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table" -kafka_only="kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro" +kafka_only="kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium" kafka_only_protocol="kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check" kafka_only_v2="kafka_big_txn_v2 kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2" diff --git a/tests/integration_tests/tidb_mysql_test/r/alter_table.result b/tests/integration_tests/tidb_mysql_test/r/alter_table.result index 994f3089989..e002c892d7b 100644 --- a/tests/integration_tests/tidb_mysql_test/r/alter_table.result +++ b/tests/integration_tests/tidb_mysql_test/r/alter_table.result @@ -1,12 +1,3 @@ -drop table if exists warehouse; -create table warehouse ( -id int PRIMARY KEY, -w_idtiny tinyint not null, -w_idmedium mediumint not null); -ALTER TABLE warehouse ADD UNIQUE id_idx (w_idtiny); -ALTER TABLE warehouse DROP INDEX id_idx; -ALTER TABLE warehouse DROP PRIMARY KEY; -ALTER TABLE warehouse DROP COLUMN w_idmedium; drop table if exists t; create table t (c1 int); alter table t add column c2 int; diff --git a/tests/integration_tests/tidb_mysql_test/r/mysql_replace.result b/tests/integration_tests/tidb_mysql_test/r/mysql_replace.result index 6eb7731cbdc..321d469d3ee 100644 --- a/tests/integration_tests/tidb_mysql_test/r/mysql_replace.result +++ b/tests/integration_tests/tidb_mysql_test/r/mysql_replace.result @@ -20,7 +20,7 @@ replace into t2 values (126,"first updated"); replace into t2 values (63,default); select * from t2; a b -127 last -126 first updated 63 default_value +126 first updated +127 last SET sql_mode = default; diff --git a/tests/integration_tests/tidb_mysql_test/t/alter_table.test b/tests/integration_tests/tidb_mysql_test/t/alter_table.test index 034b7732aa3..56b5127801b 100644 --- a/tests/integration_tests/tidb_mysql_test/t/alter_table.test +++ b/tests/integration_tests/tidb_mysql_test/t/alter_table.test @@ -1,17 +1,3 @@ -drop table if exists warehouse; - -create table warehouse ( -id int PRIMARY KEY, -w_idtiny tinyint not null, -w_idmedium mediumint not null); - -ALTER TABLE warehouse ADD UNIQUE id_idx (w_idtiny); -ALTER TABLE warehouse DROP INDEX id_idx; -#Since TiCDC test enable alter-primary-key, we close this error -#--error UNSUPPORT_MODIFY_PK_ERROR -ALTER TABLE warehouse DROP PRIMARY KEY; -ALTER TABLE warehouse DROP COLUMN w_idmedium; - # For add column drop table if exists t; create table t (c1 int); diff --git a/tests/integration_tests/tidb_mysql_test/t/mysql_replace.test b/tests/integration_tests/tidb_mysql_test/t/mysql_replace.test index ef79b9fe6ba..a0660c25bf6 100644 --- a/tests/integration_tests/tidb_mysql_test/t/mysql_replace.test +++ b/tests/integration_tests/tidb_mysql_test/t/mysql_replace.test @@ -34,7 +34,6 @@ insert into t2 values (0,"error"); replace into t2 values (0,"last updated"); replace into t2 values (126,"first updated"); replace into t2 values (63,default); -# Since ticdc open `alter-primary-key`, select result will be different select * from t2; #drop table t1; SET sql_mode = default;