Merge pull request #5704 from zhang2014/fix/ISSUES-5697

ISSUES-5697 fix insert and select query with mysql style identifier

(cherry picked from commit 8e41d89)

dbms/src/Dictionaries/ExternalQueryBuilder.cpp

@@ -53,6 +53,10 @@ void ExternalQueryBuilder::writeQuoted(const std::string & s, WriteBuffer & out)
         case IdentifierQuotingStyle::DoubleQuotes:
             writeDoubleQuotedString(s, out);
             break;
+
+        case IdentifierQuotingStyle::BackticksMySQL:
+            writeMySQLBackQuotedString(s, out);
+            break;
     }
 }
 

dbms/src/IO/WriteHelpers.h

@@ -255,14 +255,19 @@ inline void writeJSONString(const char * begin, const char * end, WriteBuffer &
 }
 
 
-template <char c>
+/** Will escape quote_character and a list of special characters('\b', '\f', '\n', '\r', '\t', '\0', '\\').
+ *   - when escape_quote_with_quote is true, use backslash to escape list of special characters,
+ *      and use quote_character to escape quote_character. such as: 'hello''world'
+ *   - otherwise use backslash to escape list of special characters and quote_character
+ */
+template <char quote_character, bool escape_quote_with_quote = false>
 void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & buf)
 {
     const char * pos = begin;
     while (true)
     {
         /// On purpose we will escape more characters than minimally necessary.
-        const char * next_pos = find_first_symbols<'\b', '\f', '\n', '\r', '\t', '\0', '\\', c>(pos, end);
+        const char * next_pos = find_first_symbols<'\b', '\f', '\n', '\r', '\t', '\0', '\\', quote_character>(pos, end);
 
         if (next_pos == end)
         {
@@ -303,10 +308,15 @@ void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & b
                     writeChar('\\', buf);
                     writeChar('\\', buf);
                     break;
-                case c:
-                    writeChar('\\', buf);
-                    writeChar(c, buf);
+                case quote_character:
+                {
+                    if constexpr (escape_quote_with_quote)
+                        writeChar(quote_character, buf);
+                    else
+                        writeChar('\\', buf);
+                    writeChar(quote_character, buf);
                     break;
+                }
                 default:
                     writeChar(*pos, buf);
             }
@@ -353,27 +363,27 @@ inline void writeEscapedString(const StringRef & ref, WriteBuffer & buf)
 }
 
 
-template <char c>
+template <char quote_character>
 void writeAnyQuotedString(const char * begin, const char * end, WriteBuffer & buf)
 {
-    writeChar(c, buf);
-    writeAnyEscapedString<c>(begin, end, buf);
-    writeChar(c, buf);
+    writeChar(quote_character, buf);
+    writeAnyEscapedString<quote_character>(begin, end, buf);
+    writeChar(quote_character, buf);
 }
 
 
 
-template <char c>
+template <char quote_character>
 void writeAnyQuotedString(const String & s, WriteBuffer & buf)
 {
-    writeAnyQuotedString<c>(s.data(), s.data() + s.size(), buf);
+    writeAnyQuotedString<quote_character>(s.data(), s.data() + s.size(), buf);
 }
 
 
-template <char c>
+template <char quote_character>
 void writeAnyQuotedString(const StringRef & ref, WriteBuffer & buf)
 {
-    writeAnyQuotedString<c>(ref.data, ref.data + ref.size, buf);
+    writeAnyQuotedString<quote_character>(ref.data, ref.data + ref.size, buf);
 }
 
 
@@ -393,12 +403,20 @@ inline void writeDoubleQuotedString(const String & s, WriteBuffer & buf)
     writeAnyQuotedString<'"'>(s, buf);
 }
 
-/// Outputs a string in backquotes, as an identifier in MySQL.
+/// Outputs a string in backquotes.
 inline void writeBackQuotedString(const String & s, WriteBuffer & buf)
 {
     writeAnyQuotedString<'`'>(s, buf);
 }
 
+/// Outputs a string in backquotes for MySQL.
+inline void writeMySQLBackQuotedString(const String & s, WriteBuffer & buf)
+{
+    writeChar('`', buf);
+    writeAnyEscapedString<'`', true>(s.data(), s.data() + s.size(), buf);
+    writeChar('`', buf);
+}
+
 
 /// The same, but quotes apply only if there are characters that do not match the identifier without quotes.
 template <typename F>
@@ -430,6 +448,11 @@ inline void writeProbablyDoubleQuotedString(const String & s, WriteBuffer & buf)
     writeProbablyQuotedStringImpl(s, buf, [](const String & s_, WriteBuffer & buf_) { return writeDoubleQuotedString(s_, buf_); });
 }
 
+inline void writeProbablyMySQLQuotedString(const String & s, WriteBuffer & buf)
+{
+    writeProbablyQuotedStringImpl(s, buf, [](const String & s_, WriteBuffer & buf_) { return writeMySQLBackQuotedString(s_, buf_); });
+}
+
 
 /** Outputs the string in for the CSV format.
   * Rules:

dbms/src/Parsers/IAST.cpp

@@ -144,6 +144,14 @@ void IAST::FormatSettings::writeIdentifier(const String & name) const
                 writeProbablyDoubleQuotedString(name, out);
             break;
         }
+        case IdentifierQuotingStyle::BackticksMySQL:
+        {
+            if (always_quote_identifiers)
+                writeMySQLBackQuotedString(name, out);
+            else
+                writeProbablyMySQLQuotedString(name, out);
+            break;
+        }
     }
 
     out.next();

dbms/src/Parsers/IdentifierQuotingStyle.h

@@ -8,9 +8,10 @@ namespace DB
 /// NOTE There could be differences in escaping rules inside quotes. Escaping rules may not match that required by specific external DBMS.
 enum class IdentifierQuotingStyle
 {
-    None,           /// Write as-is, without quotes.
-    Backticks,      /// `mysql` style
-    DoubleQuotes    /// "postgres" style
+    None,            /// Write as-is, without quotes.
+    Backticks,       /// `clickhouse` style
+    DoubleQuotes,    /// "postgres" style
+    BackticksMySQL,  /// `mysql` style, most same as Backticks, but it uses '``' to escape '`'
 };
 
 }

dbms/src/Storages/StorageMySQL.cpp

@@ -26,6 +26,15 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
 }
 
+String backQuoteMySQL(const String & x)
+{
+    String res(x.size(), '\0');
+    {
+        WriteBufferFromString wb(res);
+        writeMySQLBackQuotedString(x, wb);
+    }
+    return res;
+}
 
 StorageMySQL::StorageMySQL(const std::string & name,
     mysqlxx::Pool && pool,
@@ -57,7 +66,7 @@ BlockInputStreams StorageMySQL::read(
 {
     check(column_names);
     String query = transformQueryForExternalDatabase(
-        *query_info.query, getColumns().getOrdinary(), IdentifierQuotingStyle::Backticks, remote_database_name, remote_table_name, context);
+        *query_info.query, getColumns().getOrdinary(), IdentifierQuotingStyle::BackticksMySQL, remote_database_name, remote_table_name, context);
 
     Block sample_block;
     for (const String & column_name : column_names)
@@ -111,7 +120,7 @@ class StorageMySQLBlockOutputStream : public IBlockOutputStream
     {
         WriteBufferFromOwnString sqlbuf;
         sqlbuf << (storage.replace_query ? "REPLACE" : "INSERT") << " INTO ";
-        sqlbuf << backQuoteIfNeed(remote_database_name) << "." << backQuoteIfNeed(remote_table_name);
+        sqlbuf << backQuoteMySQL(remote_database_name) << "." << backQuoteMySQL(remote_table_name);
         sqlbuf << " (" << dumpNamesWithBackQuote(block) << ") VALUES ";
 
         auto writer = FormatFactory::instance().getOutput("Values", sqlbuf, storage.getSampleBlock(), storage.global_context);
@@ -163,7 +172,7 @@ class StorageMySQLBlockOutputStream : public IBlockOutputStream
         {
             if (it != block.begin())
                 out << ", ";
-            out << backQuoteIfNeed(it->name);
+            out << backQuoteMySQL(it->name);
         }
         return out.str();
     }

dbms/tests/integration/test_mysql_database_engine/test.py

@@ -0,0 +1,138 @@
+from contextlib import contextmanager
+
+import time
+import pytest
+
+## sudo -H pip install PyMySQL
+import pymysql.cursors
+
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+
+node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql = True)
+create_table_normal_sql_template = """
+    CREATE TABLE `clickhouse`.`{}` (
+        `id` int(11) NOT NULL,
+        `name` varchar(50) NOT NULL,
+        `age` int  NOT NULL default 0,
+        `money` int NOT NULL default 0,
+        PRIMARY KEY (`id`)
+    ) ENGINE=InnoDB;
+    """
+
+create_table_mysql_style_sql_template = """
+    CREATE TABLE `clickhouse`.`{}` (
+        `id` int(11) NOT NULL,
+        `float` float NOT NULL,
+        `Float32` float NOT NULL,
+        `test``name` varchar(50) NOT NULL,
+        PRIMARY KEY (`id`)
+    ) ENGINE=InnoDB;
+    """
+
+drop_table_sql_template = "DROP TABLE `clickhouse`.`{}`"
+
+add_column_sql_template = "ALTER TABLE `clickhouse`.`{}` ADD COLUMN `pid` int(11)"
+del_column_sql_template = "ALTER TABLE `clickhouse`.`{}` DROP COLUMN `pid`"
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+
+        conn = get_mysql_conn()
+        ## create mysql db and table
+        create_mysql_db(conn, 'clickhouse')
+        node1.query("CREATE DATABASE clickhouse_mysql ENGINE = MySQL('mysql1:3306', 'clickhouse', 'root', 'clickhouse')")
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_sync_tables_list_between_clickhouse_and_mysql(started_cluster):
+    mysql_connection = get_mysql_conn()
+    assert node1.query('SHOW TABLES FROM clickhouse_mysql FORMAT TSV').rstrip() == ''
+
+    create_normal_mysql_table(mysql_connection, 'first_mysql_table')
+    assert node1.query("SHOW TABLES FROM clickhouse_mysql LIKE 'first_mysql_table' FORMAT TSV").rstrip() == 'first_mysql_table'
+
+    create_normal_mysql_table(mysql_connection, 'second_mysql_table')
+    assert node1.query("SHOW TABLES FROM clickhouse_mysql LIKE 'second_mysql_table' FORMAT TSV").rstrip() == 'second_mysql_table'
+
+    drop_mysql_table(mysql_connection, 'second_mysql_table')
+    assert node1.query("SHOW TABLES FROM clickhouse_mysql LIKE 'second_mysql_table' FORMAT TSV").rstrip() == ''
+
+    mysql_connection.close()
+
+def test_sync_tables_structure_between_clickhouse_and_mysql(started_cluster):
+    mysql_connection = get_mysql_conn()
+
+    create_normal_mysql_table(mysql_connection, 'test_sync_column')
+
+    assert node1.query(
+        "SELECT name FROM system.columns WHERE table = 'test_sync_column' AND database = 'clickhouse_mysql' AND name = 'pid' ").rstrip() == ''
+
+    time.sleep(3)
+    add_mysql_table_column(mysql_connection, "test_sync_column")
+
+    assert node1.query(
+        "SELECT name FROM system.columns WHERE table = 'test_sync_column' AND database = 'clickhouse_mysql' AND name = 'pid' ").rstrip() == 'pid'
+
+    time.sleep(3)
+    drop_mysql_table_column(mysql_connection, "test_sync_column")
+    assert node1.query(
+        "SELECT name FROM system.columns WHERE table = 'test_sync_column' AND database = 'clickhouse_mysql' AND name = 'pid' ").rstrip() == ''
+
+    mysql_connection.close()
+
+def test_insert_select(started_cluster):
+    mysql_connection = get_mysql_conn()
+    create_normal_mysql_table(mysql_connection, 'test_insert_select')
+
+    assert node1.query("SELECT count() FROM `clickhouse_mysql`.{}".format('test_insert_select')).rstrip() == '0'
+    node1.query("INSERT INTO `clickhouse_mysql`.{}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format('test_insert_select'))
+    assert node1.query("SELECT count() FROM `clickhouse_mysql`.{}".format('test_insert_select')).rstrip() == '10000'
+    assert node1.query("SELECT sum(money) FROM `clickhouse_mysql`.{}".format('test_insert_select')).rstrip() == '30000'
+    mysql_connection.close()
+
+def test_insert_select_with_mysql_style_table(started_cluster):
+    mysql_connection = get_mysql_conn()
+    create_mysql_style_mysql_table(mysql_connection, 'test_mysql``_style_table')
+
+    assert node1.query("SELECT count() FROM `clickhouse_mysql`.`{}`".format('test_mysql\`_style_table')).rstrip() == '0'
+    node1.query("INSERT INTO `clickhouse_mysql`.`{}`(id, `float`, `Float32`, `test\`name`) select number, 3, 3, 'name' from numbers(10000) ".format('test_mysql\`_style_table'))
+    assert node1.query("SELECT count() FROM `clickhouse_mysql`.`{}`".format('test_mysql\`_style_table')).rstrip() == '10000'
+    assert node1.query("SELECT sum(`float`) FROM `clickhouse_mysql`.`{}`".format('test_mysql\`_style_table')).rstrip() == '30000'
+    mysql_connection.close()
+
+def get_mysql_conn():
+    conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.1', port=3308)
+    return conn
+
+def create_mysql_db(conn, name):
+    with conn.cursor() as cursor:
+        cursor.execute(
+            "CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name))
+
+def create_normal_mysql_table(conn, table_name):
+    with conn.cursor() as cursor:
+        cursor.execute(create_table_normal_sql_template.format(table_name))
+
+def create_mysql_style_mysql_table(conn, table_name):
+    with conn.cursor() as cursor:
+        cursor.execute(create_table_mysql_style_sql_template.format(table_name))
+
+def drop_mysql_table(conn, table_name):
+    with conn.cursor() as cursor:
+        cursor.execute(drop_table_sql_template.format(table_name))
+
+def add_mysql_table_column(conn, table_name):
+    with conn.cursor() as cursor:
+        cursor.execute(add_column_sql_template.format(table_name))
+
+def drop_mysql_table_column(conn, table_name):
+    with conn.cursor() as cursor:
+        cursor.execute(del_column_sql_template.format(table_name))