Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fix][JDBC] Fix starrocks jdbc dialect catalog conflict with starrocks connector #7578

Merged
merged 2 commits into from
Sep 13, 2024
Merged

[Fix][JDBC] Fix starrocks jdbc dialect catalog conflict with starrocks connector #7578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fc33fa4
[hotfix] fix sr jdbc dialect catalog conflict with sr connector
liunaijie Sep 3, 2024
45e5ba4
add test case
liunaijie Sep 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions ...ache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/starrocks/StarRocksDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.starrocks;

import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.DatabaseIdentifier;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.mysql.MysqlDialect;

public class StarRocksDialect extends MysqlDialect {
Expand All @@ -28,13 +27,8 @@ public StarRocksDialect(String fieldIde) {
this.fieldIde = fieldIde;
}

@Override
public String dialectName() {
return DatabaseIdentifier.STARROCKS;
}

@Override
public String hashModForField(String fieldName, int mod) {
return "ABS(md5sum_numeric(" + quoteIdentifier(fieldName) + ") % " + mod + ")";
return "ABS(murmur_hash3_32(" + quoteIdentifier(fieldName) + ") % " + mod + ")";
}
}
7 changes: 5 additions & 2 deletions ...e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public abstract class AbstractJdbcIT extends TestSuiteBase implements TestResour

abstract JdbcCase getJdbcCase();

abstract void compareResult(String executeKey) throws SQLException, IOException;
void checkResult(String executeKey, TestContainer container, Container.ExecResult execResult) {}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add more parameter to help check log.
and delete the abstract keyword, the implement class only need implement it when necessary.


abstract String driverUrl();

Expand Down Expand Up @@ -351,7 +351,10 @@ public void testJdbcDb(TestContainer container)
try {
Container.ExecResult execResult = container.executeJob(configFile);
Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
compareResult(String.format("%s in [%s]", configFile, container.identifier()));
checkResult(
String.format("%s in [%s]", configFile, container.identifier()),
container,
execResult);
} finally {
clearTable(jdbcCase.getDatabase(), jdbcCase.getSchema(), jdbcCase.getSinkTable());
}
Expand Down
3 changes: 0 additions & 3 deletions ...bc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDb2IT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/com/ibm/db2/jcc/db2jcc/db2jcc4/db2jcc-db2jcc4.jar";
Expand Down
5 changes: 4 additions & 1 deletion ...-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,14 @@
import org.apache.seatunnel.connectors.seatunnel.jdbc.source.JdbcSourceSplit;
import org.apache.seatunnel.connectors.seatunnel.jdbc.source.JdbcSourceSplitEnumerator;
import org.apache.seatunnel.connectors.seatunnel.jdbc.state.JdbcSourceState;
import org.apache.seatunnel.e2e.common.container.TestContainer;

import org.apache.commons.lang3.tuple.Pair;

import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
Expand Down Expand Up @@ -185,7 +187,8 @@ JdbcCase getJdbcCase() {
}

@Override
protected void compareResult(String executeKey) {
protected void checkResult(
String executeKey, TestContainer container, Container.ExecResult execResult) {
String[] fieldNames =
new String[] {
"c_bit_1",
Expand Down
2 changes: 1 addition & 1 deletion ...e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ JdbcCase getJdbcCase() {
}

@Override
void compareResult(String executeKey) {
void checkResult(String executeKey, TestContainer container, Container.ExecResult execResult) {
defaultCompare(executeKey, fieldNames, "INTEGER_COL");
}

Expand Down
5 changes: 4 additions & 1 deletion ...t-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOceanBaseITBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@

package org.apache.seatunnel.connectors.seatunnel.jdbc;

import org.apache.seatunnel.e2e.common.container.TestContainer;

import org.junit.jupiter.api.Assertions;
import org.testcontainers.containers.Container;
import org.testcontainers.shaded.org.apache.commons.io.IOUtils;

import java.io.InputStream;
Expand Down Expand Up @@ -46,7 +49,7 @@ public abstract class JdbcOceanBaseITBase extends AbstractJdbcIT {
abstract String getFullTableName(String tableName);

@Override
void compareResult(String executeKey) {
void checkResult(String executeKey, TestContainer container, Container.ExecResult execResult) {
String sourceSql =
String.format("select * from %s order by 1", getFullTableName(OCEANBASE_SOURCE));
String sinkSql =
Expand Down
3 changes: 0 additions & 3 deletions ...2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPhoenixIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,6 @@ public void clearTable(String schema, String table) {
}
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/com/aliyun/phoenix/ali-phoenix-shaded-thin-client/5.2.5-HBase-2.x/ali-phoenix-shaded-thin-client-5.2.5-HBase-2.x.jar";
Expand Down
11 changes: 10 additions & 1 deletion ...art-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcStarRocksdbIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@

import org.apache.seatunnel.api.table.catalog.TablePath;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.container.TestContainerId;

import org.apache.commons.lang3.tuple.Pair;

import org.junit.jupiter.api.Assertions;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.utility.DockerLoggerFactory;
Expand Down Expand Up @@ -111,7 +115,12 @@ JdbcCase getJdbcCase() {
}

@Override
void compareResult(String executeKey) {}
void checkResult(String executeKey, TestContainer container, Container.ExecResult execResult) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please revert

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

if (container.identifier().equals(TestContainerId.SEATUNNEL)) {
Assertions.assertTrue(
execResult.getStdout().contains("Loading catalog tables for catalog"));
}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i run some test, find the stdout and container log is different on different engine and version.
so here only check seatunnel engine's log.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when has this log Loading catalog tables for catalog means it was load table from MySQLCatalog. rather than
use jdbc.

related log location:
https://github.com/apache/seatunnel/blob/dev/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcCatalogUtils.java#L77

https://github.com/apache/seatunnel/blob/dev/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/JdbcCatalogUtils.java#L127

}

@Override
String driverUrl() {
Expand Down
3 changes: 0 additions & 3 deletions ...c-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcHiveIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,6 @@ protected void insertTestData() {
}
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/org/apache/hive/hive-jdbc/3.1.3/hive-jdbc-3.1.3-standalone.jar";
Expand Down
5 changes: 0 additions & 5 deletions ...e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcKingbaseIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,7 @@

import com.google.common.collect.Lists;

import java.io.IOException;
import java.math.BigDecimal;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDate;
import java.time.LocalDateTime;
Expand Down Expand Up @@ -121,9 +119,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) throws SQLException, IOException {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/cn/com/kingbase/kingbase8/8.6.0/kingbase8-8.6.0.jar";
Expand Down
5 changes: 0 additions & 5 deletions ...-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcSqlServerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,7 @@

import com.google.common.collect.Lists;

import java.io.IOException;
import java.math.BigDecimal;
import java.sql.SQLException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
Expand Down Expand Up @@ -184,9 +182,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) throws SQLException, IOException {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/9.4.1.jre8/mssql-jdbc-9.4.1.jre8.jar";
Expand Down
3 changes: 0 additions & 3 deletions ...2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcVerticaIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/com/vertica/jdbc/vertica-jdbc/12.0.3-0/vertica-jdbc-12.0.3-0.jar";
Expand Down
3 changes: 0 additions & 3 deletions ...dbc-e2e-part-5/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDmIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/com/dameng/DmJdbcDriver18/8.1.1.193/DmJdbcDriver18-8.1.1.193.jar";
Expand Down
3 changes: 0 additions & 3 deletions ...2e-part-5/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDmUpsetIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
protected void createNeededTables() {
try (Statement statement = connection.createStatement()) {
Expand Down
3 changes: 0 additions & 3 deletions ...2e-part-5/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcGBase8aIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://cdn.gbase.cn/products/30/p5CiVwXBKQYIUGN8ecHvk/gbase-connector-java-9.5.0.7-build1-bin.jar";
Expand Down
3 changes: 0 additions & 3 deletions ...-part-5/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcGreenplumIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.3.3/postgresql-42.3.3.jar";
Expand Down
3 changes: 0 additions & 3 deletions ...c-e2e-part-6/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcHanaIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/com/sap/cloud/db/jdbc/ngdbc/2.21.11/ngdbc-2.21.11.jar";
Expand Down
3 changes: 0 additions & 3 deletions .../test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleLowercaseTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,6 @@ JdbcCase getJdbcCase() {
.build();
}

@Override
void compareResult(String executeKey) {}

@Override
String driverUrl() {
return "https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/12.2.0.1/ojdbc8-12.2.0.1.jar";
Expand Down
2 changes: 1 addition & 1 deletion ...c-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcIrisIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ public String insertTable(String schema, String table, String... fields) {
}

@Override
void compareResult(String executeKey) throws SQLException, IOException {
void checkResult(String executeKey, TestContainer container, Container.ExecResult execResult) {
defaultCompare(executeKey, fieldNames, "BIGINT_COL");
}

Expand Down
4 changes: 3 additions & 1 deletion .../test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlSaveModeHandlerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.utils.JdbcUrlUtil;
import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql.MySqlCatalog;
import org.apache.seatunnel.e2e.common.container.TestContainer;

import org.apache.commons.lang3.tuple.Pair;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
Expand Down Expand Up @@ -157,7 +159,7 @@ JdbcCase getJdbcCase() {
}

@Override
void compareResult(String executeKey) {
void checkResult(String executeKey, TestContainer container, Container.ExecResult execResult) {
final TablePath tablePathSource = TablePath.of("seatunnel", "source");
final CatalogTable tableSource = catalog.getTable(tablePathSource);
final List<Column> columnsSource = tableSource.getTableSchema().getColumns();
Expand Down
4 changes: 3 additions & 1 deletion ...c-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcXuguIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.common.utils.JdbcUrlUtil;
import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.xugu.XuguCatalog;
import org.apache.seatunnel.e2e.common.container.TestContainer;

import org.apache.commons.lang3.tuple.Pair;

import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.utility.DockerLoggerFactory;
Expand Down Expand Up @@ -157,7 +159,7 @@ JdbcCase getJdbcCase() {
}

@Override
void compareResult(String executeKey) {
void checkResult(String executeKey, TestContainer container, Container.ExecResult execResult) {
defaultCompare(executeKey, fieldNames, "XUGU_INT");
}

Expand Down
Loading