Skip to content

Commit

Permalink
[fix][starrocks] add e2e
Browse files Browse the repository at this point in the history
  • Loading branch information
liunaijie committed Mar 28, 2024
1 parent 2bbe781 commit c6c8b07
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,11 @@
<version>${mysql.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>kafka</artifactId>
<version>${testcontainer.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,31 @@ public class StarRocksIT extends TestSuiteBase implements TestResource {
+ "\"storage_format\" = \"DEFAULT\""
+ ")";

private static final String DDL_FAKE_SINK_TABLE =
"create table "
+ DATABASE
+ "."
+ "fake_table_sink"
+ " (\n"
+ " id BIGINT,\n"
+ " c_string STRING,\n"
+ " c_boolean BOOLEAN,\n"
+ " c_tinyint TINYINT,\n"
+ " c_int INT,\n"
+ " c_bigint BIGINT,\n"
+ " c_float FLOAT,\n"
+ " c_double DOUBLE,\n"
+ " c_decimal Decimal(2, 1),\n"
+ " c_date DATE\n"
+ ")ENGINE=OLAP\n"
+ "DUPLICATE KEY(`id`)\n"
+ "DISTRIBUTED BY HASH(`id`) BUCKETS 1\n"
+ "PROPERTIES (\n"
+ "\"replication_num\" = \"1\",\n"
+ "\"in_memory\" = \"false\","
+ "\"storage_format\" = \"DEFAULT\""
+ ")";

private static final String INIT_DATA_SQL =
"insert into "
+ DATABASE
Expand Down Expand Up @@ -253,6 +278,13 @@ public void testStarRocksSink(TestContainer container)
}
}

@TestTemplate
public void testSinkWithCatalogTableNameOnly(TestContainer container)
throws IOException, InterruptedException {
Container.ExecResult execResult = container.executeJob("/fake-to-starrocks.conf");
Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr());
}

private void initializeJdbcConnection()
throws SQLException, ClassNotFoundException, MalformedURLException,
InstantiationException, IllegalAccessException {
Expand All @@ -274,7 +306,7 @@ private void initializeJdbcTable() {
// create source table
statement.execute(DDL_SOURCE);
// create sink table
// statement.execute(DDL_SINK);
statement.execute(DDL_FAKE_SINK_TABLE);
} catch (SQLException e) {
throw new RuntimeException("Initializing table failed!", e);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

env {
parallelism = 1
job.mode = "BATCH"
}

source {
FakeSource {
parallelism = 1
result_table_name = "fake"
row.num = 100
schema {
table = "FakeTable"
columns = [
{
name = id
type = bigint
nullable = false
defaultValue = 0
},
{
name = c_string
type = string
nullable = true
},
{
name = c_boolean
type = boolean
nullable = true
},
{
name = c_tinyint
type = tinyint
nullable = true
},
{
name = c_int
type = int
nullable = true
},
{
name = c_bigint
type = bigint
nullable = true
},
{
name = c_float
type = float
nullable = true
},
{
name = c_double
type = double
nullable = true
},
{
name = c_decimal
type = "decimal(2, 1)"
nullable = true
},
{
name = c_date
type = date
nullable = true
}
]
}
}
}

transform {
}

sink {
StarRocks {
source_table_name = "fake"
nodeUrls = ["starrocks_e2e:8030"]
username = root
password = ""
database = "test"
table = "fake_table_sink"
batch_max_rows = 100
max_retries = 3
base-url="jdbc:mysql://starrocks_e2e:9030/test"
starrocks.config = {
format = "JSON"
strip_outer_array = true
}
}
}

0 comments on commit c6c8b07

Please sign in to comment.