From a692d351a2673a46436636a0956c6e58c78cd06e Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Mon, 24 Oct 2022 09:29:03 +0800 Subject: [PATCH 01/24] add amazondynamodb connnector --- .../connector-amazondynamodb/pom.xml | 61 ++++++++++++ .../config/AmazondynamodbConfig.java | 29 ++++++ .../config/AmazondynamodbSourceOptions.java | 58 +++++++++++ .../sink/AmazondynamodbSink.java | 21 ++++ .../source/AmazondynamodbSource.java | 96 +++++++++++++++++++ .../source/AmazondynamodbSourceReader.java | 77 +++++++++++++++ .../source/AmazondynamodbSourceSplit.java | 35 +++++++ .../state/AmazonDynamodbSourceState.java | 23 +++++ seatunnel-connectors-v2/pom.xml | 1 + seatunnel-dist/pom.xml | 6 ++ 10 files changed, 407 insertions(+) create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/pom.xml create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml new file mode 100644 index 00000000000..0b04ef2ab43 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml @@ -0,0 +1,61 @@ + + + + + seatunnel-connectors-v2 + org.apache.seatunnel + ${revision} + + 4.0.0 + + connector-amazondynamodb + + + 8 + 8 + 2.18.1 + + + + + + software.amazon.awssdk + bom + ${amazon.awssdk} + pom + import + + + + + + + software.amazon.awssdk + dynamodb-enhanced + + + software.amazon.awssdk + dynamodb + + + + diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java new file mode 100644 index 00000000000..05810dc4a81 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config; + +import java.io.Serializable; + +public class AmazondynamodbConfig implements Serializable { + public static final String URL = "url"; + public static final String REGION = "region"; + public static final String ACCESS_KEY_ID = "accessKeyId"; + public static final String SECRET_ACCESS_KEY = "secretAccessKey"; + public static final String QUERY = "query"; + +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java new file mode 100644 index 00000000000..3bce7650b6d --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.io.Serializable; + +@Data +@AllArgsConstructor +public class AmazondynamodbSourceOptions implements Serializable { + + private String url; + + private String region; + + private String accessKeyId; + + private String secretAccessKey; + + private String query; + + public AmazondynamodbSourceOptions(Config config) { + if (config.hasPath(AmazondynamodbConfig.URL)) { + this.url = config.getString(AmazondynamodbConfig.URL); + } + if (config.hasPath(AmazondynamodbConfig.REGION)) { + this.region = config.getString(AmazondynamodbConfig.REGION); + } + if (config.hasPath(AmazondynamodbConfig.ACCESS_KEY_ID)) { + this.accessKeyId = config.getString(AmazondynamodbConfig.ACCESS_KEY_ID); + } + if (config.hasPath(AmazondynamodbConfig.SECRET_ACCESS_KEY)) { + this.secretAccessKey = config.getString(AmazondynamodbConfig.SECRET_ACCESS_KEY); + } + if (config.hasPath(AmazondynamodbConfig.QUERY)) { + this.query = config.getString(AmazondynamodbConfig.QUERY); + } + } +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java new file mode 100644 index 00000000000..44e0cb7c559 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; + +public class AmazondynamodbSink { +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java new file mode 100644 index 00000000000..0c9c22662e4 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; + +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state.AmazonDynamodbSourceState; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.protocol.MarshallingType; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbResponseMetadata; +import software.amazon.awssdk.services.dynamodb.model.ExecuteStatementRequest; +import software.amazon.awssdk.services.dynamodb.model.ExecuteStatementResponse; + +import java.net.URI; + +@Slf4j +@AutoService(SeaTunnelSource.class) +public class AmazondynamodbSource implements SeaTunnelSource { + + private DynamoDbClient dynamoDbClient; + + private AmazondynamodbSourceOptions amazondynamodbSourceOptions; + + @Override + public String getPluginName() { + return "Amazondynamodb"; + } + + @Override + public void prepare(Config pluginConfig) throws PrepareFailException { + amazondynamodbSourceOptions = new AmazondynamodbSourceOptions(pluginConfig); + dynamoDbClient = DynamoDbClient.builder() + .endpointOverride(URI.create(amazondynamodbSourceOptions.getUrl())) + // The region is meaningless for local DynamoDb but required for client builder validation + .region(Region.of(amazondynamodbSourceOptions.getRegion())) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(amazondynamodbSourceOptions.getAccessKeyId(), amazondynamodbSourceOptions.getSecretAccessKey()))) + .build(); + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } + + @Override + public SeaTunnelDataType getProducedType() { + ExecuteStatementResponse executeStatementResponse = dynamoDbClient.executeStatement(ExecuteStatementRequest.builder().statement(amazondynamodbSourceOptions.getQuery()).build()); + + return null; + } + + @Override + public SourceReader createReader(SourceReader.Context readerContext) { + return new AmazondynamodbSourceReader(dynamoDbClient, readerContext, amazondynamodbSourceOptions); + } + + @Override + public SourceSplitEnumerator createEnumerator(SourceSplitEnumerator.Context enumeratorContext) throws Exception { + return null; + } + + @Override + public SourceSplitEnumerator restoreEnumerator(SourceSplitEnumerator.Context enumeratorContext, AmazonDynamodbSourceState checkpointState) throws Exception { + return null; + } +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java new file mode 100644 index 00000000000..7803c83fc98 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; + +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.ExecuteStatementRequest; + +import java.io.IOException; +import java.util.List; + +public class AmazondynamodbSourceReader implements SourceReader { + + DynamoDbClient dynamoDbClient; + SourceReader.Context context; + ExecuteStatementRequest executeStatement; + AmazondynamodbSourceOptions amazondynamodbSourceOptions; + + public AmazondynamodbSourceReader(DynamoDbClient dynamoDbClient, SourceReader.Context context, AmazondynamodbSourceOptions amazondynamodbSourceOptions) { + this.dynamoDbClient = dynamoDbClient; + this.context = context; + this.amazondynamodbSourceOptions = amazondynamodbSourceOptions; + } + + @Override + public void open() throws Exception { + executeStatement = ExecuteStatementRequest.builder().statement(amazondynamodbSourceOptions.getQuery()).build(); + } + + @Override + public void close() throws IOException { + dynamoDbClient.close(); + } + + @Override + public void pollNext(Collector output) throws Exception { + } + + @Override + public List snapshotState(long checkpointId) throws Exception { + return null; + } + + @Override + public void addSplits(List splits) { + + } + + @Override + public void handleNoMoreSplits() { + + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + + } +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java new file mode 100644 index 00000000000..f2a871fb06e --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; + +import org.apache.seatunnel.api.source.SourceSplit; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class AmazondynamodbSourceSplit implements SourceSplit { + + Integer splitId; + + @Override + public String splitId() { + return splitId.toString(); + } +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java new file mode 100644 index 00000000000..9edb9dae7d4 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state; + +import java.io.Serializable; + +public class AmazonDynamodbSourceState implements Serializable { +} diff --git a/seatunnel-connectors-v2/pom.xml b/seatunnel-connectors-v2/pom.xml index 37dd5223c7e..cb91f71faf5 100644 --- a/seatunnel-connectors-v2/pom.xml +++ b/seatunnel-connectors-v2/pom.xml @@ -56,6 +56,7 @@ connector-mongodb connector-iceberg connector-influxdb + connector-amazondynamodb diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 4c602b04ee1..f7efdc6a888 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -290,6 +290,12 @@ ${project.version} provided + + org.apache.seatunnel + connector-amazondynamodb + ${project.version} + provided + From 08908af6d5923eff555505f3e619113224169757 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Thu, 27 Oct 2022 20:53:33 +0800 Subject: [PATCH 02/24] add amazondynamodb source --- .../connector-amazondynamodb/pom.xml | 5 ++ .../config/AmazondynamodbConfig.java | 1 - .../config/AmazondynamodbSourceOptions.java | 7 ++ .../source/AmazondynamodbSource.java | 32 ++----- .../source/AmazondynamodbSourceReader.java | 88 ++++++++++++++++--- .../AmazondynamodbSourceSplitEnumerator.java | 71 +++++++++++++++ .../seatunnel/common/config/CommonConfig.java | 22 +++++ 7 files changed, 192 insertions(+), 34 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/config/CommonConfig.java diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml index 0b04ef2ab43..5215b2e0d4b 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml +++ b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml @@ -48,6 +48,11 @@ + + org.apache.seatunnel + connector-common + ${project.version} + software.amazon.awssdk dynamodb-enhanced diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java index 05810dc4a81..4e8ce67f2cc 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java @@ -25,5 +25,4 @@ public class AmazondynamodbConfig implements Serializable { public static final String ACCESS_KEY_ID = "accessKeyId"; public static final String SECRET_ACCESS_KEY = "secretAccessKey"; public static final String QUERY = "query"; - } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java index 3bce7650b6d..ad4956e82a4 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config; +import org.apache.seatunnel.connectors.seatunnel.common.config.CommonConfig; + import org.apache.seatunnel.shade.com.typesafe.config.Config; import lombok.AllArgsConstructor; @@ -38,6 +40,8 @@ public class AmazondynamodbSourceOptions implements Serializable { private String query; + private Config schema; + public AmazondynamodbSourceOptions(Config config) { if (config.hasPath(AmazondynamodbConfig.URL)) { this.url = config.getString(AmazondynamodbConfig.URL); @@ -54,5 +58,8 @@ public AmazondynamodbSourceOptions(Config config) { if (config.hasPath(AmazondynamodbConfig.QUERY)) { this.query = config.getString(AmazondynamodbConfig.QUERY); } + if (config.hasPath(CommonConfig.SCHEMA)) { + this.schema = config.getConfig(CommonConfig.SCHEMA); + } } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java index 0c9c22662e4..1db1e9b0b31 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java @@ -24,32 +24,24 @@ import org.apache.seatunnel.api.source.SourceSplitEnumerator; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state.AmazonDynamodbSourceState; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; import org.apache.seatunnel.shade.com.typesafe.config.Config; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.protocol.MarshallingType; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.DynamoDbResponseMetadata; -import software.amazon.awssdk.services.dynamodb.model.ExecuteStatementRequest; -import software.amazon.awssdk.services.dynamodb.model.ExecuteStatementResponse; - -import java.net.URI; @Slf4j @AutoService(SeaTunnelSource.class) public class AmazondynamodbSource implements SeaTunnelSource { - private DynamoDbClient dynamoDbClient; - private AmazondynamodbSourceOptions amazondynamodbSourceOptions; + private SeaTunnelRowType typeInfo; + @Override public String getPluginName() { return "Amazondynamodb"; @@ -58,13 +50,7 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { amazondynamodbSourceOptions = new AmazondynamodbSourceOptions(pluginConfig); - dynamoDbClient = DynamoDbClient.builder() - .endpointOverride(URI.create(amazondynamodbSourceOptions.getUrl())) - // The region is meaningless for local DynamoDb but required for client builder validation - .region(Region.of(amazondynamodbSourceOptions.getRegion())) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create(amazondynamodbSourceOptions.getAccessKeyId(), amazondynamodbSourceOptions.getSecretAccessKey()))) - .build(); + } @Override @@ -74,14 +60,14 @@ public Boundedness getBoundedness() { @Override public SeaTunnelDataType getProducedType() { - ExecuteStatementResponse executeStatementResponse = dynamoDbClient.executeStatement(ExecuteStatementRequest.builder().statement(amazondynamodbSourceOptions.getQuery()).build()); - - return null; + Config schema = amazondynamodbSourceOptions.getSchema(); + typeInfo = SeaTunnelSchema.buildWithConfig(schema).getSeaTunnelRowType(); + return this.typeInfo; } @Override public SourceReader createReader(SourceReader.Context readerContext) { - return new AmazondynamodbSourceReader(dynamoDbClient, readerContext, amazondynamodbSourceOptions); + return new AmazondynamodbSourceReader(readerContext, amazondynamodbSourceOptions); } @Override diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index 7803c83fc98..a76780ebd77 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -19,31 +19,57 @@ import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import lombok.extern.slf4j.Slf4j; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.ExecuteStatementRequest; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.QueryRequest; +import software.amazon.awssdk.services.dynamodb.model.QueryResponse; import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Deque; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +@Slf4j public class AmazondynamodbSourceReader implements SourceReader { - DynamoDbClient dynamoDbClient; - SourceReader.Context context; - ExecuteStatementRequest executeStatement; - AmazondynamodbSourceOptions amazondynamodbSourceOptions; + protected DynamoDbClient dynamoDbClient; + protected SourceReader.Context context; + protected AmazondynamodbSourceOptions amazondynamodbSourceOptions; + protected Deque splits = new LinkedList<>(); + protected boolean noMoreSplit; + protected SeaTunnelRowType typeInfo; - public AmazondynamodbSourceReader(DynamoDbClient dynamoDbClient, SourceReader.Context context, AmazondynamodbSourceOptions amazondynamodbSourceOptions) { - this.dynamoDbClient = dynamoDbClient; + public AmazondynamodbSourceReader(SourceReader.Context context, + AmazondynamodbSourceOptions amazondynamodbSourceOptions, + SeaTunnelRowType typeInfo) { this.context = context; this.amazondynamodbSourceOptions = amazondynamodbSourceOptions; + this.typeInfo = typeInfo; } @Override public void open() throws Exception { - executeStatement = ExecuteStatementRequest.builder().statement(amazondynamodbSourceOptions.getQuery()).build(); + dynamoDbClient = DynamoDbClient.builder() + .endpointOverride(URI.create(amazondynamodbSourceOptions.getUrl())) + // The region is meaningless for local DynamoDb but required for client builder validation + .region(Region.of(amazondynamodbSourceOptions.getRegion())) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(amazondynamodbSourceOptions.getAccessKeyId(), amazondynamodbSourceOptions.getSecretAccessKey()))) + .build(); } @Override @@ -52,7 +78,25 @@ public void close() throws IOException { } @Override + @SuppressWarnings("magicnumber") public void pollNext(Collector output) throws Exception { + synchronized (output.getCheckpointLock()) { + AmazondynamodbSourceSplit split = splits.poll(); + if (null != split) { + QueryResponse query = dynamoDbClient.query(QueryRequest.builder().build()); + if (query.hasItems()) { + query.items().forEach(item -> { + output.collect(converterToRow(item, typeInfo)); + }); + } + } else if (noMoreSplit) { + // signal to the source that we have reached the end of the data. + log.info("Closed the bounded amazondynamodb source"); + context.signalNoMoreElement(); + } else { + Thread.sleep(1000L); + } + } } @Override @@ -62,16 +106,40 @@ public List snapshotState(long checkpointId) throws E @Override public void addSplits(List splits) { - + this.splits.addAll(splits); } @Override public void handleNoMoreSplits() { - + noMoreSplit = true; } @Override public void notifyCheckpointComplete(long checkpointId) throws Exception { } + + private SeaTunnelRow converterToRow(Map item, SeaTunnelRowType typeInfo) { + List fields = new ArrayList<>(); + SeaTunnelDataType[] seaTunnelDataTypes = typeInfo.getFieldTypes(); + String[] fieldNames = typeInfo.getFieldNames(); + for (int i = 1; i <= seaTunnelDataTypes.length; i++) { + Object seatunnelField; + SeaTunnelDataType seaTunnelDataType = seaTunnelDataTypes[i - 1]; + AttributeValue attributeValue = item.get(fieldNames[i]); + if (attributeValue.nul()) { + seatunnelField = null; + } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) { + seatunnelField = attributeValue.bool(); + } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) { + seatunnelField = attributeValue.s().getBytes(StandardCharsets.UTF_8); + } else { + throw new IllegalStateException("Unexpected value: " + seaTunnelDataType); + } + + fields.add(seatunnelField); + } + + return new SeaTunnelRow(fields.toArray()); + } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java new file mode 100644 index 00000000000..9159d9093c2 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; + +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state.AmazonDynamodbSourceState; + +import java.io.IOException; +import java.util.List; + +public class AmazondynamodbSourceSplitEnumerator implements SourceSplitEnumerator { + @Override + public void open() { + // No connection needs to be opened + } + + @Override + public void run() throws Exception { + + } + + @Override + public void close() throws IOException { + // nothing + } + + @Override + public void addSplitsBack(List splits, int subtaskId) { + + } + + @Override + public int currentUnassignedSplitSize() { + return 0; + } + + @Override + public void handleSplitRequest(int subtaskId) { + + } + + @Override + public void registerReader(int subtaskId) { + // nothing + } + + @Override + public AmazonDynamodbSourceState snapshotState(long checkpointId) throws Exception { + return null; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/config/CommonConfig.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/config/CommonConfig.java new file mode 100644 index 00000000000..bb695ea88da --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/config/CommonConfig.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.config; + +public class CommonConfig { + public static final String SCHEMA = "schema"; +} From d3fa1b93ca6e16e3db6eb08caabfd526a62c33f4 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Fri, 28 Oct 2022 16:16:04 +0800 Subject: [PATCH 03/24] add amazondynamodbSource --- .../connector-amazondynamodb/pom.xml | 2 - .../config/AmazondynamodbSourceOptions.java | 4 +- .../source/AmazondynamodbSource.java | 7 +- .../source/AmazondynamodbSourceReader.java | 105 ++++++++++++++---- .../AmazondynamodbSourceSplitEnumerator.java | 44 +++++++- 5 files changed, 132 insertions(+), 30 deletions(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml index 5215b2e0d4b..f540386b16a 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml +++ b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml @@ -30,8 +30,6 @@ connector-amazondynamodb - 8 - 8 2.18.1 diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java index ad4956e82a4..58227d5f287 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -38,7 +38,7 @@ public class AmazondynamodbSourceOptions implements Serializable { private String secretAccessKey; - private String query; + private String table; private Config schema; @@ -56,7 +56,7 @@ public AmazondynamodbSourceOptions(Config config) { this.secretAccessKey = config.getString(AmazondynamodbConfig.SECRET_ACCESS_KEY); } if (config.hasPath(AmazondynamodbConfig.QUERY)) { - this.query = config.getString(AmazondynamodbConfig.QUERY); + this.table = config.getString(AmazondynamodbConfig.QUERY); } if (config.hasPath(CommonConfig.SCHEMA)) { this.schema = config.getConfig(CommonConfig.SCHEMA); diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java index 1db1e9b0b31..24a093a7b62 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java @@ -50,7 +50,6 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { amazondynamodbSourceOptions = new AmazondynamodbSourceOptions(pluginConfig); - } @Override @@ -67,16 +66,16 @@ public SeaTunnelDataType getProducedType() { @Override public SourceReader createReader(SourceReader.Context readerContext) { - return new AmazondynamodbSourceReader(readerContext, amazondynamodbSourceOptions); + return new AmazondynamodbSourceReader(readerContext, amazondynamodbSourceOptions, typeInfo); } @Override public SourceSplitEnumerator createEnumerator(SourceSplitEnumerator.Context enumeratorContext) throws Exception { - return null; + return new AmazondynamodbSourceSplitEnumerator(enumeratorContext); } @Override public SourceSplitEnumerator restoreEnumerator(SourceSplitEnumerator.Context enumeratorContext, AmazonDynamodbSourceState checkpointState) throws Exception { - return null; + return new AmazondynamodbSourceSplitEnumerator(enumeratorContext); } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index a76780ebd77..c7bc8636d5b 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -19,7 +19,12 @@ import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; @@ -31,14 +36,19 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.QueryRequest; -import software.amazon.awssdk.services.dynamodb.model.QueryResponse; +import software.amazon.awssdk.services.dynamodb.model.BatchGetItemRequest; +import software.amazon.awssdk.services.dynamodb.model.BatchGetItemResponse; import java.io.IOException; +import java.math.BigDecimal; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; import java.util.ArrayList; import java.util.Deque; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -83,10 +93,13 @@ public void pollNext(Collector output) throws Exception { synchronized (output.getCheckpointLock()) { AmazondynamodbSourceSplit split = splits.poll(); if (null != split) { - QueryResponse query = dynamoDbClient.query(QueryRequest.builder().build()); - if (query.hasItems()) { - query.items().forEach(item -> { - output.collect(converterToRow(item, typeInfo)); + BatchGetItemResponse batchGetItemResponse = dynamoDbClient + .batchGetItem(BatchGetItemRequest.builder().build()); + if (batchGetItemResponse.hasResponses()) { + batchGetItemResponse.responses().forEach((s, maps) -> { + maps.forEach(item -> { + output.collect(converterToRow(item, typeInfo)); + }); }); } } else if (noMoreSplit) { @@ -101,7 +114,7 @@ public void pollNext(Collector output) throws Exception { @Override public List snapshotState(long checkpointId) throws Exception { - return null; + return new ArrayList<>(splits); } @Override @@ -120,26 +133,76 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } private SeaTunnelRow converterToRow(Map item, SeaTunnelRowType typeInfo) { - List fields = new ArrayList<>(); SeaTunnelDataType[] seaTunnelDataTypes = typeInfo.getFieldTypes(); + return new SeaTunnelRow(convertRow(seaTunnelDataTypes, item).toArray()); + } + + private List convertRow(SeaTunnelDataType[] seaTunnelDataTypes, Map item) { + List fields = new ArrayList<>(); String[] fieldNames = typeInfo.getFieldNames(); for (int i = 1; i <= seaTunnelDataTypes.length; i++) { - Object seatunnelField; SeaTunnelDataType seaTunnelDataType = seaTunnelDataTypes[i - 1]; AttributeValue attributeValue = item.get(fieldNames[i]); - if (attributeValue.nul()) { - seatunnelField = null; - } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) { - seatunnelField = attributeValue.bool(); - } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) { - seatunnelField = attributeValue.s().getBytes(StandardCharsets.UTF_8); - } else { - throw new IllegalStateException("Unexpected value: " + seaTunnelDataType); - } - - fields.add(seatunnelField); + fields.add(convert(seaTunnelDataType, attributeValue)); } + return fields; + } - return new SeaTunnelRow(fields.toArray()); + private Object convert(SeaTunnelDataType seaTunnelDataType, AttributeValue attributeValue) { + if (attributeValue.nul()) { + return null; + } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) { + return attributeValue.bool(); + } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) { + return attributeValue.s().getBytes(StandardCharsets.UTF_8)[0]; + } else if (BasicType.SHORT_TYPE.equals(seaTunnelDataType)) { + return Short.parseShort(attributeValue.n()); + } else if (BasicType.INT_TYPE.equals(seaTunnelDataType)) { + return Integer.parseInt(attributeValue.n()); + } else if (BasicType.LONG_TYPE.equals(seaTunnelDataType)) { + return Long.parseLong(attributeValue.n()); + } else if (seaTunnelDataType instanceof DecimalType) { + return new BigDecimal(attributeValue.n()); + } else if (BasicType.FLOAT_TYPE.equals(seaTunnelDataType)) { + return Float.parseFloat(attributeValue.n()); + } else if (BasicType.DOUBLE_TYPE.equals(seaTunnelDataType)) { + return Double.parseDouble(attributeValue.n()); + } else if (BasicType.STRING_TYPE.equals(seaTunnelDataType)) { + return attributeValue.s(); + } else if (LocalTimeType.LOCAL_TIME_TYPE.equals(seaTunnelDataType)) { + return LocalTime.parse(attributeValue.s()); + } else if (LocalTimeType.LOCAL_DATE_TYPE.equals(seaTunnelDataType)) { + return LocalDate.parse(attributeValue.s()); + } else if (LocalTimeType.LOCAL_DATE_TIME_TYPE.equals(seaTunnelDataType)) { + return LocalDateTime.parse(attributeValue.s()); + } else if (PrimitiveByteArrayType.INSTANCE.equals(seaTunnelDataType)) { + return attributeValue.b().asByteArray(); + } else if (seaTunnelDataType instanceof MapType) { + Map seatunnelMap = new HashMap<>(); + attributeValue.m().forEach((s, attributeValueInfo) -> { + seatunnelMap.put(s, convert(((MapType) seaTunnelDataType).getValueType(), attributeValueInfo)); + }); + return seatunnelMap; + } else if (seaTunnelDataType instanceof ArrayType) { + List seatunnelList = new ArrayList<>(attributeValue.l().size()); + if (attributeValue.hasL()) { + attributeValue.l().forEach(l -> { + seatunnelList.add(convert(((ArrayType) seaTunnelDataType).getElementType(), l)); + }); + } else if (attributeValue.hasSs()) { + seatunnelList.addAll(attributeValue.ss()); + } else if (attributeValue.hasNs()) { + attributeValue.ns().forEach(s -> { + seatunnelList.addAll(attributeValue.ns()); + }); + } else if (attributeValue.hasBs()) { + attributeValue.bs().forEach(s -> { + seatunnelList.add(s.asByteArray()); + }); + } + return seatunnelList; + } else { + throw new IllegalStateException("Unexpected value: " + seaTunnelDataType); + } } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java index 9159d9093c2..4c2caf744bc 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java @@ -20,10 +20,26 @@ import org.apache.seatunnel.api.source.SourceSplitEnumerator; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state.AmazonDynamodbSourceState; +import lombok.extern.slf4j.Slf4j; + import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +@Slf4j public class AmazondynamodbSourceSplitEnumerator implements SourceSplitEnumerator { + + private final SourceSplitEnumerator.Context enumeratorContext; + private final Map> pendingSplits; + + public AmazondynamodbSourceSplitEnumerator(SourceSplitEnumerator.Context enumeratorContext) { + this.enumeratorContext = enumeratorContext; + this.pendingSplits = new HashMap<>(); + } + @Override public void open() { // No connection needs to be opened @@ -31,7 +47,8 @@ public void open() { @Override public void run() throws Exception { - + discoverySplits(); + assignPendingSplits(); } @Override @@ -68,4 +85,29 @@ public AmazonDynamodbSourceState snapshotState(long checkpointId) throws Excepti public void notifyCheckpointComplete(long checkpointId) throws Exception { } + + private void discoverySplits() { + List allSplit = new ArrayList<>(); + log.info("Starting to calculate splits."); + allSplit.add(new AmazondynamodbSourceSplit(0)); + int numReaders = enumeratorContext.currentParallelism(); + log.debug("Assigned {} to {} readers.", allSplit, numReaders); + log.info("Calculated splits successfully, the size of splits is {}.", allSplit.size()); + } + + private void assignPendingSplits() { + // Check if there's any pending splits for given readers + for (int pendingReader : enumeratorContext.registeredReaders()) { + // Remove pending assignment for the reader + final Set pendingAssignmentForReader = + pendingSplits.remove(pendingReader); + + if (pendingAssignmentForReader != null && !pendingAssignmentForReader.isEmpty()) { + // Assign pending splits to reader + log.info("Assigning splits to readers {}", pendingAssignmentForReader); + enumeratorContext.assignSplit(pendingReader, new ArrayList<>(pendingAssignmentForReader)); + enumeratorContext.signalNoMoreSplits(pendingReader); + } + } + } } From 7ed060bf9ca615be46b4c54e34dbef34f697a264 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Fri, 28 Oct 2022 23:11:46 +0800 Subject: [PATCH 04/24] add Amazondynamodb Source --- .../source/AmazondynamodbSource.java | 20 +--- .../source/AmazondynamodbSourceReader.java | 64 +++------- .../source/AmazondynamodbSourceSplit.java | 35 ------ .../AmazondynamodbSourceSplitEnumerator.java | 113 ------------------ .../state/AmazonDynamodbSourceState.java | 23 ---- 5 files changed, 20 insertions(+), 235 deletions(-) delete mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java delete mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java delete mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java index 24a093a7b62..bef92154107 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java @@ -20,14 +20,14 @@ import org.apache.seatunnel.api.common.PrepareFailException; import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.source.SourceReader; -import org.apache.seatunnel.api.source.SourceSplitEnumerator; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; -import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state.AmazonDynamodbSourceState; import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; +import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitSource; +import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; import org.apache.seatunnel.shade.com.typesafe.config.Config; @@ -36,7 +36,7 @@ @Slf4j @AutoService(SeaTunnelSource.class) -public class AmazondynamodbSource implements SeaTunnelSource { +public class AmazondynamodbSource extends AbstractSingleSplitSource { private AmazondynamodbSourceOptions amazondynamodbSourceOptions; @@ -65,17 +65,7 @@ public SeaTunnelDataType getProducedType() { } @Override - public SourceReader createReader(SourceReader.Context readerContext) { + public AbstractSingleSplitReader createReader(SingleSplitReaderContext readerContext) throws Exception { return new AmazondynamodbSourceReader(readerContext, amazondynamodbSourceOptions, typeInfo); } - - @Override - public SourceSplitEnumerator createEnumerator(SourceSplitEnumerator.Context enumeratorContext) throws Exception { - return new AmazondynamodbSourceSplitEnumerator(enumeratorContext); - } - - @Override - public SourceSplitEnumerator restoreEnumerator(SourceSplitEnumerator.Context enumeratorContext, AmazonDynamodbSourceState checkpointState) throws Exception { - return new AmazondynamodbSourceSplitEnumerator(enumeratorContext); - } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index c7bc8636d5b..c73c7c1b037 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -18,7 +18,6 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; import org.apache.seatunnel.api.source.Collector; -import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; @@ -29,6 +28,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; import lombok.extern.slf4j.Slf4j; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; @@ -36,8 +37,8 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.BatchGetItemRequest; -import software.amazon.awssdk.services.dynamodb.model.BatchGetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.ScanRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanResponse; import java.io.IOException; import java.math.BigDecimal; @@ -47,23 +48,19 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.util.ArrayList; -import java.util.Deque; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; @Slf4j -public class AmazondynamodbSourceReader implements SourceReader { +public class AmazondynamodbSourceReader extends AbstractSingleSplitReader { protected DynamoDbClient dynamoDbClient; - protected SourceReader.Context context; + protected SingleSplitReaderContext context; protected AmazondynamodbSourceOptions amazondynamodbSourceOptions; - protected Deque splits = new LinkedList<>(); - protected boolean noMoreSplit; protected SeaTunnelRowType typeInfo; - public AmazondynamodbSourceReader(SourceReader.Context context, + public AmazondynamodbSourceReader(SingleSplitReaderContext context, AmazondynamodbSourceOptions amazondynamodbSourceOptions, SeaTunnelRowType typeInfo) { this.context = context; @@ -90,46 +87,15 @@ public void close() throws IOException { @Override @SuppressWarnings("magicnumber") public void pollNext(Collector output) throws Exception { - synchronized (output.getCheckpointLock()) { - AmazondynamodbSourceSplit split = splits.poll(); - if (null != split) { - BatchGetItemResponse batchGetItemResponse = dynamoDbClient - .batchGetItem(BatchGetItemRequest.builder().build()); - if (batchGetItemResponse.hasResponses()) { - batchGetItemResponse.responses().forEach((s, maps) -> { - maps.forEach(item -> { - output.collect(converterToRow(item, typeInfo)); - }); - }); - } - } else if (noMoreSplit) { - // signal to the source that we have reached the end of the data. - log.info("Closed the bounded amazondynamodb source"); - context.signalNoMoreElement(); - } else { - Thread.sleep(1000L); - } + ScanResponse scan = dynamoDbClient.scan(ScanRequest.builder() + .tableName(amazondynamodbSourceOptions.getTable()) + .build()); + if (scan.hasItems()) { + scan.items().forEach(item -> { + output.collect(converterToRow(item, typeInfo)); + }); } - } - - @Override - public List snapshotState(long checkpointId) throws Exception { - return new ArrayList<>(splits); - } - - @Override - public void addSplits(List splits) { - this.splits.addAll(splits); - } - - @Override - public void handleNoMoreSplits() { - noMoreSplit = true; - } - - @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception { - + context.signalNoMoreElement(); } private SeaTunnelRow converterToRow(Map item, SeaTunnelRowType typeInfo) { diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java deleted file mode 100644 index f2a871fb06e..00000000000 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplit.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; - -import org.apache.seatunnel.api.source.SourceSplit; - -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class AmazondynamodbSourceSplit implements SourceSplit { - - Integer splitId; - - @Override - public String splitId() { - return splitId.toString(); - } -} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java deleted file mode 100644 index 4c2caf744bc..00000000000 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceSplitEnumerator.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; - -import org.apache.seatunnel.api.source.SourceSplitEnumerator; -import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state.AmazonDynamodbSourceState; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Slf4j -public class AmazondynamodbSourceSplitEnumerator implements SourceSplitEnumerator { - - private final SourceSplitEnumerator.Context enumeratorContext; - private final Map> pendingSplits; - - public AmazondynamodbSourceSplitEnumerator(SourceSplitEnumerator.Context enumeratorContext) { - this.enumeratorContext = enumeratorContext; - this.pendingSplits = new HashMap<>(); - } - - @Override - public void open() { - // No connection needs to be opened - } - - @Override - public void run() throws Exception { - discoverySplits(); - assignPendingSplits(); - } - - @Override - public void close() throws IOException { - // nothing - } - - @Override - public void addSplitsBack(List splits, int subtaskId) { - - } - - @Override - public int currentUnassignedSplitSize() { - return 0; - } - - @Override - public void handleSplitRequest(int subtaskId) { - - } - - @Override - public void registerReader(int subtaskId) { - // nothing - } - - @Override - public AmazonDynamodbSourceState snapshotState(long checkpointId) throws Exception { - return null; - } - - @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception { - - } - - private void discoverySplits() { - List allSplit = new ArrayList<>(); - log.info("Starting to calculate splits."); - allSplit.add(new AmazondynamodbSourceSplit(0)); - int numReaders = enumeratorContext.currentParallelism(); - log.debug("Assigned {} to {} readers.", allSplit, numReaders); - log.info("Calculated splits successfully, the size of splits is {}.", allSplit.size()); - } - - private void assignPendingSplits() { - // Check if there's any pending splits for given readers - for (int pendingReader : enumeratorContext.registeredReaders()) { - // Remove pending assignment for the reader - final Set pendingAssignmentForReader = - pendingSplits.remove(pendingReader); - - if (pendingAssignmentForReader != null && !pendingAssignmentForReader.isEmpty()) { - // Assign pending splits to reader - log.info("Assigning splits to readers {}", pendingAssignmentForReader); - enumeratorContext.assignSplit(pendingReader, new ArrayList<>(pendingAssignmentForReader)); - enumeratorContext.signalNoMoreSplits(pendingReader); - } - } - } -} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java deleted file mode 100644 index 9edb9dae7d4..00000000000 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/state/AmazonDynamodbSourceState.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.state; - -import java.io.Serializable; - -public class AmazonDynamodbSourceState implements Serializable { -} From 5b0ab48a13b54e877dfcbe86fc9687e4be190f09 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sat, 29 Oct 2022 12:06:15 +0800 Subject: [PATCH 05/24] fix Amazondynamodb Source bug --- plugin-mapping.properties | 4 +++- .../amazondynamodb/config/AmazondynamodbConfig.java | 2 +- .../amazondynamodb/config/AmazondynamodbSourceOptions.java | 4 ++-- .../amazondynamodb/source/AmazondynamodbSourceReader.java | 6 +++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/plugin-mapping.properties b/plugin-mapping.properties index e022fa59b1b..b976e6e3fd4 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -135,4 +135,6 @@ seatunnel.sink.MongoDB = connector-mongodb seatunnel.source.Iceberg = connector-iceberg seatunnel.source.InfluxDB = connector-influxdb seatunnel.source.S3File = connector-file-s3 -seatunnel.sink.S3File = connector-file-s3 \ No newline at end of file +seatunnel.sink.S3File = connector-file-s3 +seatunnel.source.Amazondynamodb = connector-amazondynamodb +seatunnel.sink.Amazondynamodb = connector-amazondynamodb diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java index 4e8ce67f2cc..d615ee3047b 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java @@ -24,5 +24,5 @@ public class AmazondynamodbConfig implements Serializable { public static final String REGION = "region"; public static final String ACCESS_KEY_ID = "accessKeyId"; public static final String SECRET_ACCESS_KEY = "secretAccessKey"; - public static final String QUERY = "query"; + public static final String TABLE = "table"; } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java index 58227d5f287..9929d84d06b 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -55,8 +55,8 @@ public AmazondynamodbSourceOptions(Config config) { if (config.hasPath(AmazondynamodbConfig.SECRET_ACCESS_KEY)) { this.secretAccessKey = config.getString(AmazondynamodbConfig.SECRET_ACCESS_KEY); } - if (config.hasPath(AmazondynamodbConfig.QUERY)) { - this.table = config.getString(AmazondynamodbConfig.QUERY); + if (config.hasPath(AmazondynamodbConfig.TABLE)) { + this.table = config.getString(AmazondynamodbConfig.TABLE); } if (config.hasPath(CommonConfig.SCHEMA)) { this.schema = config.getConfig(CommonConfig.SCHEMA); diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index c73c7c1b037..f3656b3ccd6 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -106,8 +106,8 @@ private SeaTunnelRow converterToRow(Map item, SeaTunnelR private List convertRow(SeaTunnelDataType[] seaTunnelDataTypes, Map item) { List fields = new ArrayList<>(); String[] fieldNames = typeInfo.getFieldNames(); - for (int i = 1; i <= seaTunnelDataTypes.length; i++) { - SeaTunnelDataType seaTunnelDataType = seaTunnelDataTypes[i - 1]; + for (int i = 0; i < seaTunnelDataTypes.length; i++) { + SeaTunnelDataType seaTunnelDataType = seaTunnelDataTypes[i]; AttributeValue attributeValue = item.get(fieldNames[i]); fields.add(convert(seaTunnelDataType, attributeValue)); } @@ -115,7 +115,7 @@ private List convertRow(SeaTunnelDataType[] seaTunnelDataTypes, Map seaTunnelDataType, AttributeValue attributeValue) { - if (attributeValue.nul()) { + if (attributeValue.type().equals(AttributeValue.Type.NUL)) { return null; } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) { return attributeValue.bool(); From 2a6d350dbe7d39a0a6723d43b4b5f0c324ecb04d Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sat, 29 Oct 2022 21:08:29 +0800 Subject: [PATCH 06/24] add Amazondynamodb sink connector --- .../sink/AmazondynamodbSink.java | 59 +++++- .../sink/AmazondynamodbWriter.java | 169 ++++++++++++++++++ 2 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java index 44e0cb7c559..1dd43665344 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java @@ -17,5 +17,62 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; -public class AmazondynamodbSink { +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.REGION; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.TABLE; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.URL; + +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.sink.SeaTunnelSink; +import org.apache.seatunnel.api.sink.SinkWriter; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink; +import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import com.google.auto.service.AutoService; + +import java.io.IOException; + +@AutoService(SeaTunnelSink.class) +public class AmazondynamodbSink extends AbstractSimpleSink { + + private SeaTunnelRowType rowType; + + private AmazondynamodbSourceOptions amazondynamodbSourceOptions; + + @Override + public String getPluginName() { + return "Amazondynamodb"; + } + + @Override + public void prepare(Config pluginConfig) throws PrepareFailException { + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, URL, TABLE, REGION); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); + } + amazondynamodbSourceOptions = new AmazondynamodbSourceOptions(pluginConfig); + } + + @Override + public void setTypeInfo(SeaTunnelRowType seaTunnelRowType) { + this.rowType = seaTunnelRowType; + } + + @Override + public SeaTunnelDataType getConsumedType() { + return rowType; + } + + @Override + public AbstractSinkWriter createWriter(SinkWriter.Context context) throws IOException { + return new AmazondynamodbWriter(amazondynamodbSourceOptions, rowType); + } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java new file mode 100644 index 00000000000..db9526223e9 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; + +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class AmazondynamodbWriter extends AbstractSinkWriter { + + private final AmazondynamodbSourceOptions amazondynamodbSourceOptions; + private final SeaTunnelRowType seaTunnelRowType; + private final DynamoDbClient dynamoDbClient; + private final List measurementsType; + + public AmazondynamodbWriter(AmazondynamodbSourceOptions amazondynamodbSourceOptions, SeaTunnelRowType seaTunnelRowType) { + this.amazondynamodbSourceOptions = amazondynamodbSourceOptions; + this.seaTunnelRowType = seaTunnelRowType; + dynamoDbClient = DynamoDbClient.builder() + .endpointOverride(URI.create(amazondynamodbSourceOptions.getUrl())) + // The region is meaningless for local DynamoDb but required for client builder validation + .region(Region.of(amazondynamodbSourceOptions.getRegion())) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(amazondynamodbSourceOptions.getAccessKeyId(), amazondynamodbSourceOptions.getSecretAccessKey()))) + .build(); + this.measurementsType = convertTypes(seaTunnelRowType); + } + + @Override + public void write(SeaTunnelRow element) throws IOException { + + dynamoDbClient.putItem(convertRow(element, seaTunnelRowType)); + } + + @Override + public void close() { + if (dynamoDbClient != null) { + dynamoDbClient.close(); + } + } + + private PutItemRequest convertRow(SeaTunnelRow element, SeaTunnelRowType seaTunnelRowType) { + HashMap itemValues = new HashMap<>(); + for (int index = 0; index < seaTunnelRowType.getFieldNames().length; index++) { + itemValues.put(seaTunnelRowType.getFieldName(index), convertItem(element.getField(index), + seaTunnelRowType.getFieldType(index), + measurementsType.get(index))); + } + return PutItemRequest.builder() + .tableName(amazondynamodbSourceOptions.getTable()) + .item(itemValues) + .build(); + } + + private AttributeValue convertItem(Object value, SeaTunnelDataType seaTunnelDataType, AttributeValue.Type measurementsType) { + if (value == null) { + return AttributeValue.builder().nul(true).build(); + } + switch (measurementsType) { + case N: + return AttributeValue.builder().n(Integer.toString(((Number) value).intValue())).build(); + case S: + return AttributeValue.builder().s(String.valueOf(value)).build(); + case BOOL: + return AttributeValue.builder().bool((Boolean) value).build(); + case B: + return AttributeValue.builder().b(SdkBytes.fromByteArrayUnsafe((byte[]) value)).build(); + case SS: + return AttributeValue.builder().ss((Collection) value).build(); + case NS: + return AttributeValue.builder().ns(((Collection) value) + .stream().map(Object::toString).collect(Collectors.toList())).build(); + case BS: + return AttributeValue.builder().bs( + ((Collection) value) + .stream().map(number -> + SdkBytes.fromByteArray((byte[]) value)).collect(Collectors.toList()) + ).build(); + case M: + MapType mapType = (MapType) seaTunnelDataType; + Map map = (Map) value; + Map resultMap = new HashMap<>(map.size()); + for (Map.Entry entry : map.entrySet()) { + String mapKeyName = entry.getKey(); + resultMap.put(mapKeyName, convertItem(entry.getValue(), mapType.getValueType(), convertType(mapType.getValueType()))); + } + return AttributeValue.builder().m(resultMap).build(); + case L: + ArrayType arrayType = (ArrayType) seaTunnelDataType; + BasicType elementType = arrayType.getElementType(); + return AttributeValue.builder().l(convertItem(value, elementType, convertType(elementType))).build(); + case NUL: + return AttributeValue.builder().nul(true).build(); + default: + throw new UnsupportedOperationException("Unsupported dataType: " + measurementsType); + } + } + + private List convertTypes(SeaTunnelRowType seaTunnelRowType) { + return Arrays.stream(seaTunnelRowType.getFieldTypes()).map(seaTunnelDataType -> convertType(seaTunnelDataType)).collect(Collectors.toList()); + } + + private AttributeValue.Type convertType(SeaTunnelDataType seaTunnelDataType) { + switch (seaTunnelDataType.getSqlType()) { + case INT: + case TINYINT: + case SMALLINT: + case BIGINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + return AttributeValue.Type.N; + case STRING: + case DATE: + case TIME: + case TIMESTAMP: + return AttributeValue.Type.S; + case BOOLEAN: + return AttributeValue.Type.BOOL; + case NULL: + return AttributeValue.Type.NUL; + case BYTES: + return AttributeValue.Type.B; + case MAP: + return AttributeValue.Type.M; + case ARRAY: + return AttributeValue.Type.L; + default: + throw new UnsupportedOperationException("Unsupported dataType: " + seaTunnelDataType); + } + } +} From 4714319981111767d685e6143f1fd985b06f8950 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 10:52:22 +0800 Subject: [PATCH 07/24] fix some bug --- .../seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java index db9526223e9..1da8ee6d11e 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; public class AmazondynamodbWriter extends AbstractSinkWriter { @@ -125,7 +126,10 @@ private AttributeValue convertItem(Object value, SeaTunnelDataType seaTunnelData case L: ArrayType arrayType = (ArrayType) seaTunnelDataType; BasicType elementType = arrayType.getElementType(); - return AttributeValue.builder().l(convertItem(value, elementType, convertType(elementType))).build(); + Object[] l = (Object[]) value; + return AttributeValue.builder() + .l(Stream.of(l).map(o -> convertItem(o, elementType, convertType(elementType))) + .collect(Collectors.toList())).build(); case NUL: return AttributeValue.builder().nul(true).build(); default: From 9f2d18b4c91b3729838ded737320910b9b098528 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 11:47:35 +0800 Subject: [PATCH 08/24] add doc --- docs/en/connector-v2/sink/Amazondynamodb.md | 71 ++++++++++++ docs/en/connector-v2/source/Amazondynamodb.md | 108 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 docs/en/connector-v2/sink/Amazondynamodb.md create mode 100644 docs/en/connector-v2/source/Amazondynamodb.md diff --git a/docs/en/connector-v2/sink/Amazondynamodb.md b/docs/en/connector-v2/sink/Amazondynamodb.md new file mode 100644 index 00000000000..fc0cf748441 --- /dev/null +++ b/docs/en/connector-v2/sink/Amazondynamodb.md @@ -0,0 +1,71 @@ +```` +# Amazondynamodb + +> Amazondynamodb sink connector + +## Description + +Write data to `Amazondynamodb` + +## Key features + +- [x] [batch](../../concept/connector-v2-features.md) +- [ ] [exactly-once](../../concept/connector-v2-features.md) +- [ ] [schema projection](../../concept/connector-v2-features.md) +- [ ] [parallelism](../../concept/connector-v2-features.md) +- [ ] [support user-defined split](../../concept/connector-v2-features.md) + +## Options + +| name | type | required | default value | +|--------------- | ------ |----------| ------------- | +| url | string | yes | - | +| region | string | yes | - | +| accessKeyId | string | yes | - | +| secretAccessKey| string | yes | - | +| table | string | yes | - | +| common-options | | no | - | + +### url [string] + +url to write to Amazondynamodb. + +### region [string] + +The region of Amazondynamodb. + +### accessKeyId [string] + +The access id of Amazondynamodb. + +### secretAccessKey [string] + +The access secret of Amazondynamodb. + +### table [string] + +The table of Amazondynamodb. + +### common options + +Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details. + +## Example + +```bash +Amazondynamodb { + url = "http://127.0.0.1:8000" + region = "us-east-1" + accessKeyId = "dummy-key" + secretAccessKey = "dummy-secret" + table = "TableName" + } +``` + +## Changelog + +### next version + +- Add Amazondynamodb Sink Connector + +```` \ No newline at end of file diff --git a/docs/en/connector-v2/source/Amazondynamodb.md b/docs/en/connector-v2/source/Amazondynamodb.md new file mode 100644 index 00000000000..aa2f38180a7 --- /dev/null +++ b/docs/en/connector-v2/source/Amazondynamodb.md @@ -0,0 +1,108 @@ +# Amazondynamodb + +> Amazondynamodb source connector + +## Description + +Read data from Amazondynamodb. + +## Key features + +- [x] [batch](../../concept/connector-v2-features.md) +- [ ] [stream](../../concept/connector-v2-features.md) +- [ ] [exactly-once](../../concept/connector-v2-features.md) +- [x] [schema projection](../../concept/connector-v2-features.md) +- [ ] [parallelism](../../concept/connector-v2-features.md) +- [ ] [support user-defined split](../../concept/connector-v2-features.md) + +## Options + +| name | type | required | default value | +| -------------- | ------ | -------- | ------------- | +| url | string | yes | - | +| region | string | yes | - | +| accessKeyId | string | yes | - | +| secretAccessKey| string | yes | - | +| table | string | yes | - | +| schema | object | yes | - | +| common-options | | yes | - | + +### url [string] + +url to read to Amazondynamodb. + +### region [string] + +The region of Amazondynamodb. + +### accessKeyId [string] + +The access id of Amazondynamodb. + +### secretAccessKey [string] + +The access secret of Amazondynamodb. + +### table [string] + +The table of Amazondynamodb. + +### schema [object] + +#### fields [Config] + +Amazon Dynamodb is a NOSQL database service of support keys-value storage and document data structure,there is no way to get the data type.Therefore, we must configure schma. + +such as: + +``` +schema { + fields { + id = int + key_aa = string + key_bb = string + } +} +``` + +### common options + +Source Plugin common parameters, refer to [Source Plugin](common-options.md) for details + +## Example + +```bash + Amazondynamodb { + url = "http://127.0.0.1:8000" + region = "us-east-1" + accessKeyId = "dummy-key" + secretAccessKey = "dummy-secret" + table = "TableName" + schema = { + fields { + artist = string + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + } + } + } +``` + +## Changelog + +### next version + +- Add Amazondynamodb Source Connector \ No newline at end of file From 24071a30483f9b84bd23ff4f10af08ab9746eae0 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 20:04:43 +0800 Subject: [PATCH 09/24] add e2e test --- .../connector-amazondynamodb-e2e/pom.xml | 54 +++ .../amazondynamodb/AmazondynamodbIT.java | 347 ++++++++++++++++++ .../amazondynamodbIT_source_to_sink.conf | 72 ++++ .../seatunnel-connector-v2-e2e/pom.xml | 3 +- 4 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml new file mode 100644 index 00000000000..e25e5d4e07b --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml @@ -0,0 +1,54 @@ + + + + + org.apache.seatunnel + seatunnel-connector-v2-e2e + ${revision} + + 4.0.0 + connector-amazondynamodb-e2e + + 2.18.1 + + + + + software.amazon.awssdk + bom + ${amazon.awssdk} + pom + import + + + + + + org.apache.seatunnel + connector-amazondynamodb + ${project.version} + test + + + software.amazon.awssdk + dynamodb + test + + + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java new file mode 100644 index 00000000000..56bba3d2995 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.connector.amazondynamodb; + +import static org.awaitility.Awaitility.given; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.TestContainer; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerLoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.waiters.WaiterResponse; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; +import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest; +import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest; +import software.amazon.awssdk.services.dynamodb.model.DescribeTableResponse; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; +import software.amazon.awssdk.services.dynamodb.model.KeyType; +import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; +import software.amazon.awssdk.services.dynamodb.model.ScanRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanResponse; +import software.amazon.awssdk.services.dynamodb.waiters.DynamoDbWaiter; + +import java.math.BigDecimal; +import java.net.ConnectException; +import java.net.URI; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class AmazondynamodbIT extends TestSuiteBase implements TestResource { + private static final String AMAZONDYNAMODB_DOCKER_IMAGE = "amazon/dynamodb-local"; + private static final String AMAZONDYNAMODB_CONTAINER_HOST = "dynamodb-host"; + private static final int AMAZONDYNAMODB_CONTAINER_PORT = 8000; + private static final String AMAZONDYNAMODB_JOB_CONFIG = "/clickhouse_to_clickhouse.conf"; + private static final String SINK_TABLE = "sink_table"; + private static final String SOURCE_TABLE = "source_table"; + private static final String PARTITION_KEY = "id"; + + private GenericContainer dynamoDB; + protected DynamoDbClient dynamoDbClient; + + @TestTemplate + public void testClickhouse(TestContainer container) throws Exception { + Container.ExecResult execResult = container.executeJob(AMAZONDYNAMODB_JOB_CONFIG); + Assertions.assertEquals(0, execResult.getExitCode()); + assertHasData(); + compareResult(); + clearSinkTable(); + } + + @BeforeAll + @Override + public void startUp() throws Exception { + dynamoDB = new GenericContainer<>(AMAZONDYNAMODB_DOCKER_IMAGE) + .withNetwork(NETWORK) + .withNetworkAliases(AMAZONDYNAMODB_CONTAINER_HOST) + .withExposedPorts(AMAZONDYNAMODB_CONTAINER_PORT) + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(AMAZONDYNAMODB_DOCKER_IMAGE))); + Startables.deepStart(Stream.of(dynamoDB)).join(); + log.info("dynamodb container started"); + given().ignoreExceptions() + .await() + .atLeast(100, TimeUnit.MILLISECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .untilAsserted(this::initializeDynamodbClient); + batchInsertData(); + + } + + @AfterAll + @Override + public void tearDown() throws Exception { + if (dynamoDB != null) { + dynamoDB.close(); + } + } + + private void initializeDynamodbClient() throws ConnectException { + dynamoDbClient = DynamoDbClient.builder() + .endpointOverride(URI.create("http://" + AMAZONDYNAMODB_CONTAINER_HOST + ":" + AMAZONDYNAMODB_CONTAINER_PORT)) + // The region is meaningless for local DynamoDb but required for client builder validation + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("dummy-key", "dummy-secret"))) + .build(); + + createTable(dynamoDbClient, SOURCE_TABLE); + createTable(dynamoDbClient, SINK_TABLE); + } + + private void batchInsertData() { + dynamoDbClient.putItem(PutItemRequest.builder() + .tableName(SOURCE_TABLE) + .item(randomRow()) + .build()); + } + + private void clearSinkTable() { + dynamoDbClient.deleteTable(DeleteTableRequest.builder().tableName(SINK_TABLE).build()); + createTable(dynamoDbClient, SINK_TABLE); + } + + private void assertHasData() { + ScanResponse scan = dynamoDbClient.scan(ScanRequest.builder().tableName(SINK_TABLE).build()); + Assertions.assertTrue(scan.hasItems(), "sink table is empty."); + } + + private void compareResult() { + Map sourceAttributeValueMap = dynamoDbClient.scan(ScanRequest.builder().tableName(SOURCE_TABLE).build()).items().get(0); + Map sinkAttributeValueMap = dynamoDbClient.scan(ScanRequest.builder().tableName(SINK_TABLE).build()).items().get(0); + sourceAttributeValueMap.keySet().forEach(key -> { + AttributeValue sourceAttributeValue = sourceAttributeValueMap.get(key); + AttributeValue sinkAttributeValue = sinkAttributeValueMap.get(key); + Assertions.assertEquals(sourceAttributeValue, sinkAttributeValue); + }); + + } + + private Map randomRow() { + SeaTunnelRowType seatunnelRowType = new SeaTunnelRowType( + new String[]{ + "id", + "c_map", + "c_array", + "c_string", + "c_boolean", + "c_tinyint", + "c_smallint", + "c_int", + "c_bigint", + "c_float", + "c_double", + "c_decimal", + "c_bytes", + "c_date", + "c_timestamp" + }, + new SeaTunnelDataType[]{ + BasicType.LONG_TYPE, + new MapType(BasicType.STRING_TYPE, BasicType.SHORT_TYPE), + ArrayType.BYTE_ARRAY_TYPE, + BasicType.STRING_TYPE, + BasicType.BOOLEAN_TYPE, + BasicType.BYTE_TYPE, + BasicType.SHORT_TYPE, + BasicType.INT_TYPE, + BasicType.LONG_TYPE, + BasicType.FLOAT_TYPE, + BasicType.DOUBLE_TYPE, + new DecimalType(2, 1), + PrimitiveByteArrayType.INSTANCE, + LocalTimeType.LOCAL_DATE_TYPE, + LocalTimeType.LOCAL_DATE_TIME_TYPE + } + ); + + SeaTunnelRow row = new SeaTunnelRow( + new Object[]{ + 1L, + Collections.singletonMap("key", Short.parseShort("1")), + new Byte[]{Byte.parseByte("1")}, + "string", + Boolean.FALSE, + Byte.parseByte("1"), + Short.parseShort("1"), + Integer.parseInt("1"), + Long.parseLong("1"), + Float.parseFloat("1.1"), + Double.parseDouble("1.1"), + BigDecimal.valueOf(11, 1), + "test".getBytes(), + LocalDate.now(), + LocalDateTime.now() + }); + + Map data = new HashMap<>(seatunnelRowType.getTotalFields()); + for (int index = 0; index < seatunnelRowType.getTotalFields(); index++) { + data.put(seatunnelRowType.getFieldName(index), + convertItem(row.getField(index), seatunnelRowType.getFieldType(index), convertType(seatunnelRowType.getFieldType(index)))); + } + return data; + } + + private static void createTable(DynamoDbClient ddb, String tableName) { + DynamoDbWaiter dbWaiter = ddb.waiter(); + CreateTableRequest request = CreateTableRequest.builder() + .attributeDefinitions(AttributeDefinition.builder() + .attributeName(PARTITION_KEY) + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName(PARTITION_KEY) + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .tableName(tableName) + .build(); + + try { + ddb.createTable(request); + DescribeTableRequest tableRequest = DescribeTableRequest.builder() + .tableName(tableName) + .build(); + + // Wait until the Amazon DynamoDB table is created. + WaiterResponse waiterResponse = dbWaiter.waitUntilTableExists(tableRequest); + waiterResponse.matched().response().ifPresent(describeTableResponse -> { + log.info(describeTableResponse.toString()); + }); + + } catch (DynamoDbException e) { + log.error(e.getMessage()); + } + } + + private AttributeValue convertItem(Object value, SeaTunnelDataType seaTunnelDataType, AttributeValue.Type measurementsType) { + if (value == null) { + return AttributeValue.builder().nul(true).build(); + } + switch (measurementsType) { + case N: + return AttributeValue.builder().n(Integer.toString(((Number) value).intValue())).build(); + case S: + return AttributeValue.builder().s(String.valueOf(value)).build(); + case BOOL: + return AttributeValue.builder().bool((Boolean) value).build(); + case B: + return AttributeValue.builder().b(SdkBytes.fromByteArrayUnsafe((byte[]) value)).build(); + case SS: + return AttributeValue.builder().ss((Collection) value).build(); + case NS: + return AttributeValue.builder().ns(((Collection) value) + .stream().map(Object::toString).collect(Collectors.toList())).build(); + case BS: + return AttributeValue.builder().bs( + ((Collection) value) + .stream().map(number -> + SdkBytes.fromByteArray((byte[]) value)).collect(Collectors.toList()) + ).build(); + case M: + MapType mapType = (MapType) seaTunnelDataType; + Map map = (Map) value; + Map resultMap = new HashMap<>(map.size()); + for (Map.Entry entry : map.entrySet()) { + String mapKeyName = entry.getKey(); + resultMap.put(mapKeyName, convertItem(entry.getValue(), mapType.getValueType(), convertType(mapType.getValueType()))); + } + return AttributeValue.builder().m(resultMap).build(); + case L: + ArrayType arrayType = (ArrayType) seaTunnelDataType; + BasicType elementType = arrayType.getElementType(); + Object[] l = (Object[]) value; + return AttributeValue.builder() + .l(Stream.of(l).map(o -> convertItem(o, elementType, convertType(elementType))) + .collect(Collectors.toList())).build(); + case NUL: + return AttributeValue.builder().nul(true).build(); + default: + throw new UnsupportedOperationException("Unsupported dataType: " + measurementsType); + } + } + + private AttributeValue.Type convertType(SeaTunnelDataType seaTunnelDataType) { + switch (seaTunnelDataType.getSqlType()) { + case INT: + case TINYINT: + case SMALLINT: + case BIGINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + return AttributeValue.Type.N; + case STRING: + case DATE: + case TIME: + case TIMESTAMP: + return AttributeValue.Type.S; + case BOOLEAN: + return AttributeValue.Type.BOOL; + case NULL: + return AttributeValue.Type.NUL; + case BYTES: + return AttributeValue.Type.B; + case MAP: + return AttributeValue.Type.M; + case ARRAY: + return AttributeValue.Type.L; + default: + throw new UnsupportedOperationException("Unsupported dataType: " + seaTunnelDataType); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf new file mode 100644 index 00000000000..4f4e770b44b --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + Amazondynamodb { + url = "http://dynamodb-host:8000" + region = "us-east-1" + accessKeyId = "dummy-key" + secretAccessKey = "dummy-secret" + table = "SINK_TABLE" + schema = { + fields { + id = bigint + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(2, 1)" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + } + } + } +} + +transform { + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql +} + +sink { + Amazondynamodb { + url = "http://dynamodb-host:8000" + region = "us-east-1" + accessKeyId = "dummy-key" + secretAccessKey = "dummy-secret" + table = "SINK_TABLE" + } + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/category/sink-v2 +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml index d5404c565d7..6754dfaf86a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml @@ -29,6 +29,7 @@ connector-redis-e2e connector-clickhouse-e2e connector-influxdb-e2e + connector-amazondynamodb-e2e seatunnel-connector-v2-e2e @@ -55,4 +56,4 @@ - \ No newline at end of file + From 05c05d9d095a5e8ac5a2c01c1de41045425de480 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 21:00:35 +0800 Subject: [PATCH 10/24] fix e2e test error --- .../e2e/connector/amazondynamodb/AmazondynamodbIT.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java index 56bba3d2995..30ff6c99d41 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java @@ -32,6 +32,7 @@ import org.apache.seatunnel.e2e.common.TestSuiteBase; import org.apache.seatunnel.e2e.common.container.TestContainer; +import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -107,13 +108,14 @@ public void startUp() throws Exception { .withNetworkAliases(AMAZONDYNAMODB_CONTAINER_HOST) .withExposedPorts(AMAZONDYNAMODB_CONTAINER_PORT) .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(AMAZONDYNAMODB_DOCKER_IMAGE))); + dynamoDB.setPortBindings(Lists.newArrayList(String.format("%s:%s", AMAZONDYNAMODB_CONTAINER_PORT, AMAZONDYNAMODB_CONTAINER_PORT))); Startables.deepStart(Stream.of(dynamoDB)).join(); log.info("dynamodb container started"); given().ignoreExceptions() .await() .atLeast(100, TimeUnit.MILLISECONDS) .pollInterval(500, TimeUnit.MILLISECONDS) - .atMost(30, TimeUnit.SECONDS) + .atMost(120, TimeUnit.SECONDS) .untilAsserted(this::initializeDynamodbClient); batchInsertData(); @@ -129,7 +131,7 @@ public void tearDown() throws Exception { private void initializeDynamodbClient() throws ConnectException { dynamoDbClient = DynamoDbClient.builder() - .endpointOverride(URI.create("http://" + AMAZONDYNAMODB_CONTAINER_HOST + ":" + AMAZONDYNAMODB_CONTAINER_PORT)) + .endpointOverride(URI.create("http://" + dynamoDB.getHost() + ":" + AMAZONDYNAMODB_CONTAINER_PORT)) // The region is meaningless for local DynamoDb but required for client builder validation .region(Region.US_EAST_1) .credentialsProvider(StaticCredentialsProvider.create( @@ -188,7 +190,7 @@ private Map randomRow() { "c_timestamp" }, new SeaTunnelDataType[]{ - BasicType.LONG_TYPE, + BasicType.STRING_TYPE, new MapType(BasicType.STRING_TYPE, BasicType.SHORT_TYPE), ArrayType.BYTE_ARRAY_TYPE, BasicType.STRING_TYPE, @@ -208,7 +210,7 @@ private Map randomRow() { SeaTunnelRow row = new SeaTunnelRow( new Object[]{ - 1L, + "1", Collections.singletonMap("key", Short.parseShort("1")), new Byte[]{Byte.parseByte("1")}, "string", From b586ce96a7e2d43efd1beef776fec7ddddadaba2 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 21:02:39 +0800 Subject: [PATCH 11/24] fix e2e test error --- .../e2e/connector/amazondynamodb/AmazondynamodbIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java index 30ff6c99d41..14730d25558 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java @@ -83,7 +83,7 @@ public class AmazondynamodbIT extends TestSuiteBase implements TestResource { private static final String AMAZONDYNAMODB_DOCKER_IMAGE = "amazon/dynamodb-local"; private static final String AMAZONDYNAMODB_CONTAINER_HOST = "dynamodb-host"; private static final int AMAZONDYNAMODB_CONTAINER_PORT = 8000; - private static final String AMAZONDYNAMODB_JOB_CONFIG = "/clickhouse_to_clickhouse.conf"; + private static final String AMAZONDYNAMODB_JOB_CONFIG = "/amazondynamodbIT_source_to_sink.conf"; private static final String SINK_TABLE = "sink_table"; private static final String SOURCE_TABLE = "source_table"; private static final String PARTITION_KEY = "id"; From c0a97519c313966ff099d19732f0663a717c8e1e Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 22:00:36 +0800 Subject: [PATCH 12/24] fix e2e test error --- .../src/test/resources/amazondynamodbIT_source_to_sink.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf index 4f4e770b44b..cfbd7b4f97d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/resources/amazondynamodbIT_source_to_sink.conf @@ -30,10 +30,10 @@ source { region = "us-east-1" accessKeyId = "dummy-key" secretAccessKey = "dummy-secret" - table = "SINK_TABLE" + table = "source_table" schema = { fields { - id = bigint + id = string c_map = "map" c_array = "array" c_string = string @@ -65,7 +65,7 @@ sink { region = "us-east-1" accessKeyId = "dummy-key" secretAccessKey = "dummy-secret" - table = "SINK_TABLE" + table = "sink_table" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, # please go to https://seatunnel.apache.org/docs/category/sink-v2 From 91ab78bcf83776a3e4c743febb801192718a37fb Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 22:32:35 +0800 Subject: [PATCH 13/24] fix e2e test error --- .../amazondynamodb/source/AmazondynamodbSourceReader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index f3656b3ccd6..5c9a54faf5b 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -120,6 +120,9 @@ private Object convert(SeaTunnelDataType seaTunnelDataType, AttributeValue at } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) { return attributeValue.bool(); } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) { + if (attributeValue.n() != null) { + return Byte.parseByte(attributeValue.n()); + } return attributeValue.s().getBytes(StandardCharsets.UTF_8)[0]; } else if (BasicType.SHORT_TYPE.equals(seaTunnelDataType)) { return Short.parseShort(attributeValue.n()); From 38692ef267b12c5f3013e1df570837ecb4aaa6b2 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 23:37:19 +0800 Subject: [PATCH 14/24] fix e2e test error --- .../source/AmazondynamodbSource.java | 4 +-- .../source/AmazondynamodbSourceReader.java | 33 ++++++++++++------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java index bef92154107..b19854777f4 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java @@ -50,6 +50,8 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { amazondynamodbSourceOptions = new AmazondynamodbSourceOptions(pluginConfig); + Config schema = amazondynamodbSourceOptions.getSchema(); + typeInfo = SeaTunnelSchema.buildWithConfig(schema).getSeaTunnelRowType(); } @Override @@ -59,8 +61,6 @@ public Boundedness getBoundedness() { @Override public SeaTunnelDataType getProducedType() { - Config schema = amazondynamodbSourceOptions.getSchema(); - typeInfo = SeaTunnelSchema.buildWithConfig(schema).getSeaTunnelRowType(); return this.typeInfo; } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index 5c9a54faf5b..d5d85e83df4 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -34,6 +34,7 @@ import lombok.extern.slf4j.Slf4j; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; @@ -41,6 +42,7 @@ import software.amazon.awssdk.services.dynamodb.model.ScanResponse; import java.io.IOException; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -153,23 +155,30 @@ private Object convert(SeaTunnelDataType seaTunnelDataType, AttributeValue at }); return seatunnelMap; } else if (seaTunnelDataType instanceof ArrayType) { - List seatunnelList = new ArrayList<>(attributeValue.l().size()); + Object array = Array.newInstance(String.class, attributeValue.l().size()); if (attributeValue.hasL()) { - attributeValue.l().forEach(l -> { - seatunnelList.add(convert(((ArrayType) seaTunnelDataType).getElementType(), l)); - }); + List datas = attributeValue.l(); + array = Array.newInstance(((ArrayType) seaTunnelDataType).getElementType().getTypeClass(), attributeValue.l().size()); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, convert(((ArrayType) seaTunnelDataType).getElementType(), datas.get(index))); + } } else if (attributeValue.hasSs()) { - seatunnelList.addAll(attributeValue.ss()); + List datas = attributeValue.ss(); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, AttributeValue.fromS(datas.get(index))); + } } else if (attributeValue.hasNs()) { - attributeValue.ns().forEach(s -> { - seatunnelList.addAll(attributeValue.ns()); - }); + List datas = attributeValue.ns(); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, AttributeValue.fromS(datas.get(index))); + } } else if (attributeValue.hasBs()) { - attributeValue.bs().forEach(s -> { - seatunnelList.add(s.asByteArray()); - }); + List datas = attributeValue.bs(); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, AttributeValue.fromB(datas.get(index))); + } } - return seatunnelList; + return array; } else { throw new IllegalStateException("Unexpected value: " + seaTunnelDataType); } From efebf11e31f31a8c1057701d823411812f389ffc Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sun, 30 Oct 2022 23:46:26 +0800 Subject: [PATCH 15/24] fix e2e test error --- .../e2e/connector/amazondynamodb/AmazondynamodbIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java index 14730d25558..de78241c9c5 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java @@ -92,7 +92,7 @@ public class AmazondynamodbIT extends TestSuiteBase implements TestResource { protected DynamoDbClient dynamoDbClient; @TestTemplate - public void testClickhouse(TestContainer container) throws Exception { + public void testAmazondynamodb(TestContainer container) throws Exception { Container.ExecResult execResult = container.executeJob(AMAZONDYNAMODB_JOB_CONFIG); Assertions.assertEquals(0, execResult.getExitCode()); assertHasData(); From 73bb6dfbdae5cfb4a59e9228f76be1e2488199f2 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Mon, 31 Oct 2022 14:46:44 +0800 Subject: [PATCH 16/24] add deserializer and serializer --- .../DefaultSeaTunnelRowDeserializer.java | 137 ++++++++++++++++ .../DefaultSeaTunnelRowSerializer.java | 147 ++++++++++++++++++ .../serialize/SeaTunnelRowDeserializer.java | 29 ++++ .../serialize/SeaTunnelRowSerializer.java | 27 ++++ .../sink/AmazondynamodbWriter.java | 121 +------------- .../sink/DdynamoDbSinkClient.java | 29 ++++ .../source/AmazondynamodbSourceReader.java | 109 +------------ 7 files changed, 379 insertions(+), 220 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowDeserializer.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowSerializer.java create mode 100644 seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java new file mode 100644 index 00000000000..f990665a90a --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; + +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +import java.lang.reflect.Array; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DefaultSeaTunnelRowDeserializer implements SeaTunnelRowDeserializer { + + private final SeaTunnelRowType typeInfo; + + public DefaultSeaTunnelRowDeserializer(SeaTunnelRowType typeInfo) { + this.typeInfo = typeInfo; + } + + @Override + public SeaTunnelRow deserialize(Map item) { + SeaTunnelDataType[] seaTunnelDataTypes = typeInfo.getFieldTypes(); + return new SeaTunnelRow(convertRow(seaTunnelDataTypes, item).toArray()); + } + + private List convertRow(SeaTunnelDataType[] seaTunnelDataTypes, Map item) { + List fields = new ArrayList<>(); + String[] fieldNames = typeInfo.getFieldNames(); + for (int i = 0; i < seaTunnelDataTypes.length; i++) { + SeaTunnelDataType seaTunnelDataType = seaTunnelDataTypes[i]; + AttributeValue attributeValue = item.get(fieldNames[i]); + fields.add(convert(seaTunnelDataType, attributeValue)); + } + return fields; + } + + private Object convert(SeaTunnelDataType seaTunnelDataType, AttributeValue attributeValue) { + if (attributeValue.type().equals(AttributeValue.Type.NUL)) { + return null; + } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) { + return attributeValue.bool(); + } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) { + if (attributeValue.n() != null) { + return Byte.parseByte(attributeValue.n()); + } + return attributeValue.s().getBytes(StandardCharsets.UTF_8)[0]; + } else if (BasicType.SHORT_TYPE.equals(seaTunnelDataType)) { + return Short.parseShort(attributeValue.n()); + } else if (BasicType.INT_TYPE.equals(seaTunnelDataType)) { + return Integer.parseInt(attributeValue.n()); + } else if (BasicType.LONG_TYPE.equals(seaTunnelDataType)) { + return Long.parseLong(attributeValue.n()); + } else if (seaTunnelDataType instanceof DecimalType) { + return new BigDecimal(attributeValue.n()); + } else if (BasicType.FLOAT_TYPE.equals(seaTunnelDataType)) { + return Float.parseFloat(attributeValue.n()); + } else if (BasicType.DOUBLE_TYPE.equals(seaTunnelDataType)) { + return Double.parseDouble(attributeValue.n()); + } else if (BasicType.STRING_TYPE.equals(seaTunnelDataType)) { + return attributeValue.s(); + } else if (LocalTimeType.LOCAL_TIME_TYPE.equals(seaTunnelDataType)) { + return LocalTime.parse(attributeValue.s()); + } else if (LocalTimeType.LOCAL_DATE_TYPE.equals(seaTunnelDataType)) { + return LocalDate.parse(attributeValue.s()); + } else if (LocalTimeType.LOCAL_DATE_TIME_TYPE.equals(seaTunnelDataType)) { + return LocalDateTime.parse(attributeValue.s()); + } else if (PrimitiveByteArrayType.INSTANCE.equals(seaTunnelDataType)) { + return attributeValue.b().asByteArray(); + } else if (seaTunnelDataType instanceof MapType) { + Map seatunnelMap = new HashMap<>(); + attributeValue.m().forEach((s, attributeValueInfo) -> { + seatunnelMap.put(s, convert(((MapType) seaTunnelDataType).getValueType(), attributeValueInfo)); + }); + return seatunnelMap; + } else if (seaTunnelDataType instanceof ArrayType) { + Object array = Array.newInstance(String.class, attributeValue.l().size()); + if (attributeValue.hasL()) { + List datas = attributeValue.l(); + array = Array.newInstance(((ArrayType) seaTunnelDataType).getElementType().getTypeClass(), attributeValue.l().size()); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, convert(((ArrayType) seaTunnelDataType).getElementType(), datas.get(index))); + } + } else if (attributeValue.hasSs()) { + List datas = attributeValue.ss(); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, AttributeValue.fromS(datas.get(index))); + } + } else if (attributeValue.hasNs()) { + List datas = attributeValue.ns(); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, AttributeValue.fromS(datas.get(index))); + } + } else if (attributeValue.hasBs()) { + List datas = attributeValue.bs(); + for (int index = 0; index < datas.size(); index++) { + Array.set(array, index, AttributeValue.fromB(datas.get(index))); + } + } + return array; + } else { + throw new IllegalStateException("Unexpected value: " + seaTunnelDataType); + } + } + +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java new file mode 100644 index 00000000000..017436c20b3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; + +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class DefaultSeaTunnelRowSerializer implements SeaTunnelRowSerializer { + + private final SeaTunnelRowType seaTunnelRowType; + private final AmazondynamodbSourceOptions amazondynamodbSourceOptions; + private final List measurementsType; + + public DefaultSeaTunnelRowSerializer(SeaTunnelRowType seaTunnelRowType, AmazondynamodbSourceOptions amazondynamodbSourceOptions) { + this.seaTunnelRowType = seaTunnelRowType; + this.amazondynamodbSourceOptions = amazondynamodbSourceOptions; + this.measurementsType = convertTypes(seaTunnelRowType); + } + + @Override + public PutItemRequest serialize(SeaTunnelRow seaTunnelRow) { + HashMap itemValues = new HashMap<>(); + for (int index = 0; index < seaTunnelRowType.getFieldNames().length; index++) { + itemValues.put(seaTunnelRowType.getFieldName(index), convertItem(seaTunnelRow.getField(index), + seaTunnelRowType.getFieldType(index), + measurementsType.get(index))); + } + return PutItemRequest.builder() + .tableName(amazondynamodbSourceOptions.getTable()) + .item(itemValues) + .build(); + } + + private List convertTypes(SeaTunnelRowType seaTunnelRowType) { + return Arrays.stream(seaTunnelRowType.getFieldTypes()).map(this::convertType).collect(Collectors.toList()); + } + + private AttributeValue.Type convertType(SeaTunnelDataType seaTunnelDataType) { + switch (seaTunnelDataType.getSqlType()) { + case INT: + case TINYINT: + case SMALLINT: + case BIGINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + return AttributeValue.Type.N; + case STRING: + case DATE: + case TIME: + case TIMESTAMP: + return AttributeValue.Type.S; + case BOOLEAN: + return AttributeValue.Type.BOOL; + case NULL: + return AttributeValue.Type.NUL; + case BYTES: + return AttributeValue.Type.B; + case MAP: + return AttributeValue.Type.M; + case ARRAY: + return AttributeValue.Type.L; + default: + throw new UnsupportedOperationException("Unsupported dataType: " + seaTunnelDataType); + } + } + + private AttributeValue convertItem(Object value, SeaTunnelDataType seaTunnelDataType, AttributeValue.Type measurementsType) { + if (value == null) { + return AttributeValue.builder().nul(true).build(); + } + switch (measurementsType) { + case N: + return AttributeValue.builder().n(Integer.toString(((Number) value).intValue())).build(); + case S: + return AttributeValue.builder().s(String.valueOf(value)).build(); + case BOOL: + return AttributeValue.builder().bool((Boolean) value).build(); + case B: + return AttributeValue.builder().b(SdkBytes.fromByteArrayUnsafe((byte[]) value)).build(); + case SS: + return AttributeValue.builder().ss((Collection) value).build(); + case NS: + return AttributeValue.builder().ns(((Collection) value) + .stream().map(Object::toString).collect(Collectors.toList())).build(); + case BS: + return AttributeValue.builder().bs( + ((Collection) value) + .stream().map(number -> + SdkBytes.fromByteArray((byte[]) value)).collect(Collectors.toList()) + ).build(); + case M: + MapType mapType = (MapType) seaTunnelDataType; + Map map = (Map) value; + Map resultMap = new HashMap<>(map.size()); + for (Map.Entry entry : map.entrySet()) { + String mapKeyName = entry.getKey(); + resultMap.put(mapKeyName, convertItem(entry.getValue(), mapType.getValueType(), convertType(mapType.getValueType()))); + } + return AttributeValue.builder().m(resultMap).build(); + case L: + ArrayType arrayType = (ArrayType) seaTunnelDataType; + BasicType elementType = arrayType.getElementType(); + Object[] l = (Object[]) value; + return AttributeValue.builder() + .l(Stream.of(l).map(o -> convertItem(o, elementType, convertType(elementType))) + .collect(Collectors.toList())).build(); + case NUL: + return AttributeValue.builder().nul(true).build(); + default: + throw new UnsupportedOperationException("Unsupported dataType: " + measurementsType); + } + } + +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowDeserializer.java new file mode 100644 index 00000000000..5596701a1a3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowDeserializer.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +import java.util.Map; + +public interface SeaTunnelRowDeserializer { + + SeaTunnelRow deserialize(Map item); +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowSerializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowSerializer.java new file mode 100644 index 00000000000..c4b30e27f0c --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/SeaTunnelRowSerializer.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; + +public interface SeaTunnelRowSerializer { + + PutItemRequest serialize(SeaTunnelRow seaTunnelRow); +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java index 1da8ee6d11e..6b439add386 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java @@ -17,43 +17,27 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; -import org.apache.seatunnel.api.table.type.ArrayType; -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.MapType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize.DefaultSeaTunnelRowSerializer; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize.SeaTunnelRowSerializer; import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; import java.io.IOException; import java.net.URI; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; public class AmazondynamodbWriter extends AbstractSinkWriter { - private final AmazondynamodbSourceOptions amazondynamodbSourceOptions; - private final SeaTunnelRowType seaTunnelRowType; private final DynamoDbClient dynamoDbClient; - private final List measurementsType; + private final SeaTunnelRowSerializer serializer; public AmazondynamodbWriter(AmazondynamodbSourceOptions amazondynamodbSourceOptions, SeaTunnelRowType seaTunnelRowType) { - this.amazondynamodbSourceOptions = amazondynamodbSourceOptions; - this.seaTunnelRowType = seaTunnelRowType; dynamoDbClient = DynamoDbClient.builder() .endpointOverride(URI.create(amazondynamodbSourceOptions.getUrl())) // The region is meaningless for local DynamoDb but required for client builder validation @@ -61,13 +45,12 @@ public AmazondynamodbWriter(AmazondynamodbSourceOptions amazondynamodbSourceOpti .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create(amazondynamodbSourceOptions.getAccessKeyId(), amazondynamodbSourceOptions.getSecretAccessKey()))) .build(); - this.measurementsType = convertTypes(seaTunnelRowType); + serializer = new DefaultSeaTunnelRowSerializer(seaTunnelRowType, amazondynamodbSourceOptions); } @Override public void write(SeaTunnelRow element) throws IOException { - - dynamoDbClient.putItem(convertRow(element, seaTunnelRowType)); + dynamoDbClient.putItem(serializer.serialize(element)); } @Override @@ -76,98 +59,4 @@ public void close() { dynamoDbClient.close(); } } - - private PutItemRequest convertRow(SeaTunnelRow element, SeaTunnelRowType seaTunnelRowType) { - HashMap itemValues = new HashMap<>(); - for (int index = 0; index < seaTunnelRowType.getFieldNames().length; index++) { - itemValues.put(seaTunnelRowType.getFieldName(index), convertItem(element.getField(index), - seaTunnelRowType.getFieldType(index), - measurementsType.get(index))); - } - return PutItemRequest.builder() - .tableName(amazondynamodbSourceOptions.getTable()) - .item(itemValues) - .build(); - } - - private AttributeValue convertItem(Object value, SeaTunnelDataType seaTunnelDataType, AttributeValue.Type measurementsType) { - if (value == null) { - return AttributeValue.builder().nul(true).build(); - } - switch (measurementsType) { - case N: - return AttributeValue.builder().n(Integer.toString(((Number) value).intValue())).build(); - case S: - return AttributeValue.builder().s(String.valueOf(value)).build(); - case BOOL: - return AttributeValue.builder().bool((Boolean) value).build(); - case B: - return AttributeValue.builder().b(SdkBytes.fromByteArrayUnsafe((byte[]) value)).build(); - case SS: - return AttributeValue.builder().ss((Collection) value).build(); - case NS: - return AttributeValue.builder().ns(((Collection) value) - .stream().map(Object::toString).collect(Collectors.toList())).build(); - case BS: - return AttributeValue.builder().bs( - ((Collection) value) - .stream().map(number -> - SdkBytes.fromByteArray((byte[]) value)).collect(Collectors.toList()) - ).build(); - case M: - MapType mapType = (MapType) seaTunnelDataType; - Map map = (Map) value; - Map resultMap = new HashMap<>(map.size()); - for (Map.Entry entry : map.entrySet()) { - String mapKeyName = entry.getKey(); - resultMap.put(mapKeyName, convertItem(entry.getValue(), mapType.getValueType(), convertType(mapType.getValueType()))); - } - return AttributeValue.builder().m(resultMap).build(); - case L: - ArrayType arrayType = (ArrayType) seaTunnelDataType; - BasicType elementType = arrayType.getElementType(); - Object[] l = (Object[]) value; - return AttributeValue.builder() - .l(Stream.of(l).map(o -> convertItem(o, elementType, convertType(elementType))) - .collect(Collectors.toList())).build(); - case NUL: - return AttributeValue.builder().nul(true).build(); - default: - throw new UnsupportedOperationException("Unsupported dataType: " + measurementsType); - } - } - - private List convertTypes(SeaTunnelRowType seaTunnelRowType) { - return Arrays.stream(seaTunnelRowType.getFieldTypes()).map(seaTunnelDataType -> convertType(seaTunnelDataType)).collect(Collectors.toList()); - } - - private AttributeValue.Type convertType(SeaTunnelDataType seaTunnelDataType) { - switch (seaTunnelDataType.getSqlType()) { - case INT: - case TINYINT: - case SMALLINT: - case BIGINT: - case FLOAT: - case DOUBLE: - case DECIMAL: - return AttributeValue.Type.N; - case STRING: - case DATE: - case TIME: - case TIMESTAMP: - return AttributeValue.Type.S; - case BOOLEAN: - return AttributeValue.Type.BOOL; - case NULL: - return AttributeValue.Type.NUL; - case BYTES: - return AttributeValue.Type.B; - case MAP: - return AttributeValue.Type.M; - case ARRAY: - return AttributeValue.Type.L; - default: - throw new UnsupportedOperationException("Unsupported dataType: " + seaTunnelDataType); - } - } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java new file mode 100644 index 00000000000..dbeaac4e8b8 --- /dev/null +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; + +/** + * Copyright @ 2022科大讯飞。 All rights reserved. + * @Title DdynamoDbSinkClient + * @Project incubator-seatunnel + * @Description TODO + * @author gdliu3 + * @date 2022/10/31 14:21 + */ +public class DdynamoDbSinkClient { +} diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index d5d85e83df4..b4d4575cb50 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -18,41 +18,24 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; import org.apache.seatunnel.api.source.Collector; -import org.apache.seatunnel.api.table.type.ArrayType; -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.DecimalType; -import org.apache.seatunnel.api.table.type.LocalTimeType; -import org.apache.seatunnel.api.table.type.MapType; -import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize.DefaultSeaTunnelRowDeserializer; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize.SeaTunnelRowDeserializer; import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; import lombok.extern.slf4j.Slf4j; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.ScanRequest; import software.amazon.awssdk.services.dynamodb.model.ScanResponse; import java.io.IOException; -import java.lang.reflect.Array; -import java.math.BigDecimal; import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; @Slf4j public class AmazondynamodbSourceReader extends AbstractSingleSplitReader { @@ -61,6 +44,7 @@ public class AmazondynamodbSourceReader extends AbstractSingleSplitReader output) throws Exception { .build()); if (scan.hasItems()) { scan.items().forEach(item -> { - output.collect(converterToRow(item, typeInfo)); + output.collect(seaTunnelRowDeserializer.deserialize(item)); }); } context.signalNoMoreElement(); } - - private SeaTunnelRow converterToRow(Map item, SeaTunnelRowType typeInfo) { - SeaTunnelDataType[] seaTunnelDataTypes = typeInfo.getFieldTypes(); - return new SeaTunnelRow(convertRow(seaTunnelDataTypes, item).toArray()); - } - - private List convertRow(SeaTunnelDataType[] seaTunnelDataTypes, Map item) { - List fields = new ArrayList<>(); - String[] fieldNames = typeInfo.getFieldNames(); - for (int i = 0; i < seaTunnelDataTypes.length; i++) { - SeaTunnelDataType seaTunnelDataType = seaTunnelDataTypes[i]; - AttributeValue attributeValue = item.get(fieldNames[i]); - fields.add(convert(seaTunnelDataType, attributeValue)); - } - return fields; - } - - private Object convert(SeaTunnelDataType seaTunnelDataType, AttributeValue attributeValue) { - if (attributeValue.type().equals(AttributeValue.Type.NUL)) { - return null; - } else if (BasicType.BOOLEAN_TYPE.equals(seaTunnelDataType)) { - return attributeValue.bool(); - } else if (BasicType.BYTE_TYPE.equals(seaTunnelDataType)) { - if (attributeValue.n() != null) { - return Byte.parseByte(attributeValue.n()); - } - return attributeValue.s().getBytes(StandardCharsets.UTF_8)[0]; - } else if (BasicType.SHORT_TYPE.equals(seaTunnelDataType)) { - return Short.parseShort(attributeValue.n()); - } else if (BasicType.INT_TYPE.equals(seaTunnelDataType)) { - return Integer.parseInt(attributeValue.n()); - } else if (BasicType.LONG_TYPE.equals(seaTunnelDataType)) { - return Long.parseLong(attributeValue.n()); - } else if (seaTunnelDataType instanceof DecimalType) { - return new BigDecimal(attributeValue.n()); - } else if (BasicType.FLOAT_TYPE.equals(seaTunnelDataType)) { - return Float.parseFloat(attributeValue.n()); - } else if (BasicType.DOUBLE_TYPE.equals(seaTunnelDataType)) { - return Double.parseDouble(attributeValue.n()); - } else if (BasicType.STRING_TYPE.equals(seaTunnelDataType)) { - return attributeValue.s(); - } else if (LocalTimeType.LOCAL_TIME_TYPE.equals(seaTunnelDataType)) { - return LocalTime.parse(attributeValue.s()); - } else if (LocalTimeType.LOCAL_DATE_TYPE.equals(seaTunnelDataType)) { - return LocalDate.parse(attributeValue.s()); - } else if (LocalTimeType.LOCAL_DATE_TIME_TYPE.equals(seaTunnelDataType)) { - return LocalDateTime.parse(attributeValue.s()); - } else if (PrimitiveByteArrayType.INSTANCE.equals(seaTunnelDataType)) { - return attributeValue.b().asByteArray(); - } else if (seaTunnelDataType instanceof MapType) { - Map seatunnelMap = new HashMap<>(); - attributeValue.m().forEach((s, attributeValueInfo) -> { - seatunnelMap.put(s, convert(((MapType) seaTunnelDataType).getValueType(), attributeValueInfo)); - }); - return seatunnelMap; - } else if (seaTunnelDataType instanceof ArrayType) { - Object array = Array.newInstance(String.class, attributeValue.l().size()); - if (attributeValue.hasL()) { - List datas = attributeValue.l(); - array = Array.newInstance(((ArrayType) seaTunnelDataType).getElementType().getTypeClass(), attributeValue.l().size()); - for (int index = 0; index < datas.size(); index++) { - Array.set(array, index, convert(((ArrayType) seaTunnelDataType).getElementType(), datas.get(index))); - } - } else if (attributeValue.hasSs()) { - List datas = attributeValue.ss(); - for (int index = 0; index < datas.size(); index++) { - Array.set(array, index, AttributeValue.fromS(datas.get(index))); - } - } else if (attributeValue.hasNs()) { - List datas = attributeValue.ns(); - for (int index = 0; index < datas.size(); index++) { - Array.set(array, index, AttributeValue.fromS(datas.get(index))); - } - } else if (attributeValue.hasBs()) { - List datas = attributeValue.bs(); - for (int index = 0; index < datas.size(); index++) { - Array.set(array, index, AttributeValue.fromB(datas.get(index))); - } - } - return array; - } else { - throw new IllegalStateException("Unexpected value: " + seaTunnelDataType); - } - } } From 39414cb7bf9b2fc9617d140a019106554c1850ed Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Mon, 31 Oct 2022 14:49:14 +0800 Subject: [PATCH 17/24] remove unnecessary code --- .../amazondynamodb/sink/DdynamoDbSinkClient.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java index dbeaac4e8b8..75166b69fc2 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java @@ -17,13 +17,5 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; -/** - * Copyright @ 2022科大讯飞。 All rights reserved. - * @Title DdynamoDbSinkClient - * @Project incubator-seatunnel - * @Description TODO - * @author gdliu3 - * @date 2022/10/31 14:21 - */ public class DdynamoDbSinkClient { } From 4e1e17c6b430c76a0e6eb22aa5c6d31441c9def8 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Mon, 31 Oct 2022 19:38:17 +0800 Subject: [PATCH 18/24] add batch write. --- docs/en/connector-v2/sink/Amazondynamodb.md | 20 ++-- docs/en/connector-v2/source/Amazondynamodb.md | 20 ++-- .../config/AmazondynamodbConfig.java | 6 +- .../config/AmazondynamodbSourceOptions.java | 12 ++ .../DefaultSeaTunnelRowDeserializer.java | 6 +- .../sink/AmazondynamodbWriter.java | 24 +--- .../sink/DdynamoDbSinkClient.java | 113 ++++++++++++++++++ .../source/AmazondynamodbSourceReader.java | 2 - .../amazondynamodbIT_source_to_sink.conf | 8 +- 9 files changed, 161 insertions(+), 50 deletions(-) diff --git a/docs/en/connector-v2/sink/Amazondynamodb.md b/docs/en/connector-v2/sink/Amazondynamodb.md index fc0cf748441..5e30613dee2 100644 --- a/docs/en/connector-v2/sink/Amazondynamodb.md +++ b/docs/en/connector-v2/sink/Amazondynamodb.md @@ -17,14 +17,16 @@ Write data to `Amazondynamodb` ## Options -| name | type | required | default value | -|--------------- | ------ |----------| ------------- | -| url | string | yes | - | -| region | string | yes | - | -| accessKeyId | string | yes | - | -| secretAccessKey| string | yes | - | -| table | string | yes | - | -| common-options | | no | - | +| name | type | required | default value | +|----------------- | ------ |----------| ------------- | +| url | string | yes | - | +| region | string | yes | - | +| access_key_id | string | yes | - | +| secret_access_key| string | yes | - | +| table | string | yes | - | +| batch_size | string | no | 25 | +| batch_interval_ms| string | no | 1000 | +| common-options | | no | - | ### url [string] @@ -68,4 +70,4 @@ Amazondynamodb { - Add Amazondynamodb Sink Connector -```` \ No newline at end of file +```` diff --git a/docs/en/connector-v2/source/Amazondynamodb.md b/docs/en/connector-v2/source/Amazondynamodb.md index aa2f38180a7..050a95f951a 100644 --- a/docs/en/connector-v2/source/Amazondynamodb.md +++ b/docs/en/connector-v2/source/Amazondynamodb.md @@ -17,15 +17,15 @@ Read data from Amazondynamodb. ## Options -| name | type | required | default value | -| -------------- | ------ | -------- | ------------- | -| url | string | yes | - | -| region | string | yes | - | -| accessKeyId | string | yes | - | -| secretAccessKey| string | yes | - | -| table | string | yes | - | -| schema | object | yes | - | -| common-options | | yes | - | +| name | type | required | default value | +| ---------------- | ------ | -------- | ------------- | +| url | string | yes | - | +| region | string | yes | - | +| access_key_id | string | yes | - | +| secret_access_key| string | yes | - | +| table | string | yes | - | +| schema | object | yes | - | +| common-options | | yes | - | ### url [string] @@ -105,4 +105,4 @@ Source Plugin common parameters, refer to [Source Plugin](common-options.md) for ### next version -- Add Amazondynamodb Source Connector \ No newline at end of file +- Add Amazondynamodb Source Connector diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java index d615ee3047b..46f5a7f4e80 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbConfig.java @@ -22,7 +22,9 @@ public class AmazondynamodbConfig implements Serializable { public static final String URL = "url"; public static final String REGION = "region"; - public static final String ACCESS_KEY_ID = "accessKeyId"; - public static final String SECRET_ACCESS_KEY = "secretAccessKey"; + public static final String ACCESS_KEY_ID = "access_key_id"; + public static final String SECRET_ACCESS_KEY = "secret_access_key"; public static final String TABLE = "table"; + public static final String BATCH_SIZE = "batch_size"; + public static final String DEFAULT_BATCH_INTERVAL_MS = "batch_interval_ms"; } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java index 9929d84d06b..d04b888f647 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -30,6 +30,9 @@ @AllArgsConstructor public class AmazondynamodbSourceOptions implements Serializable { + private static final int DEFAULT_BATCH_SIZE = 25; + private static final int DEFAULT_BATCH_INTERVAL_MS = 1000; + private String url; private String region; @@ -42,6 +45,9 @@ public class AmazondynamodbSourceOptions implements Serializable { private Config schema; + public int batchSize = DEFAULT_BATCH_SIZE; + public int batchIntervalMs = DEFAULT_BATCH_INTERVAL_MS; + public AmazondynamodbSourceOptions(Config config) { if (config.hasPath(AmazondynamodbConfig.URL)) { this.url = config.getString(AmazondynamodbConfig.URL); @@ -61,5 +67,11 @@ public AmazondynamodbSourceOptions(Config config) { if (config.hasPath(CommonConfig.SCHEMA)) { this.schema = config.getConfig(CommonConfig.SCHEMA); } + if (config.hasPath(AmazondynamodbConfig.BATCH_SIZE)) { + this.batchSize = config.getInt(AmazondynamodbConfig.BATCH_SIZE); + } + if (config.hasPath(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS)) { + this.batchIntervalMs = config.getInt(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS); + } } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java index f990665a90a..e70c1c1285a 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java @@ -27,6 +27,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import lombok.AllArgsConstructor; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; @@ -41,14 +42,11 @@ import java.util.List; import java.util.Map; +@AllArgsConstructor public class DefaultSeaTunnelRowDeserializer implements SeaTunnelRowDeserializer { private final SeaTunnelRowType typeInfo; - public DefaultSeaTunnelRowDeserializer(SeaTunnelRowType typeInfo) { - this.typeInfo = typeInfo; - } - @Override public SeaTunnelRow deserialize(Map item) { SeaTunnelDataType[] seaTunnelDataTypes = typeInfo.getFieldTypes(); diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java index 6b439add386..9f6405790b6 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbWriter.java @@ -24,39 +24,25 @@ import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize.SeaTunnelRowSerializer; import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; - import java.io.IOException; -import java.net.URI; public class AmazondynamodbWriter extends AbstractSinkWriter { - private final DynamoDbClient dynamoDbClient; + private final DdynamoDbSinkClient ddynamoDbSinkClient; private final SeaTunnelRowSerializer serializer; public AmazondynamodbWriter(AmazondynamodbSourceOptions amazondynamodbSourceOptions, SeaTunnelRowType seaTunnelRowType) { - dynamoDbClient = DynamoDbClient.builder() - .endpointOverride(URI.create(amazondynamodbSourceOptions.getUrl())) - // The region is meaningless for local DynamoDb but required for client builder validation - .region(Region.of(amazondynamodbSourceOptions.getRegion())) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create(amazondynamodbSourceOptions.getAccessKeyId(), amazondynamodbSourceOptions.getSecretAccessKey()))) - .build(); + ddynamoDbSinkClient = new DdynamoDbSinkClient(amazondynamodbSourceOptions, seaTunnelRowType); serializer = new DefaultSeaTunnelRowSerializer(seaTunnelRowType, amazondynamodbSourceOptions); } @Override public void write(SeaTunnelRow element) throws IOException { - dynamoDbClient.putItem(serializer.serialize(element)); + ddynamoDbSinkClient.write(serializer.serialize(element)); } @Override - public void close() { - if (dynamoDbClient != null) { - dynamoDbClient.close(); - } + public void close() throws IOException { + ddynamoDbSinkClient.close(); } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java index 75166b69fc2..d9d54919cd1 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java @@ -17,5 +17,118 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize.DefaultSeaTunnelRowDeserializer; +import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.serialize.SeaTunnelRowDeserializer; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.awssdk.services.dynamodb.model.PutRequest; +import software.amazon.awssdk.services.dynamodb.model.WriteRequest; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + public class DdynamoDbSinkClient { + private final AmazondynamodbSourceOptions amazondynamodbSourceOptions; + private ScheduledExecutorService scheduler; + private ScheduledFuture scheduledFuture; + private volatile boolean initialize; + private volatile Exception flushException; + private DynamoDbClient dynamoDbClient; + private final List batchList; + protected SeaTunnelRowDeserializer seaTunnelRowDeserializer; + + public DdynamoDbSinkClient(AmazondynamodbSourceOptions amazondynamodbSourceOptions, SeaTunnelRowType typeInfo) { + this.amazondynamodbSourceOptions = amazondynamodbSourceOptions; + this.batchList = new ArrayList<>(); + this.seaTunnelRowDeserializer = new DefaultSeaTunnelRowDeserializer(typeInfo); + } + + private void tryInit() throws IOException { + if (initialize) { + return; + } + dynamoDbClient = DynamoDbClient.builder() + .endpointOverride(URI.create(amazondynamodbSourceOptions.getUrl())) + // The region is meaningless for local DynamoDb but required for client builder validation + .region(Region.of(amazondynamodbSourceOptions.getRegion())) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(amazondynamodbSourceOptions.getAccessKeyId(), amazondynamodbSourceOptions.getSecretAccessKey()))) + .build(); + + scheduler = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("DdynamoDb-sink-output-%s").build()); + scheduledFuture = scheduler.scheduleAtFixedRate( + () -> { + try { + flush(); + } catch (IOException e) { + flushException = e; + } + }, + amazondynamodbSourceOptions.getBatchIntervalMs(), + amazondynamodbSourceOptions.getBatchIntervalMs(), + TimeUnit.MILLISECONDS); + + initialize = true; + } + + public synchronized void write(PutItemRequest putItemRequest) throws IOException { + tryInit(); + checkFlushException(); + batchList.add(WriteRequest.builder().putRequest( + PutRequest.builder().item(putItemRequest.item()).build()).build()); + if (amazondynamodbSourceOptions.getBatchSize() > 0 + && batchList.size() >= amazondynamodbSourceOptions.getBatchSize()) { + flush(); + } + } + + public synchronized void close() throws IOException { + if (scheduledFuture != null) { + scheduledFuture.cancel(false); + scheduler.shutdown(); + } + flush(); + if (dynamoDbClient != null) { + dynamoDbClient.close(); + } + } + + synchronized void flush() throws IOException { + checkFlushException(); + if (batchList.isEmpty()) { + return; + } + Map> requestItems = new HashMap<>(1); + requestItems.put(amazondynamodbSourceOptions.getTable(), batchList); + dynamoDbClient.batchWriteItem(BatchWriteItemRequest + .builder() + .requestItems(requestItems) + .build()); + + batchList.clear(); + } + + private void checkFlushException() { + if (flushException != null) { + throw new RuntimeException("Writing items to DdynamoDb failed.", flushException); + } + } + } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java index b4d4575cb50..fe8fd47bb7d 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSourceReader.java @@ -43,7 +43,6 @@ public class AmazondynamodbSourceReader extends AbstractSingleSplitReader Date: Tue, 1 Nov 2022 12:01:01 +0800 Subject: [PATCH 19/24] modify sink doc --- docs/en/connector-v2/sink/Amazondynamodb.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/en/connector-v2/sink/Amazondynamodb.md b/docs/en/connector-v2/sink/Amazondynamodb.md index 5e30613dee2..47f003d6a49 100644 --- a/docs/en/connector-v2/sink/Amazondynamodb.md +++ b/docs/en/connector-v2/sink/Amazondynamodb.md @@ -1,4 +1,4 @@ -```` + # Amazondynamodb > Amazondynamodb sink connector @@ -9,11 +9,8 @@ Write data to `Amazondynamodb` ## Key features -- [x] [batch](../../concept/connector-v2-features.md) - [ ] [exactly-once](../../concept/connector-v2-features.md) - [ ] [schema projection](../../concept/connector-v2-features.md) -- [ ] [parallelism](../../concept/connector-v2-features.md) -- [ ] [support user-defined split](../../concept/connector-v2-features.md) ## Options @@ -70,4 +67,3 @@ Amazondynamodb { - Add Amazondynamodb Sink Connector -```` From aaf3d1d08c56b3cfe73da2ccc081931bca51c424 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Wed, 2 Nov 2022 18:05:41 +0800 Subject: [PATCH 20/24] fix cv bug. --- .../amazondynamodb/sink/AmazondynamodbSink.java | 4 +++- .../amazondynamodb/sink/DdynamoDbSinkClient.java | 2 +- .../source/AmazondynamodbSource.java | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java index 1dd43665344..3d196f92bd8 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/AmazondynamodbSink.java @@ -17,7 +17,9 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.sink; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.ACCESS_KEY_ID; import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.REGION; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.SECRET_ACCESS_KEY; import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.TABLE; import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.URL; @@ -54,7 +56,7 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { - CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, URL, TABLE, REGION); + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, URL, TABLE, REGION, ACCESS_KEY_ID, SECRET_ACCESS_KEY); if (!result.isSuccess()) { throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java index d9d54919cd1..8066261aeb5 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/sink/DdynamoDbSinkClient.java @@ -104,8 +104,8 @@ public synchronized void close() throws IOException { scheduledFuture.cancel(false); scheduler.shutdown(); } - flush(); if (dynamoDbClient != null) { + flush(); dynamoDbClient.close(); } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java index b19854777f4..a770d0f4300 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java @@ -17,12 +17,22 @@ package org.apache.seatunnel.connectors.seatunnel.amazondynamodb.source; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.ACCESS_KEY_ID; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.REGION; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.SECRET_ACCESS_KEY; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.TABLE; +import static org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbConfig.URL; +import static org.apache.seatunnel.connectors.seatunnel.common.config.CommonConfig.SCHEMA; + import org.apache.seatunnel.api.common.PrepareFailException; import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.PluginType; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazondynamodbSourceOptions; import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; @@ -49,6 +59,10 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, URL, TABLE, REGION, ACCESS_KEY_ID, SECRET_ACCESS_KEY, SCHEMA); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); + } amazondynamodbSourceOptions = new AmazondynamodbSourceOptions(pluginConfig); Config schema = amazondynamodbSourceOptions.getSchema(); typeInfo = SeaTunnelSchema.buildWithConfig(schema).getSeaTunnelRowType(); From 5a7ff67e07c5e308cd8a6946958e2849d0ff4257 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Wed, 2 Nov 2022 22:15:32 +0800 Subject: [PATCH 21/24] fix some error --- .../config/AmazondynamodbSourceOptions.java | 37 ++++++------------- .../source/AmazondynamodbSource.java | 3 +- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java index d04b888f647..10843fe1984 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -45,33 +45,18 @@ public class AmazondynamodbSourceOptions implements Serializable { private Config schema; - public int batchSize = DEFAULT_BATCH_SIZE; - public int batchIntervalMs = DEFAULT_BATCH_INTERVAL_MS; + private int batchSize; + + private int batchIntervalMs; public AmazondynamodbSourceOptions(Config config) { - if (config.hasPath(AmazondynamodbConfig.URL)) { - this.url = config.getString(AmazondynamodbConfig.URL); - } - if (config.hasPath(AmazondynamodbConfig.REGION)) { - this.region = config.getString(AmazondynamodbConfig.REGION); - } - if (config.hasPath(AmazondynamodbConfig.ACCESS_KEY_ID)) { - this.accessKeyId = config.getString(AmazondynamodbConfig.ACCESS_KEY_ID); - } - if (config.hasPath(AmazondynamodbConfig.SECRET_ACCESS_KEY)) { - this.secretAccessKey = config.getString(AmazondynamodbConfig.SECRET_ACCESS_KEY); - } - if (config.hasPath(AmazondynamodbConfig.TABLE)) { - this.table = config.getString(AmazondynamodbConfig.TABLE); - } - if (config.hasPath(CommonConfig.SCHEMA)) { - this.schema = config.getConfig(CommonConfig.SCHEMA); - } - if (config.hasPath(AmazondynamodbConfig.BATCH_SIZE)) { - this.batchSize = config.getInt(AmazondynamodbConfig.BATCH_SIZE); - } - if (config.hasPath(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS)) { - this.batchIntervalMs = config.getInt(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS); - } + this.url = config.getString(AmazondynamodbConfig.URL); + this.region = config.getString(AmazondynamodbConfig.REGION); + this.accessKeyId = config.getString(AmazondynamodbConfig.ACCESS_KEY_ID); + this.secretAccessKey = config.getString(AmazondynamodbConfig.SECRET_ACCESS_KEY); + this.table = config.getString(AmazondynamodbConfig.TABLE); + this.schema = config.getConfig(CommonConfig.SCHEMA); + this.batchSize = config.getInt(AmazondynamodbConfig.BATCH_SIZE); + this.batchIntervalMs = config.getInt(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS); } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java index a770d0f4300..40ac9eeef34 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/source/AmazondynamodbSource.java @@ -64,8 +64,7 @@ public void prepare(Config pluginConfig) throws PrepareFailException { throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); } amazondynamodbSourceOptions = new AmazondynamodbSourceOptions(pluginConfig); - Config schema = amazondynamodbSourceOptions.getSchema(); - typeInfo = SeaTunnelSchema.buildWithConfig(schema).getSeaTunnelRowType(); + typeInfo = SeaTunnelSchema.buildWithConfig(amazondynamodbSourceOptions.getSchema()).getSeaTunnelRowType(); } @Override From b26981ab5d5f2e8059aa62f4333be39ff616af00 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Thu, 3 Nov 2022 11:30:41 +0800 Subject: [PATCH 22/24] fix bug. --- .../config/AmazondynamodbSourceOptions.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java index 10843fe1984..b38ec648dfb 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -45,9 +45,8 @@ public class AmazondynamodbSourceOptions implements Serializable { private Config schema; - private int batchSize; - - private int batchIntervalMs; + public int batchSize = DEFAULT_BATCH_SIZE; + public int batchIntervalMs = DEFAULT_BATCH_INTERVAL_MS; public AmazondynamodbSourceOptions(Config config) { this.url = config.getString(AmazondynamodbConfig.URL); @@ -56,7 +55,12 @@ public AmazondynamodbSourceOptions(Config config) { this.secretAccessKey = config.getString(AmazondynamodbConfig.SECRET_ACCESS_KEY); this.table = config.getString(AmazondynamodbConfig.TABLE); this.schema = config.getConfig(CommonConfig.SCHEMA); - this.batchSize = config.getInt(AmazondynamodbConfig.BATCH_SIZE); - this.batchIntervalMs = config.getInt(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS); + + if (config.hasPath(AmazondynamodbConfig.BATCH_SIZE)) { + this.batchSize = config.getInt(AmazondynamodbConfig.BATCH_SIZE); + } + if (config.hasPath(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS)) { + this.batchIntervalMs = config.getInt(AmazondynamodbConfig.DEFAULT_BATCH_INTERVAL_MS); + } } } From 1e6e186d0bca4a198941192af969f7f5538065f8 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Thu, 3 Nov 2022 11:40:35 +0800 Subject: [PATCH 23/24] fix bug. --- .../amazondynamodb/config/AmazondynamodbSourceOptions.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java index b38ec648dfb..0d0eec4a4b1 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/config/AmazondynamodbSourceOptions.java @@ -54,8 +54,10 @@ public AmazondynamodbSourceOptions(Config config) { this.accessKeyId = config.getString(AmazondynamodbConfig.ACCESS_KEY_ID); this.secretAccessKey = config.getString(AmazondynamodbConfig.SECRET_ACCESS_KEY); this.table = config.getString(AmazondynamodbConfig.TABLE); - this.schema = config.getConfig(CommonConfig.SCHEMA); + if (config.hasPath(CommonConfig.SCHEMA)) { + this.schema = config.getConfig(CommonConfig.SCHEMA); + } if (config.hasPath(AmazondynamodbConfig.BATCH_SIZE)) { this.batchSize = config.getInt(AmazondynamodbConfig.BATCH_SIZE); } From fca481e130433c55964c91faecdb811f481adfb0 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Thu, 3 Nov 2022 23:05:21 +0800 Subject: [PATCH 24/24] fix ci error --- tools/update_modules_check/update_modules_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py index 59aaccfee66..0600b82f3db 100644 --- a/tools/update_modules_check/update_modules_check.py +++ b/tools/update_modules_check/update_modules_check.py @@ -40,13 +40,13 @@ def get_modules(files, index, start_pre, root_module): update_files = json.loads(files) modules_name_set = set([]) for file in update_files: - module_name = file.split('/')[index] + names = file.split('/') + module_name = names[index] if module_name.startswith(start_pre): modules_name_set.add(module_name) - sub_module_name = file.split('/')[index + 1] - if sub_module_name.startswith(start_pre): - modules_name_set.add(sub_module_name) + if len(names) > index + 1 and names[index + 1].startswith(start_pre): + modules_name_set.add(names[index + 1]) output_module = "" if len(modules_name_set) > 0: @@ -135,4 +135,4 @@ def main(argv): get_sub_modules(argv[2]) if __name__ == "__main__": - main(sys.argv) \ No newline at end of file + main(sys.argv)