Skip to content

Commit

Permalink
Post Cherry Pick: Add gradle and other settings
Browse files Browse the repository at this point in the history
  • Loading branch information
ismailsimsek committed Jan 15, 2025
1 parent 47534b8 commit 9e64c5b
Show file tree
Hide file tree
Showing 6 changed files with 45 additions and 17 deletions.
1 change: 1 addition & 0 deletions .github/workflows/kafka-connect-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ jobs:
- run: |
./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions= -DkafkaVersions=3 \
:iceberg-kafka-connect:iceberg-kafka-connect-events:check \
:iceberg-kafka-connect:iceberg-kafka-connect-transforms:check \
:iceberg-kafka-connect:iceberg-kafka-connect:check \
:iceberg-kafka-connect:iceberg-kafka-connect-runtime:check \
-Pquick=true -x javadoc
Expand Down
3 changes: 3 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ awaitility = "4.2.2"
awssdk-bom = "2.29.43"
azuresdk-bom = "1.2.30"
awssdk-s3accessgrants = "2.3.0"
bson-ver = "4.11.0"
caffeine = "2.9.3"
calcite = "1.10.0"
datasketches = "6.1.1"
Expand Down Expand Up @@ -100,6 +101,7 @@ avro-avro = { module = "org.apache.avro:avro", version.ref = "avro" }
awssdk-bom = { module = "software.amazon.awssdk:bom", version.ref = "awssdk-bom" }
awssdk-s3accessgrants = { module = "software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin", version.ref = "awssdk-s3accessgrants" }
azuresdk-bom = { module = "com.azure:azure-sdk-bom", version.ref = "azuresdk-bom" }
bson = { module = "org.mongodb:bson", version.ref = "bson-ver"}
caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" }
calcite-core = { module = "org.apache.calcite:calcite-core", version.ref = "calcite" }
calcite-druid = { module = "org.apache.calcite:calcite-druid", version.ref = "calcite" }
Expand Down Expand Up @@ -159,6 +161,7 @@ jaxb-runtime = { module = "org.glassfish.jaxb:jaxb-runtime", version.ref = "jaxb
kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka" }
kafka-connect-api = { module = "org.apache.kafka:connect-api", version.ref = "kafka" }
kafka-connect-json = { module = "org.apache.kafka:connect-json", version.ref = "kafka" }
kafka-connect-transforms = { module = "org.apache.kafka:connect-transforms", version.ref = "kafka" }
microprofile-openapi-api = { module = "org.eclipse.microprofile.openapi:microprofile-openapi-api", version.ref = "microprofile-openapi-api" }
nessie-client = { module = "org.projectnessie.nessie:nessie-client", version.ref = "nessie" }
netty-buffer = { module = "io.netty:netty-buffer", version.ref = "netty-buffer" }
Expand Down
17 changes: 17 additions & 0 deletions kafka-connect/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {

dependencies {
implementation project(':iceberg-kafka-connect:iceberg-kafka-connect')
implementation project(':iceberg-kafka-connect:iceberg-kafka-connect-transforms')
implementation(libs.hadoop3.common) {
exclude group: 'log4j'
exclude group: 'org.slf4j'
Expand Down Expand Up @@ -242,3 +243,19 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {

assemble.dependsOn distZip, hiveDistZip
}

project(':iceberg-kafka-connect:iceberg-kafka-connect-transforms') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
implementation libs.bson
// implementation libs.slf4j //TODO DISABLED! do we need this??
compileOnly libs.kafka.clients
compileOnly libs.kafka.connect.api
compileOnly libs.kafka.connect.json
compileOnly libs.kafka.connect.transforms
}

test {
useJUnitPlatform()
}
}
21 changes: 20 additions & 1 deletion kafka-connect/kafka-connect-transforms/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,22 @@
<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-->

# SMTs for the Apache Iceberg Sink Connector

This project contains some SMTs that could be useful when transforming Kafka data for use by
Expand All @@ -19,7 +38,7 @@ The `CopyValue` SMT copies a value from one field to a new field.

```
"transforms": "copyId",
"transforms.copyId.type": "io.tabular.iceberg.connect.transforms.CopyValue",
"transforms.copyId.type": "org.apache.iceberg.connect.transforms.CopyValue",
"transforms.copyId.source.field": "id",
"transforms.copyId.target.field": "id_copy",
```
Expand Down
16 changes: 0 additions & 16 deletions kafka-connect/kafka-connect-transforms/build.gradle

This file was deleted.

4 changes: 4 additions & 0 deletions settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -211,4 +211,8 @@ if (kafkaVersions.contains("3")) {
include ":iceberg-kafka-connect:kafka-connect-runtime"
project(":iceberg-kafka-connect:kafka-connect-runtime").projectDir = file('kafka-connect/kafka-connect-runtime')
project(":iceberg-kafka-connect:kafka-connect-runtime").name = "iceberg-kafka-connect-runtime"

include ":iceberg-kafka-connect:kafka-connect-transforms"
project(":iceberg-kafka-connect:kafka-connect-transforms").projectDir = file('kafka-connect/kafka-connect-transforms')
project(":iceberg-kafka-connect:kafka-connect-transforms").name = "iceberg-kafka-connect-transforms"
}

0 comments on commit 9e64c5b

Please sign in to comment.