From 3bdbdcf8a400936bed0a0bee5acbff9e32a60e62 Mon Sep 17 00:00:00 2001 From: William Wen Date: Wed, 7 Jun 2023 16:43:15 -0700 Subject: [PATCH 1/4] feat(java-binding): bundle jni library to jar --- ci/scripts/build.sh | 4 +- ci/scripts/common.sh | 20 ------ ci/scripts/e2e-iceberg-sink-test.sh | 4 -- ci/scripts/e2e-sink-test.sh | 5 -- ci/scripts/e2e-source-test.sh | 2 - ci/scripts/java-binding-test.sh | 4 +- docker/Dockerfile | 8 +-- docker/Dockerfile.hdfs | 8 +-- java/connector-node/README.md | 5 +- .../assembly/scripts/start-service.sh | 2 +- java/java-binding/pom.xml | 72 ++++++++----------- .../com/risingwave/java/binding/Binding.java | 4 +- src/compute/src/server.rs | 4 +- src/java_binding/Cargo.toml | 2 +- src/java_binding/make-java-binding.toml | 20 ++---- src/java_binding/run_demo.sh | 2 +- 16 files changed, 48 insertions(+), 118 deletions(-) diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 79105a1259b50..716911fa81af4 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -52,13 +52,11 @@ cargo build \ -p risingwave_sqlsmith \ -p risingwave_compaction_test \ -p risingwave_backup_cmd \ - -p risingwave_java_binding \ -p risingwave_e2e_extended_mode_test \ $RISINGWAVE_FEATURE_FLAGS \ --profile "$profile" -# the file name suffix of artifact for risingwave_java_binding is so only for linux. It is dylib for MacOS -artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test librisingwave_java_binding.so) +artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test) echo "--- Show link info" ldd target/"$profile"/risingwave diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh index f1ad1304b47a3..716ba6a51ef7b 100644 --- a/ci/scripts/common.sh +++ b/ci/scripts/common.sh @@ -84,23 +84,3 @@ function download_and_prepare_rw() { cargo make pre-start-dev cargo make link-all-in-one-binaries } - -# Arguments: -# $1: cargo build `profile` of the binaries -function download_java_binding() { - echo "--- Download java binding" - if [ -z "$1" ]; then - echo "download_java_binding: missing argument profile" - exit 1 - fi - - profile=$1 - - echo -e "\033[33mDownload artifacts\033[0m" - - mkdir -p target/debug - download-and-decompress-artifact librisingwave_java_binding.so-"$profile" target/debug - mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so - - export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug -} diff --git a/ci/scripts/e2e-iceberg-sink-test.sh b/ci/scripts/e2e-iceberg-sink-test.sh index 1b366cc9e0a56..d9223b51d7d09 100755 --- a/ci/scripts/e2e-iceberg-sink-test.sh +++ b/ci/scripts/e2e-iceberg-sink-test.sh @@ -23,10 +23,6 @@ shift $((OPTIND -1)) download_and_prepare_rw "$profile" source -download_java_binding "$profile" - -export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk - echo "--- Download connector node package" buildkite-agent artifact download risingwave-connector.tar.gz ./ mkdir ./connector-node diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh index 8ca8e5a1768f4..2ec1373162ee0 100755 --- a/ci/scripts/e2e-sink-test.sh +++ b/ci/scripts/e2e-sink-test.sh @@ -23,11 +23,6 @@ shift $((OPTIND -1)) download_and_prepare_rw "$profile" source -download_java_binding "$profile" - -# TODO: Switch to stream_chunk encoding once it's completed, and then remove json encoding as well as this env var. -export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk - # Change process number limit echo "--- os limits" ulimit -a diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 987412560f9d2..032a4c3442318 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -26,8 +26,6 @@ shift $((OPTIND -1)) download_and_prepare_rw "$profile" source -download_java_binding "$profile" - echo "--- Download connector node package" buildkite-agent artifact download risingwave-connector.tar.gz ./ mkdir ./connector-node diff --git a/ci/scripts/java-binding-test.sh b/ci/scripts/java-binding-test.sh index 878a8b779364f..efc47bf3a4013 100755 --- a/ci/scripts/java-binding-test.sh +++ b/ci/scripts/java-binding-test.sh @@ -23,13 +23,11 @@ shift $((OPTIND -1)) download_and_prepare_rw "$profile" source -download_java_binding "$profile" - echo "--- starting risingwave cluster" cargo make ci-start java-binding-demo echo "--- Build java binding demo" -cargo make build-java-binding-java +cargo make build-java-binding echo "--- ingest data and run java binding" cargo make ingest-data-and-run-java-binding diff --git a/docker/Dockerfile b/docker/Dockerfile index 3f1864da12588..3c388d3190ca6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,10 +34,9 @@ RUN rustup self update \ RUN cargo fetch -RUN cargo build -p risingwave_cmd_all -p risingwave_java_binding --release --features "rw-static-link" && \ +RUN cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \ mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \ - mkdir -p /risingwave/lib && mv /risingwave/target/release/librisingwave_java_binding.so /risingwave/lib && \ - cargo clean + mkdir -p /risingwave/lib && cargo clean RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \ mkdir -p /risingwave/bin/connector-node && \ @@ -52,9 +51,6 @@ RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node COPY --from=builder /risingwave/ui /risingwave/ui -COPY --from=builder /risingwave/lib/librisingwave_java_binding.so /risingwave/lib/librisingwave_java_binding.so -# Set java.library.path env to /risingwave/lib -ENV RW_JAVA_BINDING_LIB_PATH /risingwave/lib # Set default playground mode to docker-playground profile ENV PLAYGROUND_PROFILE docker-playground # Set default dashboard UI to local path instead of github proxy diff --git a/docker/Dockerfile.hdfs b/docker/Dockerfile.hdfs index c81fd051a5db9..002da91405c9d 100644 --- a/docker/Dockerfile.hdfs +++ b/docker/Dockerfile.hdfs @@ -39,10 +39,9 @@ ARG JAVA_HOME_PATH ENV JAVA_HOME ${JAVA_HOME_PATH} ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH} -RUN cargo build -p risingwave_cmd_all -p risingwave_java_binding --release --features "static-link static-log-level" && \ +RUN cargo build -p risingwave_cmd_all --release --features "static-link static-log-level" && \ mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \ - mkdir -p /risingwave/lib && mv /risingwave/target/release/librisingwave_java_binding.so /risingwave/lib && \ - cargo clean + mkdir -p /risingwave/lib && cargo clean RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \ mkdir -p /risingwave/bin/connector-node && \ @@ -57,7 +56,6 @@ RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node COPY --from=builder /risingwave/ui /risingwave/ui -COPY --from=builder /risingwave/lib/librisingwave_java_binding.so /risingwave/lib/librisingwave_java_binding.so # hadoop RUN wget https://rw-yufan.s3.ap-southeast-1.amazonaws.com/hadoop-2.7.3.tar.gz -P /root/ @@ -75,8 +73,6 @@ ARG JAVA_HOME_PATH ENV JAVA_HOME ${JAVA_HOME_PATH} ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH} -# Set java.library.path env to /risingwave/lib -ENV RW_JAVA_BINDING_LIB_PATH /risingwave/lib # Set default playground mode to docker-playground profile ENV PLAYGROUND_PROFILE docker-playground # Set default dashboard UI to local path instead of github proxy diff --git a/java/connector-node/README.md b/java/connector-node/README.md index ab2201c3770d6..c146f92492da8 100644 --- a/java/connector-node/README.md +++ b/java/connector-node/README.md @@ -35,10 +35,7 @@ Sometimes, you need to specify the shared library path. For example, when progra ``` RISINGWAVE_ROOT=$(git rev-parse --show-toplevel) CONNECTOR_ROOT=$JAVA_ROOT/connector-node -# Build shared library file -cd $RISINGWAVE_ROOT && cargo build -p risingwave_java_binding -# specify the Djava.library.path, please make sure the shared library you needed exist in /target/debug -cd ${CONNECTOR_ROOT}/assembly/target && java -classpath "./libs/*" -Djava.library.path=${RISINGWAVE_ROOT}/target/debug com.risingwave.connector.ConnectorService +cd ${CONNECTOR_ROOT}/assembly/target && java -classpath "./libs/*" com.risingwave.connector.ConnectorService ``` ## Docker image diff --git a/java/connector-node/assembly/scripts/start-service.sh b/java/connector-node/assembly/scripts/start-service.sh index 86b59386c43a3..2560a3ef437c5 100755 --- a/java/connector-node/assembly/scripts/start-service.sh +++ b/java/connector-node/assembly/scripts/start-service.sh @@ -25,4 +25,4 @@ if [ -z "${port}" ]; then port=$PORT fi -java -classpath "${DIR}/libs/*" -Djava.library.path="${RW_JAVA_BINDING_LIB_PATH}" $MAIN --port ${port} +java -classpath "${DIR}/libs/*" $MAIN --port ${port} diff --git a/java/java-binding/pom.xml b/java/java-binding/pom.xml index e410dc2d792a7..67a7746523e7e 100644 --- a/java/java-binding/pom.xml +++ b/java/java-binding/pom.xml @@ -11,15 +11,21 @@ java-binding - + UTF-8 11 11 1.0-SNAPSHOT + 1.0.0 + + org.questdb + jar-jni + ${jni.loader.version} + com.risingwave.java proto @@ -37,48 +43,26 @@ - - - - - maven-clean-plugin - 3.1.0 - - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - - maven-surefire-plugin - 2.22.1 - - - maven-jar-plugin - 3.0.2 - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - - maven-site-plugin - 3.7.1 - - - maven-project-info-reports-plugin - 3.0.0 - - - + + + org.questdb + rust-maven-plugin + ${jni.loader.version} + + + str-reverse + + build + + + ../../src/java_binding + false + ${project.build.directory}/classes/risingwave/jni + true + + + + + diff --git a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java index be6c521e5baf8..04dbb444c3673 100644 --- a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java +++ b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java @@ -14,9 +14,11 @@ package com.risingwave.java.binding; +import io.questdb.jar.jni.JarJniLoader; + public class Binding { static { - System.loadLibrary("risingwave_java_binding"); + JarJniLoader.loadLib(Binding.class, "/risingwave/jni", "risingwave_java_binding"); } public static native int vnodeCount(); diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index 200a3c60afa77..2b5691f5982d7 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -322,8 +322,8 @@ pub async fn compute_node_serve( let connector_params = risingwave_connector::ConnectorParams { connector_rpc_endpoint: opts.connector_rpc_endpoint, sink_payload_format: match opts.connector_rpc_sink_payload_format.as_deref() { - None | Some("json") => SinkPayloadFormat::Json, - Some("stream_chunk") => SinkPayloadFormat::StreamChunk, + None | Some("stream_chunk") => SinkPayloadFormat::StreamChunk, + Some("json") => SinkPayloadFormat::Json, _ => { unreachable!( "invalid sink payload format: {:?}. Should be either json or stream_chunk", diff --git a/src/java_binding/Cargo.toml b/src/java_binding/Cargo.toml index 68bc4b554ab5e..f00f4ce16600d 100644 --- a/src/java_binding/Cargo.toml +++ b/src/java_binding/Cargo.toml @@ -37,7 +37,7 @@ tracing = "0.1" risingwave_expr = { path = "../expr" } [lib] -crate_type = ["cdylib"] +crate-type = ["cdylib"] [[bin]] name = "data-chunk-payload-generator" diff --git a/src/java_binding/make-java-binding.toml b/src/java_binding/make-java-binding.toml index 796fa12debbc9..9ec0bd3935fa7 100644 --- a/src/java_binding/make-java-binding.toml +++ b/src/java_binding/make-java-binding.toml @@ -7,21 +7,13 @@ cd java javac -h . -cp java-binding/src/main/java/ java-binding/src/main/java/com/risingwave/java/binding/Binding.java ''' -[tasks.build-java-binding-rust] -description = "Build the java binding rust code" -script = ''' -#!/usr/bin/env bash -set -ex -cargo build -p risingwave_java_binding -''' - -[tasks.build-java-binding-java] +[tasks.build-java-binding] description = "Build the java binding java code" script = ''' #!/usr/bin/env bash set -ex cd java -mvn clean install --no-transfer-progress --pl java-binding-integration-test --am -DskipTests=true +mvn install --no-transfer-progress --pl java-binding-integration-test --am -DskipTests=true mvn dependency:copy-dependencies --no-transfer-progress --pl java-binding-integration-test ''' @@ -71,8 +63,7 @@ bash ${RUST_JAVA_BINDING_ROOT}/run_demo.sh [tasks.run-java-binding-demo] description = "Run the java binding demo" dependencies = [ - "build-java-binding-rust", - "build-java-binding-java", + "build-java-binding", "start-java-binding-demo-cluster", "ingest-data-and-run-java-binding", "kill-java-binding-demo-cluster" @@ -81,8 +72,7 @@ dependencies = [ [tasks.run-java-binding-stream-chunk-demo] description = "Run the java binding stream chunk demo" dependencies = [ - "build-java-binding-rust", - "build-java-binding-java", + "build-java-binding", ] script = ''' #!/usr/bin/env bash @@ -94,7 +84,7 @@ cd ${RISINGWAVE_ROOT}/java (cd ${RISINGWAVE_ROOT} && cargo run --bin data-chunk-payload-generator) | \ java -cp "./java-binding-integration-test/target/dependency/*:./java-binding-integration-test/target/classes" \ - -Djava.library.path=${RISINGWAVE_ROOT}/target/debug com.risingwave.java.binding.StreamChunkDemo + com.risingwave.java.binding.StreamChunkDemo ''' diff --git a/src/java_binding/run_demo.sh b/src/java_binding/run_demo.sh index 9c7fa0fd8158f..67cd474050eaf 100644 --- a/src/java_binding/run_demo.sh +++ b/src/java_binding/run_demo.sh @@ -20,7 +20,7 @@ set -x cd ${RISINGWAVE_ROOT}/java java -cp "./java-binding-integration-test/target/dependency/*:./java-binding-integration-test/target/classes" \ - -Djava.library.path=${RISINGWAVE_ROOT}/target/debug com.risingwave.java.binding.HummockReadDemo + com.risingwave.java.binding.HummockReadDemo psql -d dev -h localhost -p 4566 -U root << EOF DROP TABLE ${TABLE_NAME}; From 480ee4a49b89272fdf79721b79e1411736387195 Mon Sep 17 00:00:00 2001 From: William Wen Date: Wed, 7 Jun 2023 17:26:49 -0700 Subject: [PATCH 2/4] disable building rust in connector node integration test --- .../workflows/connector-node-integration.yml | 2 +- java/java-binding/pom.xml | 24 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/connector-node-integration.yml b/.github/workflows/connector-node-integration.yml index 6e40536532087..b9cf0eb86e5d4 100644 --- a/.github/workflows/connector-node-integration.yml +++ b/.github/workflows/connector-node-integration.yml @@ -41,7 +41,7 @@ jobs: echo "--- build connector node" cd ${RISINGWAVE_ROOT}/java - mvn --batch-mode --update-snapshots clean package + mvn --batch-mode --update-snapshots clean package -P no-build-jni-rust echo "--- install postgresql client" sudo apt install postgresql postgresql-contrib libpq-dev diff --git a/java/java-binding/pom.xml b/java/java-binding/pom.xml index 67a7746523e7e..7b07d2bda8af4 100644 --- a/java/java-binding/pom.xml +++ b/java/java-binding/pom.xml @@ -42,6 +42,28 @@ + + + no-build-jni-rust + + + + org.questdb + rust-maven-plugin + ${jni.loader.version} + + + build-jni-rust + + none + + + + + + + + @@ -50,7 +72,7 @@ ${jni.loader.version} - str-reverse + build-jni-rust build From 7680ca4864a746c032b42c1686d25afd59f78f4a Mon Sep 17 00:00:00 2001 From: William Wen Date: Thu, 8 Jun 2023 11:36:15 -0700 Subject: [PATCH 3/4] disable building rust via setting property --- .github/workflows/connector-node-integration.yml | 2 +- java/java-binding/pom.xml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/connector-node-integration.yml b/.github/workflows/connector-node-integration.yml index b9cf0eb86e5d4..ad3bdfeafdf63 100644 --- a/.github/workflows/connector-node-integration.yml +++ b/.github/workflows/connector-node-integration.yml @@ -41,7 +41,7 @@ jobs: echo "--- build connector node" cd ${RISINGWAVE_ROOT}/java - mvn --batch-mode --update-snapshots clean package -P no-build-jni-rust + mvn --batch-mode --update-snapshots clean package -Dno-build-rust echo "--- install postgresql client" sudo apt install postgresql postgresql-contrib libpq-dev diff --git a/java/java-binding/pom.xml b/java/java-binding/pom.xml index 7b07d2bda8af4..7a7af9dbfd937 100644 --- a/java/java-binding/pom.xml +++ b/java/java-binding/pom.xml @@ -45,6 +45,11 @@ no-build-jni-rust + + + no-build-rust + + From 01e4ceca52c3836e7b6ffd2f0a0397f02ff519fe Mon Sep 17 00:00:00 2001 From: William Wen Date: Thu, 15 Jun 2023 16:09:42 +0800 Subject: [PATCH 4/4] add property to enable release build --- ci/scripts/release.sh | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.hdfs | 2 +- java/java-binding/pom.xml | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index c5af7aaab1228..4b3ed00ad0d94 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -83,7 +83,7 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz echo "--- Release build and upload risingwave connector node jar asset" - cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true + cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true cd connector-node/assembly/target && mv risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz fi diff --git a/docker/Dockerfile b/docker/Dockerfile index 3c388d3190ca6..2da3e6e072f9e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,7 @@ RUN cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \ mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \ mkdir -p /risingwave/lib && cargo clean -RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \ +RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \ mkdir -p /risingwave/bin/connector-node && \ tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node diff --git a/docker/Dockerfile.hdfs b/docker/Dockerfile.hdfs index 002da91405c9d..15b959d6953a9 100644 --- a/docker/Dockerfile.hdfs +++ b/docker/Dockerfile.hdfs @@ -43,7 +43,7 @@ RUN cargo build -p risingwave_cmd_all --release --features "static-link static-l mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \ mkdir -p /risingwave/lib && cargo clean -RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \ +RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \ mkdir -p /risingwave/bin/connector-node && \ tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node diff --git a/java/java-binding/pom.xml b/java/java-binding/pom.xml index 7a7af9dbfd937..a39288ca9e6c8 100644 --- a/java/java-binding/pom.xml +++ b/java/java-binding/pom.xml @@ -18,6 +18,7 @@ 11 1.0-SNAPSHOT 1.0.0 + false @@ -83,7 +84,7 @@ ../../src/java_binding - false + ${java.binding.release} ${project.build.directory}/classes/risingwave/jni true