Merge branch 'main' into jon/stream-hop-window-frontend-derive

ci/scripts/docker.sh

@@ -6,11 +6,6 @@ set -euo pipefail
 ghcraddr="ghcr.io/risingwavelabs/risingwave"
 dockerhubaddr="risingwavelabs/risingwave"
 arch="$(uname -m)"
-connector_node_version=$(cat ci/connector-node-version)
-
-# Git clone risingwave-connector-node repo
-git clone https://"$GITHUB_TOKEN"@github.com/risingwavelabs/risingwave-connector-node.git
-cd risingwave-connector-node && git checkout ${connector_node_version} && cd ..
 
 # Build RisingWave docker image ${BUILDKITE_COMMIT}-${arch}
 echo "--- docker build and tag"

ci/scripts/release.sh

@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-connector_node_version=$(cat ci/connector-node-version)
+REPO_ROOT=${PWD}
 
 echo "--- Check env"
 if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhook" ] && [[ -z "${BINARY_NAME+x}" ]]; then
@@ -63,10 +63,9 @@ if [[ -n "${BUILDKITE_TAG+x}" ]]; then
   gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz
 
   echo "--- Release build and upload risingwave connector node jar asset"
-#  git clone https://"$GITHUB_TOKEN"@github.com/risingwavelabs/risingwave-connector-node.git
-#  cd risingwave-connector-node && git checkout ${connector_node_version} && mvn -B package -Dmaven.test.skip=true
-#  cd assembly/target && mv risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz
-#  gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz
+  cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true
+  cd connector-node/assembly/target && mv risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz
+  gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz
 fi
 
 

ci/scripts/s3-source-test.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+
+while getopts 'p:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+echo "--- Download artifacts"
+mkdir -p target/debug
+buildkite-agent artifact download risingwave-"$profile" target/debug/
+buildkite-agent artifact download risedev-dev-"$profile" target/debug/
+
+mv target/debug/risingwave-"$profile" target/debug/risingwave
+mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
+
+echo "--- Adjust permission"
+chmod +x ./target/debug/risingwave
+chmod +x ./target/debug/risedev-dev
+
+echo "--- Generate RiseDev CI config"
+cp ci/risedev-components.ci.env risedev-components.user.env
+
+echo "--- Prepare RiseDev dev cluster"
+cargo make pre-start-dev
+cargo make link-all-in-one-binaries
+
+echo "--- starting risingwave cluster with connector node"
+cargo make ci-start ci-1cn-1fe
+
+echo "--- Run test"
+python3 -m pip install minio psycopg2-binary
+python3 e2e_test/s3/run.py
+
+echo "--- Kill cluster"
+cargo make ci-kill

ci/workflows/main-cron.yml

@@ -183,3 +183,35 @@ steps:
           files: ./**/*.sh
     timeout_in_minutes: 5
     retry: *auto-retry
+
+  - label: "S3 source check on AWS"
+    command: "ci/scripts/s3-source-test.sh -p ci-release"
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+    timeout_in_minutes: 20
+    retry: *auto-retry  
+
+  - label: "S3 source check on lyvecloud.seagate.com"
+    command: "ci/scripts/s3-source-test.sh -p ci-release"
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_lyvecloud
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+    timeout_in_minutes: 20
+    retry: *auto-retry  

dashboard/components/Relations.tsx

@@ -92,7 +92,7 @@ export const primaryKeyColumn: Column<RwTable> = {
   width: 1,
   content: (r) =>
     r.pk
-      .map((order) => order.index)
+      .map((order) => order.columnIndex)
       .map((i) => r.columns[i])
       .map((col) => extractColumnInfo(col))
       .join(", "),