[improve][misc] Sync commits from apache into 3.1_ds (19 Feb)

.github/actions/clean-disk/action.yml

@@ -46,6 +46,22 @@ runs:
             time df -BM / /mnt
             echo "::endgroup::"
           done
+          if [[ "${{ inputs.mode }}" == "full" ]]; then
+            echo "::group::Moving /var/lib/docker to /mnt/docker"
+            sudo systemctl stop docker
+            if sudo test ! -f /etc/docker/daemon.json; then
+              # create daemon.json file
+              echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
+            else
+              # modify existing daemon.json file
+              sudo jq '.["data-root"]="/mnt/docker"' /etc/docker/daemon.json | sudo tee /tmp/daemon_temp.json$$
+              sudo mv /tmp/daemon_temp.json$$ /etc/docker/daemon.json
+            fi
+            sudo mv /var/lib/docker /mnt/docker
+            sudo systemctl start docker
+            time df -BM / /mnt
+            echo "::endgroup::"
+          fi
           echo "::group::Cleaning apt state"
           time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
           time df -BM / /mnt

.github/workflows/pulsar-ci.yaml

@@ -1312,15 +1312,21 @@ jobs:
           restore-keys: |
             owasp-dependency-check-data-
 
+      - name: Log warning when skipped
+        if: ${{ !steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }}
+        run: |
+          echo "::warning::OWASP Dependency Check was skipped since the OWASP Dependency check data wasn't found in the cache. Run ci-owasp-dependency-check.yaml workflow to update the cache."
+
       # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
       - name: trigger dependency check
+        if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }}
         run: |
           mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
             -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
 
       - name: Upload report
         uses: actions/upload-artifact@v4
-        if: ${{ cancelled() || failure() }}
+        if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key && (cancelled() || failure()) }}
         continue-on-error: true
         with:
           name: dependency report

buildtools/pom.xml

@@ -35,6 +35,13 @@
   <packaging>jar</packaging>
   <name>Pulsar Build Tools</name>
 
+  <developers>
+    <developer>
+      <organization>Apache Pulsar developers</organization>
+      <organizationUrl>http://pulsar.apache.org/</organizationUrl>
+    </developer>
+  </developers>
+
   <properties>
     <project.build.outputTimestamp>2025-01-20T05:51:16Z</project.build.outputTimestamp>
     <maven.compiler.source>1.8</maven.compiler.source>
@@ -47,7 +54,7 @@
     <license-maven-plugin.version>4.1</license-maven-plugin.version>
     <puppycrawl.checkstyle.version>10.14.2</puppycrawl.checkstyle.version>
     <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
-    <netty.version>4.1.117.Final</netty.version>
+    <netty.version>4.1.118.Final</netty.version>
     <guice.version>4.2.3</guice.version>
     <guava.version>32.1.1-jre</guava.version>
     <ant.version>1.10.12</ant.version>

conf/proxy.conf

@@ -59,6 +59,13 @@ bindAddress=0.0.0.0
 # If not set, the value of `InetAddress.getLocalHost().getCanonicalHostName()` is used.
 advertisedAddress=
 
+# Specifies the interval (in seconds) for sending ping messages to the client. Set to 0 to disable
+# ping messages. This setting applies to client connections used for topic lookups and
+# partition metadata requests. When a client establishes a broker connection via the proxy,
+# the client and broker will communicate directly without the proxy intercepting the messages.
+# In that case, the broker's keepAliveIntervalSeconds configuration becomes relevant.
+keepAliveIntervalSeconds=30
+
 # Enable or disable the HAProxy protocol.
 # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
 haProxyProtocolEnabled=false

distribution/server/src/assemble/LICENSE.bin.txt

@@ -289,33 +289,33 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.11.jar
     - org.apache.commons-commons-text-1.10.0.jar
  * Netty
-    - io.netty-netty-buffer-4.1.117.Final.jar
-    - io.netty-netty-codec-4.1.117.Final.jar
-    - io.netty-netty-codec-dns-4.1.117.Final.jar
-    - io.netty-netty-codec-http-4.1.117.Final.jar
-    - io.netty-netty-codec-http2-4.1.117.Final.jar
-    - io.netty-netty-codec-socks-4.1.117.Final.jar
-    - io.netty-netty-codec-haproxy-4.1.117.Final.jar
-    - io.netty-netty-common-4.1.117.Final.jar
-    - io.netty-netty-handler-4.1.117.Final.jar
-    - io.netty-netty-handler-proxy-4.1.117.Final.jar
-    - io.netty-netty-resolver-4.1.117.Final.jar
-    - io.netty-netty-resolver-dns-4.1.117.Final.jar
-    - io.netty-netty-resolver-dns-classes-macos-4.1.117.Final.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.117.Final-osx-aarch_64.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.117.Final-osx-x86_64.jar
-    - io.netty-netty-transport-4.1.117.Final.jar
-    - io.netty-netty-transport-classes-epoll-4.1.117.Final.jar
-    - io.netty-netty-transport-native-epoll-4.1.117.Final-linux-aarch_64.jar
-    - io.netty-netty-transport-native-epoll-4.1.117.Final-linux-x86_64.jar
-    - io.netty-netty-transport-native-unix-common-4.1.117.Final.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar
-    - io.netty-netty-tcnative-classes-2.0.69.Final.jar
+    - io.netty-netty-buffer-4.1.118.Final.jar
+    - io.netty-netty-codec-4.1.118.Final.jar
+    - io.netty-netty-codec-dns-4.1.118.Final.jar
+    - io.netty-netty-codec-http-4.1.118.Final.jar
+    - io.netty-netty-codec-http2-4.1.118.Final.jar
+    - io.netty-netty-codec-socks-4.1.118.Final.jar
+    - io.netty-netty-codec-haproxy-4.1.118.Final.jar
+    - io.netty-netty-common-4.1.118.Final.jar
+    - io.netty-netty-handler-4.1.118.Final.jar
+    - io.netty-netty-handler-proxy-4.1.118.Final.jar
+    - io.netty-netty-resolver-4.1.118.Final.jar
+    - io.netty-netty-resolver-dns-4.1.118.Final.jar
+    - io.netty-netty-resolver-dns-classes-macos-4.1.118.Final.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.118.Final-osx-aarch_64.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.118.Final-osx-x86_64.jar
+    - io.netty-netty-transport-4.1.118.Final.jar
+    - io.netty-netty-transport-classes-epoll-4.1.118.Final.jar
+    - io.netty-netty-transport-native-epoll-4.1.118.Final-linux-aarch_64.jar
+    - io.netty-netty-transport-native-epoll-4.1.118.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-unix-common-4.1.118.Final.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar
+    - io.netty-netty-tcnative-classes-2.0.70.Final.jar
     - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar
     - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
     - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
@@ -577,31 +577,6 @@ Bouncy Castle License
     - org.bouncycastle-bcprov-jdk18on-1.78.1.jar
     - org.bouncycastle-bcutil-jdk18on-1.78.1.jar
 
-Datastax
-    - com.datastax.oss-managed-ledger-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-broker-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-broker-auth-oidc-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-broker-auth-sasl-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-broker-common-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-config-validation-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-docs-tools-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-api-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-instance-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-local-runner-original-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-proto-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-runtime-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-secrets-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-utils-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-functions-worker-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-io-core-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-metadata-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-proxy-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-testclient-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-transaction-common-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-pulsar-transaction-coordinator-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-testmocks-3.1.4.13-SNAPSHOT.jar
-    - com.datastax.oss-jetcd-core-shaded-3.1.4.13-SNAPSHOT.jar
-
 ------------------------
 
 Additionaly, Netty includes code with the following licenses:

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -344,35 +344,35 @@ The Apache Software License, Version 2.0
     - commons-text-1.10.0.jar
     - commons-compress-1.26.0.jar
  * Netty
-    - netty-buffer-4.1.117.Final.jar
-    - netty-codec-4.1.117.Final.jar
-    - netty-codec-dns-4.1.117.Final.jar
-    - netty-codec-http-4.1.117.Final.jar
-    - netty-codec-socks-4.1.117.Final.jar
-    - netty-codec-haproxy-4.1.117.Final.jar
-    - netty-common-4.1.117.Final.jar
-    - netty-handler-4.1.117.Final.jar
-    - netty-handler-proxy-4.1.117.Final.jar
-    - netty-resolver-4.1.117.Final.jar
-    - netty-resolver-dns-4.1.117.Final.jar
-    - netty-transport-4.1.117.Final.jar
-    - netty-transport-classes-epoll-4.1.117.Final.jar
-    - netty-transport-native-epoll-4.1.117.Final-linux-aarch_64.jar
-    - netty-transport-native-epoll-4.1.117.Final-linux-x86_64.jar
-    - netty-transport-native-unix-common-4.1.117.Final.jar
-    - netty-tcnative-boringssl-static-2.0.69.Final.jar
-    - netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar
-    - netty-tcnative-classes-2.0.69.Final.jar
+    - netty-buffer-4.1.118.Final.jar
+    - netty-codec-4.1.118.Final.jar
+    - netty-codec-dns-4.1.118.Final.jar
+    - netty-codec-http-4.1.118.Final.jar
+    - netty-codec-socks-4.1.118.Final.jar
+    - netty-codec-haproxy-4.1.118.Final.jar
+    - netty-common-4.1.118.Final.jar
+    - netty-handler-4.1.118.Final.jar
+    - netty-handler-proxy-4.1.118.Final.jar
+    - netty-resolver-4.1.118.Final.jar
+    - netty-resolver-dns-4.1.118.Final.jar
+    - netty-transport-4.1.118.Final.jar
+    - netty-transport-classes-epoll-4.1.118.Final.jar
+    - netty-transport-native-epoll-4.1.118.Final-linux-aarch_64.jar
+    - netty-transport-native-epoll-4.1.118.Final-linux-x86_64.jar
+    - netty-transport-native-unix-common-4.1.118.Final.jar
+    - netty-tcnative-boringssl-static-2.0.70.Final.jar
+    - netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar
+    - netty-tcnative-classes-2.0.70.Final.jar
     - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar
     - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
     - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
-    - netty-resolver-dns-classes-macos-4.1.117.Final.jar
-    - netty-resolver-dns-native-macos-4.1.117.Final-osx-aarch_64.jar
-    - netty-resolver-dns-native-macos-4.1.117.Final-osx-x86_64.jar
+    - netty-resolver-dns-classes-macos-4.1.118.Final.jar
+    - netty-resolver-dns-native-macos-4.1.118.Final-osx-aarch_64.jar
+    - netty-resolver-dns-native-macos-4.1.118.Final-osx-x86_64.jar
  * Prometheus client
     - simpleclient-0.16.0.jar
     - simpleclient_log4j2-0.16.0.jar

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java

@@ -1137,16 +1137,17 @@ public ManagedCursor newNonDurableCursor(Position startCursorPosition, String cu
             return cachedCursor;
         }
 
-        NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, this, cursorName,
-                (PositionImpl) startCursorPosition, initialPosition, isReadCompacted);
-        cursor.setActive();
-
-        log.info("[{}] Opened new cursor: {}", name, cursor);
+        // The backlog of a non-durable cursor could be incorrect if the cursor is created before `internalTrimLedgers`
+        // and added to the managed ledger after `internalTrimLedgers`.
+        // For more details, see https://github.com/apache/pulsar/pull/23951.
         synchronized (this) {
+            NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, this, cursorName,
+                    (PositionImpl) startCursorPosition, initialPosition, isReadCompacted);
+            cursor.setActive();
+            log.info("[{}] Opened new cursor: {}", name, cursor);
             addCursor(cursor);
+            return cursor;
         }
-
-        return cursor;
     }
 
     @Override

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java

@@ -456,8 +456,13 @@ public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws InvalidProto
             try {
                 MLDataFormats.ManagedLedgerInfoMetadata metadata =
                         MLDataFormats.ManagedLedgerInfoMetadata.parseFrom(metadataBytes);
-                return ManagedLedgerInfo.parseFrom(getCompressionCodec(metadata.getCompressionType())
-                        .decode(byteBuf, metadata.getUncompressedSize()).nioBuffer());
+                ByteBuf uncompressed = getCompressionCodec(metadata.getCompressionType())
+                        .decode(byteBuf, metadata.getUncompressedSize());
+                try {
+                    return ManagedLedgerInfo.parseFrom(uncompressed.nioBuffer());
+                } finally {
+                    uncompressed.release();
+                }
             } catch (Exception e) {
                 log.error("Failed to parse managedLedgerInfo metadata, "
                         + "fall back to parse managedLedgerInfo directly.", e);
@@ -478,8 +483,13 @@ public ManagedCursorInfo parseManagedCursorInfo(byte[] data) throws InvalidProto
             try {
                 MLDataFormats.ManagedCursorInfoMetadata metadata =
                         MLDataFormats.ManagedCursorInfoMetadata.parseFrom(metadataBytes);
-                return ManagedCursorInfo.parseFrom(getCompressionCodec(metadata.getCompressionType())
-                        .decode(byteBuf, metadata.getUncompressedSize()).nioBuffer());
+                ByteBuf uncompressed = getCompressionCodec(metadata.getCompressionType())
+                        .decode(byteBuf, metadata.getUncompressedSize());
+                try {
+                    return ManagedCursorInfo.parseFrom(uncompressed.nioBuffer());
+                } finally {
+                    uncompressed.release();
+                }
             } catch (Exception e) {
                 log.error("Failed to parse ManagedCursorInfo metadata, "
                         + "fall back to parse ManagedCursorInfo directly", e);
@@ -503,29 +513,23 @@ private byte[] compressManagedInfo(byte[] info, byte[] metadata, int metadataSer
         if (compressionType == null || compressionType.equals(CompressionType.NONE)) {
             return info;
         }
-        ByteBuf metadataByteBuf = null;
-        ByteBuf encodeByteBuf = null;
+
+        CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer();
         try {
-            metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(metadataSerializedSize + 6,
+            ByteBuf metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(metadataSerializedSize + 6,
                     metadataSerializedSize + 6);
             metadataByteBuf.writeShort(MAGIC_MANAGED_INFO_METADATA);
             metadataByteBuf.writeInt(metadataSerializedSize);
             metadataByteBuf.writeBytes(metadata);
-            encodeByteBuf = getCompressionCodec(compressionType)
+            ByteBuf encodeByteBuf = getCompressionCodec(compressionType)
                     .encode(Unpooled.wrappedBuffer(info));
-            CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer();
             compositeByteBuf.addComponent(true, metadataByteBuf);
             compositeByteBuf.addComponent(true, encodeByteBuf);
             byte[] dataBytes = new byte[compositeByteBuf.readableBytes()];
             compositeByteBuf.readBytes(dataBytes);
             return dataBytes;
         } finally {
-            if (metadataByteBuf != null) {
-                metadataByteBuf.release();
-            }
-            if (encodeByteBuf != null) {
-                encodeByteBuf.release();
-            }
+            compositeByteBuf.release();
         }
     }