From 908a50ba838d2ca6a40260e7868413b226f9c330 Mon Sep 17 00:00:00 2001 From: Arun Pandian Date: Tue, 4 Feb 2025 00:55:23 -0800 Subject: [PATCH] [Dataflow Streaming] Enable Windmill GetWork Response Batching by default (#33847) With this change user workers will request batched responses from backend and backend will send multiple WorkItems in the same response proto. Backend changes are rollbacksafe. The feature can be disabled by passing `--windmillRequestBatchedGetWorkResponse=false` --- CHANGES.md | 4 ++++ .../dataflow/options/DataflowStreamingPipelineOptions.java | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index fde00b9da4ce..7709a18c62b0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -87,6 +87,10 @@ * Support the AnyOf Environment for execution in Prism ([#33705](https://github.com/apache/beam/pull/33705)) * This improves support for developing Xlang pipelines, when using a compatible cross language service. * Partitions are now configurable for the DaskRunner in the Python SDK ([#33805](https://github.com/apache/beam/pull/33805)). +* [Dataflow Streaming] Enable Windmill GetWork Response Batching by default ([#33847](https://github.com/apache/beam/pull/33847)). + * With this change user workers will request batched GetWork responses from backend and backend will send multiple WorkItems in the same response proto. + * The feature can be disabled by passing `--windmillRequestBatchedGetWorkResponse=false` + ## Breaking Changes diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowStreamingPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowStreamingPipelineOptions.java index 2c037ef05569..4c1a82418848 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowStreamingPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowStreamingPipelineOptions.java @@ -137,6 +137,7 @@ public interface DataflowStreamingPipelineOptions extends PipelineOptions { void setUseSeparateWindmillHeartbeatStreams(Boolean value); @Description("If true, GetWorkStreams will request multiple work items in a response chunk.") + @Default.Boolean(true) boolean getWindmillRequestBatchedGetWorkResponse(); void setWindmillRequestBatchedGetWorkResponse(boolean value);