Skip to content

Commit 26ced36

Browse files
dsmith3197dsmith3197
authored
fix(config)!: fix concurrency default & docs (#18651)
* fix(config): fix concurrency default & docs * fix wording * fix test * remove examples * update cue * feedback * upgrade guide * nit
1 parent 483096c commit 26ced36

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+369
-141
lines changed

‎src/sinks/util/service.rs

+4-4
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use tower::{
1515
use vector_config::configurable_component;
1616

1717
pub use crate::sinks::util::service::{
18-
concurrency::{concurrency_is_none, Concurrency},
18+
concurrency::{concurrency_is_adaptive, Concurrency},
1919
health::{HealthConfig, HealthLogic, HealthService},
2020
map::Map,
2121
};
@@ -93,7 +93,7 @@ impl<L> ServiceBuilderExt<L> for ServiceBuilder<L> {
9393
pub struct TowerRequestConfig {
9494
#[configurable(derived)]
9595
#[serde(default = "default_concurrency")]
96-
#[serde(skip_serializing_if = "concurrency_is_none")]
96+
#[serde(skip_serializing_if = "concurrency_is_adaptive")]
9797
pub concurrency: Concurrency,
9898

9999
/// The time a request can take before being aborted.
@@ -144,7 +144,7 @@ pub struct TowerRequestConfig {
144144
}
145145

146146
const fn default_concurrency() -> Concurrency {
147-
Concurrency::None
147+
Concurrency::Adaptive
148148
}
149149

150150
const fn default_timeout_secs() -> Option<u64> {
@@ -457,7 +457,7 @@ mod tests {
457457
toml::from_str::<TowerRequestConfig>(&toml).expect("Default config failed");
458458

459459
let cfg = toml::from_str::<TowerRequestConfig>("").expect("Empty config failed");
460-
assert_eq!(cfg.concurrency, Concurrency::None);
460+
assert_eq!(cfg.concurrency, Concurrency::Adaptive);
461461

462462
let cfg = toml::from_str::<TowerRequestConfig>("concurrency = 10")
463463
.expect("Fixed concurrency failed");

‎src/sinks/util/service/concurrency.rs

+19-10
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ use serde::{
1717
};
1818

1919
/// Configuration for outbound request concurrency.
20+
///
21+
/// This can be set either to one of the below enum values or to a positive integer, which denotes
22+
/// a fixed concurrency limit.
2023
#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)]
2124
pub enum Concurrency {
2225
/// A fixed concurrency of 1.
@@ -48,28 +51,29 @@ impl Serialize for Concurrency {
4851

4952
impl Default for Concurrency {
5053
fn default() -> Self {
51-
Self::None
54+
Self::Adaptive
5255
}
5356
}
5457

5558
impl Concurrency {
56-
pub const fn if_none(self, other: Self) -> Self {
59+
const fn if_adaptive(self, other: Self) -> Self {
5760
match self {
58-
Self::None => other,
61+
Self::Adaptive => other,
5962
_ => self,
6063
}
6164
}
6265

63-
pub const fn parse_concurrency(&self, default: Self) -> Option<usize> {
64-
match self.if_none(default) {
65-
Concurrency::None | Concurrency::Adaptive => None,
66+
pub const fn parse_concurrency(&self, other: Self) -> Option<usize> {
67+
match self.if_adaptive(other) {
68+
Concurrency::None => Some(1),
69+
Concurrency::Adaptive => None,
6670
Concurrency::Fixed(limit) => Some(limit),
6771
}
6872
}
6973
}
7074

71-
pub const fn concurrency_is_none(concurrency: &Concurrency) -> bool {
72-
matches!(concurrency, Concurrency::None)
75+
pub const fn concurrency_is_adaptive(concurrency: &Concurrency) -> bool {
76+
matches!(concurrency, Concurrency::Adaptive)
7377
}
7478

7579
impl<'de> Deserialize<'de> for Concurrency {
@@ -93,7 +97,7 @@ impl<'de> Deserialize<'de> for Concurrency {
9397
} else if value == "none" {
9498
Ok(Concurrency::None)
9599
} else {
96-
Err(de::Error::unknown_variant(value, &["adaptive"]))
100+
Err(de::Error::unknown_variant(value, &["adaptive", "none"]))
97101
}
98102
}
99103

@@ -132,7 +136,12 @@ impl Configurable for Concurrency {
132136

133137
fn metadata() -> Metadata {
134138
let mut metadata = Metadata::default();
135-
metadata.set_description("Configuration for outbound request concurrency.");
139+
metadata.set_description(
140+
r#"Configuration for outbound request concurrency.
141+
142+
This can be set either to one of the below enum values or to a positive integer, which denotes
143+
a fixed concurrency limit."#,
144+
);
136145
metadata.add_custom_attribute(CustomAttribute::kv("docs::enum_tagging", "external"));
137146
metadata
138147
}

‎website/content/en/highlights/2023-09-26-0-33-0-upgrade-guide.md

+10-1
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@ and **deprecations**:
1818

1919
1. [Default config location change](#default-config-location-change)
2020
1. [Renaming the `armv7` rpm package](#armv7-rename)
21-
2. [Metadata field in the Vector protobuf definition](#vector-proto-metadata)
21+
1. [Metadata field in the Vector protobuf definition](#vector-proto-metadata)
2222

2323
and **potentially impactful changes**:
2424

2525
1. [Async runtime default number of worker threads](#runtime-worker-threads)
26+
1. [Setting `request.concurrency = "none"`](#request-concurrency)
2627

2728
We cover them below to help you upgrade quickly:
2829

@@ -91,3 +92,11 @@ has limited quotas, but note this change may impact performance.
9192

9293
The number of worker threads used can be seen by enabling debug logging, and the value can
9394
be overriden by setting the `VECTOR_THREADS` environment variable.
95+
96+
#### Setting `request.concurrency = "none"` {#request-concurrency}
97+
98+
Explicitly setting `request.concurrency = "none"` in a sink configuration now properly configures
99+
the sink to have a fixed concurrency limit of 1, as was stated in the documentation. Previously, the
100+
above configuration would result in the sink using adaptive request concurrency. The default setting
101+
remains unchanged (adaptive request concurrency).
102+

‎website/cue/reference/components/sinks/base/appsignal.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -198,11 +198,16 @@ base: components: sinks: appsignal: configuration: {
198198
}
199199
}
200200
concurrency: {
201-
description: "Configuration for outbound request concurrency."
202-
required: false
201+
description: """
202+
Configuration for outbound request concurrency.
203+
204+
This can be set either to one of the below enum values or to a positive integer, which denotes
205+
a fixed concurrency limit.
206+
"""
207+
required: false
203208
type: {
204209
string: {
205-
default: "none"
210+
default: "adaptive"
206211
enum: {
207212
adaptive: """
208213
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -509,11 +509,16 @@ base: components: sinks: aws_cloudwatch_logs: configuration: {
509509
}
510510
}
511511
concurrency: {
512-
description: "Configuration for outbound request concurrency."
513-
required: false
512+
description: """
513+
Configuration for outbound request concurrency.
514+
515+
This can be set either to one of the below enum values or to a positive integer, which denotes
516+
a fixed concurrency limit.
517+
"""
518+
required: false
514519
type: {
515520
string: {
516-
default: "none"
521+
default: "adaptive"
517522
enum: {
518523
adaptive: """
519524
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -286,11 +286,16 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: {
286286
}
287287
}
288288
concurrency: {
289-
description: "Configuration for outbound request concurrency."
290-
required: false
289+
description: """
290+
Configuration for outbound request concurrency.
291+
292+
This can be set either to one of the below enum values or to a positive integer, which denotes
293+
a fixed concurrency limit.
294+
"""
295+
required: false
291296
type: {
292297
string: {
293-
default: "none"
298+
default: "adaptive"
294299
enum: {
295300
adaptive: """
296301
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -480,11 +480,16 @@ base: components: sinks: aws_kinesis_firehose: configuration: {
480480
}
481481
}
482482
concurrency: {
483-
description: "Configuration for outbound request concurrency."
484-
required: false
483+
description: """
484+
Configuration for outbound request concurrency.
485+
486+
This can be set either to one of the below enum values or to a positive integer, which denotes
487+
a fixed concurrency limit.
488+
"""
489+
required: false
485490
type: {
486491
string: {
487-
default: "none"
492+
default: "adaptive"
488493
enum: {
489494
adaptive: """
490495
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/aws_kinesis_streams.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -489,11 +489,16 @@ base: components: sinks: aws_kinesis_streams: configuration: {
489489
}
490490
}
491491
concurrency: {
492-
description: "Configuration for outbound request concurrency."
493-
required: false
492+
description: """
493+
Configuration for outbound request concurrency.
494+
495+
This can be set either to one of the below enum values or to a positive integer, which denotes
496+
a fixed concurrency limit.
497+
"""
498+
required: false
494499
type: {
495500
string: {
496-
default: "none"
501+
default: "adaptive"
497502
enum: {
498503
adaptive: """
499504
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/aws_s3.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -726,11 +726,16 @@ base: components: sinks: aws_s3: configuration: {
726726
}
727727
}
728728
concurrency: {
729-
description: "Configuration for outbound request concurrency."
730-
required: false
729+
description: """
730+
Configuration for outbound request concurrency.
731+
732+
This can be set either to one of the below enum values or to a positive integer, which denotes
733+
a fixed concurrency limit.
734+
"""
735+
required: false
731736
type: {
732737
string: {
733-
default: "none"
738+
default: "adaptive"
734739
enum: {
735740
adaptive: """
736741
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/aws_sns.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -437,11 +437,16 @@ base: components: sinks: aws_sns: configuration: {
437437
}
438438
}
439439
concurrency: {
440-
description: "Configuration for outbound request concurrency."
441-
required: false
440+
description: """
441+
Configuration for outbound request concurrency.
442+
443+
This can be set either to one of the below enum values or to a positive integer, which denotes
444+
a fixed concurrency limit.
445+
"""
446+
required: false
442447
type: {
443448
string: {
444-
default: "none"
449+
default: "adaptive"
445450
enum: {
446451
adaptive: """
447452
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/aws_sqs.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -442,11 +442,16 @@ base: components: sinks: aws_sqs: configuration: {
442442
}
443443
}
444444
concurrency: {
445-
description: "Configuration for outbound request concurrency."
446-
required: false
445+
description: """
446+
Configuration for outbound request concurrency.
447+
448+
This can be set either to one of the below enum values or to a positive integer, which denotes
449+
a fixed concurrency limit.
450+
"""
451+
required: false
447452
type: {
448453
string: {
449-
default: "none"
454+
default: "adaptive"
450455
enum: {
451456
adaptive: """
452457
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/axiom.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -136,11 +136,16 @@ base: components: sinks: axiom: configuration: {
136136
}
137137
}
138138
concurrency: {
139-
description: "Configuration for outbound request concurrency."
140-
required: false
139+
description: """
140+
Configuration for outbound request concurrency.
141+
142+
This can be set either to one of the below enum values or to a positive integer, which denotes
143+
a fixed concurrency limit.
144+
"""
145+
required: false
141146
type: {
142147
string: {
143-
default: "none"
148+
default: "adaptive"
144149
enum: {
145150
adaptive: """
146151
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/azure_blob.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -474,11 +474,16 @@ base: components: sinks: azure_blob: configuration: {
474474
}
475475
}
476476
concurrency: {
477-
description: "Configuration for outbound request concurrency."
478-
required: false
477+
description: """
478+
Configuration for outbound request concurrency.
479+
480+
This can be set either to one of the below enum values or to a positive integer, which denotes
481+
a fixed concurrency limit.
482+
"""
483+
required: false
479484
type: {
480485
string: {
481-
default: "none"
486+
default: "adaptive"
482487
enum: {
483488
adaptive: """
484489
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/azure_monitor_logs.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -194,11 +194,16 @@ base: components: sinks: azure_monitor_logs: configuration: {
194194
}
195195
}
196196
concurrency: {
197-
description: "Configuration for outbound request concurrency."
198-
required: false
197+
description: """
198+
Configuration for outbound request concurrency.
199+
200+
This can be set either to one of the below enum values or to a positive integer, which denotes
201+
a fixed concurrency limit.
202+
"""
203+
required: false
199204
type: {
200205
string: {
201-
default: "none"
206+
default: "adaptive"
202207
enum: {
203208
adaptive: """
204209
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/clickhouse.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -247,11 +247,16 @@ base: components: sinks: clickhouse: configuration: {
247247
}
248248
}
249249
concurrency: {
250-
description: "Configuration for outbound request concurrency."
251-
required: false
250+
description: """
251+
Configuration for outbound request concurrency.
252+
253+
This can be set either to one of the below enum values or to a positive integer, which denotes
254+
a fixed concurrency limit.
255+
"""
256+
required: false
252257
type: {
253258
string: {
254-
default: "none"
259+
default: "adaptive"
255260
enum: {
256261
adaptive: """
257262
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

‎website/cue/reference/components/sinks/base/databend.cue

+8-3
Original file line numberDiff line numberDiff line change
@@ -350,11 +350,16 @@ base: components: sinks: databend: configuration: {
350350
}
351351
}
352352
concurrency: {
353-
description: "Configuration for outbound request concurrency."
354-
required: false
353+
description: """
354+
Configuration for outbound request concurrency.
355+
356+
This can be set either to one of the below enum values or to a positive integer, which denotes
357+
a fixed concurrency limit.
358+
"""
359+
required: false
355360
type: {
356361
string: {
357-
default: "none"
362+
default: "adaptive"
358363
enum: {
359364
adaptive: """
360365
Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature.

0 commit comments

Comments
 (0)