Skip to content

Commit f15144b

Browse files
fuchsnjfuchsnj
authored
fix(json codec): Fix deserializing non-object values with the Vector namespace (vectordotdev#18379)
* fix remap array return values when using the Vector namespace * fix json codec when using vector namespace
1 parent 76ffdab commit f15144b

File tree

5 files changed

+95
-48
lines changed

5 files changed

+95
-48
lines changed

‎lib/codecs/src/decoding/format/json.rs

+19-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
use std::convert::TryInto;
2-
31
use bytes::Bytes;
42
use chrono::Utc;
53
use derivative::Derivative;
@@ -122,9 +120,9 @@ impl Deserializer for JsonDeserializer {
122120
let mut events = match json {
123121
serde_json::Value::Array(values) => values
124122
.into_iter()
125-
.map(TryInto::try_into)
123+
.map(|json| Event::from_json_value(json, log_namespace))
126124
.collect::<Result<SmallVec<[Event; 1]>, _>>()?,
127-
_ => smallvec![json.try_into()?],
125+
_ => smallvec![Event::from_json_value(json, log_namespace)?],
128126
};
129127

130128
let events = match log_namespace {
@@ -160,6 +158,7 @@ impl From<&JsonDeserializerConfig> for JsonDeserializer {
160158
#[cfg(test)]
161159
mod tests {
162160
use vector_core::config::log_schema;
161+
use vrl::core::Value;
163162

164163
use super::*;
165164

@@ -190,6 +189,22 @@ mod tests {
190189
}
191190
}
192191

192+
#[test]
193+
fn deserialize_non_object_vector_namespace() {
194+
let input = Bytes::from(r#"null"#);
195+
let deserializer = JsonDeserializer::default();
196+
197+
let namespace = LogNamespace::Vector;
198+
let events = deserializer.parse(input.clone(), namespace).unwrap();
199+
let mut events = events.into_iter();
200+
201+
let event = events.next().unwrap();
202+
let log = event.as_log();
203+
assert_eq!(log["."], Value::Null);
204+
205+
assert_eq!(events.next(), None);
206+
}
207+
193208
#[test]
194209
fn deserialize_json_array() {
195210
let input = Bytes::from(r#"[{ "foo": 123 }, { "bar": 456 }]"#);

‎lib/codecs/src/decoding/format/native_json.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,8 @@ mod test {
132132

133133
let events = deserializer.parse(input, LogNamespace::Legacy).unwrap();
134134

135-
let event1 = Event::try_from(json1).unwrap();
136-
let event2 = Event::try_from(json2).unwrap();
135+
let event1 = Event::from_json_value(json1, LogNamespace::Legacy).unwrap();
136+
let event2 = Event::from_json_value(json2, LogNamespace::Legacy).unwrap();
137137
let expected: SmallVec<[Event; 1]> = smallvec![event1, event2];
138138
assert_eq!(events, expected);
139139
}

‎lib/vector-core/src/event/mod.rs

+27-25
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
1-
use std::{
2-
collections::BTreeMap,
3-
convert::{TryFrom, TryInto},
4-
fmt::Debug,
5-
sync::Arc,
6-
};
1+
use std::{collections::BTreeMap, convert::TryInto, fmt::Debug, sync::Arc};
72

3+
use crate::config::LogNamespace;
84
use crate::{config::OutputId, ByteSizeOf};
95
pub use array::{into_event_stream, EventArray, EventContainer, LogArray, MetricArray, TraceArray};
106
pub use estimated_json_encoded_size_of::EstimatedJsonEncodedSizeOf;
@@ -324,6 +320,31 @@ impl Event {
324320
self.metadata_mut().set_upstream_id(upstream_id);
325321
self
326322
}
323+
324+
/// Creates an Event from a JSON value.
325+
///
326+
/// # Errors
327+
/// If a non-object JSON value is passed in with the `Legacy` namespace, this will return an error.
328+
pub fn from_json_value(
329+
value: serde_json::Value,
330+
log_namespace: LogNamespace,
331+
) -> crate::Result<Self> {
332+
match log_namespace {
333+
LogNamespace::Vector => Ok(LogEvent::from(Value::from(value)).into()),
334+
LogNamespace::Legacy => match value {
335+
serde_json::Value::Object(fields) => Ok(LogEvent::from(
336+
fields
337+
.into_iter()
338+
.map(|(k, v)| (k, v.into()))
339+
.collect::<BTreeMap<_, _>>(),
340+
)
341+
.into()),
342+
_ => Err(crate::Error::from(
343+
"Attempted to convert non-Object JSON into an Event.",
344+
)),
345+
},
346+
}
347+
}
327348
}
328349

329350
impl EventDataEq for Event {
@@ -348,25 +369,6 @@ impl finalization::AddBatchNotifier for Event {
348369
}
349370
}
350371

351-
impl TryFrom<serde_json::Value> for Event {
352-
type Error = crate::Error;
353-
354-
fn try_from(map: serde_json::Value) -> Result<Self, Self::Error> {
355-
match map {
356-
serde_json::Value::Object(fields) => Ok(LogEvent::from(
357-
fields
358-
.into_iter()
359-
.map(|(k, v)| (k, v.into()))
360-
.collect::<BTreeMap<_, _>>(),
361-
)
362-
.into()),
363-
_ => Err(crate::Error::from(
364-
"Attempted to convert non-Object JSON into an Event.",
365-
)),
366-
}
367-
}
368-
}
369-
370372
impl TryInto<serde_json::Value> for Event {
371373
type Error = serde_json::Error;
372374

‎src/transforms/remap.rs

+34-13
Original file line numberDiff line numberDiff line change
@@ -1024,8 +1024,11 @@ mod tests {
10241024

10251025
#[test]
10261026
fn remap_timezone_fallback() {
1027-
let error =
1028-
Event::try_from(serde_json::json!({"timestamp": "2022-12-27 00:00:00"})).unwrap();
1027+
let error = Event::from_json_value(
1028+
serde_json::json!({"timestamp": "2022-12-27 00:00:00"}),
1029+
LogNamespace::Legacy,
1030+
)
1031+
.unwrap();
10291032
let conf = RemapConfig {
10301033
source: Some(formatdoc! {r#"
10311034
.timestamp = parse_timestamp!(.timestamp, format: "%Y-%m-%d %H:%M:%S")
@@ -1058,8 +1061,11 @@ mod tests {
10581061

10591062
#[test]
10601063
fn remap_timezone_override() {
1061-
let error =
1062-
Event::try_from(serde_json::json!({"timestamp": "2022-12-27 00:00:00"})).unwrap();
1064+
let error = Event::from_json_value(
1065+
serde_json::json!({"timestamp": "2022-12-27 00:00:00"}),
1066+
LogNamespace::Legacy,
1067+
)
1068+
.unwrap();
10631069
let conf = RemapConfig {
10641070
source: Some(formatdoc! {r#"
10651071
.timestamp = parse_timestamp!(.timestamp, format: "%Y-%m-%d %H:%M:%S")
@@ -1093,9 +1099,16 @@ mod tests {
10931099

10941100
#[test]
10951101
fn check_remap_branching() {
1096-
let happy = Event::try_from(serde_json::json!({"hello": "world"})).unwrap();
1097-
let abort = Event::try_from(serde_json::json!({"hello": "goodbye"})).unwrap();
1098-
let error = Event::try_from(serde_json::json!({"hello": 42})).unwrap();
1102+
let happy =
1103+
Event::from_json_value(serde_json::json!({"hello": "world"}), LogNamespace::Legacy)
1104+
.unwrap();
1105+
let abort = Event::from_json_value(
1106+
serde_json::json!({"hello": "goodbye"}),
1107+
LogNamespace::Legacy,
1108+
)
1109+
.unwrap();
1110+
let error =
1111+
Event::from_json_value(serde_json::json!({"hello": 42}), LogNamespace::Legacy).unwrap();
10991112

11001113
let happy_metric = {
11011114
let mut metric = Metric::new(
@@ -1287,9 +1300,9 @@ mod tests {
12871300
#[test]
12881301
fn check_remap_branching_assert_with_message() {
12891302
let error_trigger_assert_custom_message =
1290-
Event::try_from(serde_json::json!({"hello": 42})).unwrap();
1303+
Event::from_json_value(serde_json::json!({"hello": 42}), LogNamespace::Legacy).unwrap();
12911304
let error_trigger_default_assert_message =
1292-
Event::try_from(serde_json::json!({"hello": 0})).unwrap();
1305+
Event::from_json_value(serde_json::json!({"hello": 0}), LogNamespace::Legacy).unwrap();
12931306
let conf = RemapConfig {
12941307
source: Some(formatdoc! {r#"
12951308
assert_eq!(.hello, 0, "custom message here")
@@ -1349,7 +1362,8 @@ mod tests {
13491362

13501363
#[test]
13511364
fn check_remap_branching_abort_with_message() {
1352-
let error = Event::try_from(serde_json::json!({"hello": 42})).unwrap();
1365+
let error =
1366+
Event::from_json_value(serde_json::json!({"hello": 42}), LogNamespace::Legacy).unwrap();
13531367
let conf = RemapConfig {
13541368
source: Some(formatdoc! {r#"
13551369
abort "custom message here"
@@ -1387,9 +1401,16 @@ mod tests {
13871401

13881402
#[test]
13891403
fn check_remap_branching_disabled() {
1390-
let happy = Event::try_from(serde_json::json!({"hello": "world"})).unwrap();
1391-
let abort = Event::try_from(serde_json::json!({"hello": "goodbye"})).unwrap();
1392-
let error = Event::try_from(serde_json::json!({"hello": 42})).unwrap();
1404+
let happy =
1405+
Event::from_json_value(serde_json::json!({"hello": "world"}), LogNamespace::Legacy)
1406+
.unwrap();
1407+
let abort = Event::from_json_value(
1408+
serde_json::json!({"hello": "goodbye"}),
1409+
LogNamespace::Legacy,
1410+
)
1411+
.unwrap();
1412+
let error =
1413+
Event::from_json_value(serde_json::json!({"hello": 42}), LogNamespace::Legacy).unwrap();
13931414

13941415
let conf = RemapConfig {
13951416
source: Some(formatdoc! {r#"

‎src/transforms/route.rs

+13-4
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,9 @@ mod test {
189189
#[test]
190190
fn route_pass_all_route_conditions() {
191191
let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE];
192-
let event = Event::try_from(
192+
let event = Event::from_json_value(
193193
serde_json::json!({"message": "hello world", "second": "second", "third": "third"}),
194+
LogNamespace::Legacy,
194195
)
195196
.unwrap();
196197
let config = toml::from_str::<RouteConfig>(
@@ -234,7 +235,11 @@ mod test {
234235
#[test]
235236
fn route_pass_one_route_condition() {
236237
let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE];
237-
let event = Event::try_from(serde_json::json!({"message": "hello world"})).unwrap();
238+
let event = Event::from_json_value(
239+
serde_json::json!({"message": "hello world"}),
240+
LogNamespace::Legacy,
241+
)
242+
.unwrap();
238243
let config = toml::from_str::<RouteConfig>(
239244
r#"
240245
route.first.type = "vrl"
@@ -275,7 +280,9 @@ mod test {
275280
#[test]
276281
fn route_pass_no_route_condition() {
277282
let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE];
278-
let event = Event::try_from(serde_json::json!({"message": "NOPE"})).unwrap();
283+
let event =
284+
Event::from_json_value(serde_json::json!({"message": "NOPE"}), LogNamespace::Legacy)
285+
.unwrap();
279286
let config = toml::from_str::<RouteConfig>(
280287
r#"
281288
route.first.type = "vrl"
@@ -316,7 +323,9 @@ mod test {
316323
#[test]
317324
fn route_no_unmatched_output() {
318325
let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE];
319-
let event = Event::try_from(serde_json::json!({"message": "NOPE"})).unwrap();
326+
let event =
327+
Event::from_json_value(serde_json::json!({"message": "NOPE"}), LogNamespace::Legacy)
328+
.unwrap();
320329
let config = toml::from_str::<RouteConfig>(
321330
r#"
322331
reroute_unmatched = false

0 commit comments

Comments
 (0)