Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(remap): Make del function only take one arg #5633

Merged
merged 29 commits into from
Jan 7, 2021
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
bca8dcb
Make del function only take one arg
Dec 20, 2020
ca18075
Update function logic to use remove_and_get
Dec 21, 2020
7c4ec31
Add append function
Dec 28, 2020
e69b2aa
Add CUE docs sources for append function
Dec 28, 2020
b7d4aa0
Add TOML unit test for append function
Dec 28, 2020
40b22f1
Fix formatting
Dec 28, 2020
fb0c35e
Fix CUE formatting
Dec 28, 2020
62957dc
Resolve md5/sha1 import ambiguity
Dec 29, 2020
b988901
Merge branch 'master' into del-function-one-arg
Dec 29, 2020
7dc4407
Clarify append semantics in docs
Dec 29, 2020
0c80b9d
Merge branch 'master' into remap-append-function
Dec 29, 2020
3599d3d
Revert behavior to null only on non-existing field
Dec 29, 2020
f5bcf6f
Add dummy non-implementations of remove_and_get for LogEvent and Metric
Dec 29, 2020
3ef679a
Update docs to reflect new single-field logic
Dec 29, 2020
f766a9f
Update function description re: return value
Dec 29, 2020
1b71b24
Implemented remove_and_get for LogEvent
StephenWakely Dec 31, 2020
b5d9764
Implemented remove_and_get for metrics
StephenWakely Jan 1, 2021
2ba76ba
Merge branch 'del-function-one-arg' of https://github.com/timberio/ve…
Jan 2, 2021
7e3ab13
Merge branch 'master' into del-function-one-arg
Jan 2, 2021
e9e2424
Update docs with rename field example
Jan 4, 2021
dff9788
Rename field to path
Jan 4, 2021
fc6241e
Remove errant CUE doc
Jan 5, 2021
a06a806
changes
JeanMertz Jan 6, 2021
04f82cf
more changes
JeanMertz Jan 6, 2021
6f1366b
some more changes
JeanMertz Jan 6, 2021
1feae2b
improvements
JeanMertz Jan 6, 2021
3f42c45
Merge branch 'master' into del-function-one-arg
Jan 7, 2021
9c0f32b
Fix formatting issue in CUE source
Jan 7, 2021
2c83be4
changes
JeanMertz Jan 7, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/reference/remap.cue
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ remap: {

arguments: [...#Argument] // Allow for empty list
return: [#RemapReturnTypes, ...#RemapReturnTypes]
category: "Coerce" | "Encode" | "Enumerate" | "Event" | "Hash" | "IP" | "Map" | "Number" | "Parse" | "Random" | "String" | "Test" | "Timestamp"
category: "Array" | "Coerce" | "Encode" | "Enumerate" | "Event" | "Hash" | "IP" | "Map" | "Number" | "Parse" | "Random" | "String" | "Test" | "Timestamp"
JeanMertz marked this conversation as resolved.
Show resolved Hide resolved
description: string
examples?: [#RemapExample, ...#RemapExample]
name: Name
Expand Down
26 changes: 17 additions & 9 deletions docs/reference/remap/del.cue
Original file line number Diff line number Diff line change
Expand Up @@ -3,32 +3,40 @@ package metadata
remap: functions: del: {
arguments: [
{
name: "paths"
description: "The paths of the fields to delete."
name: "path"
description: "The path of the field to delete."
required: true
multiple: true
type: ["string"]
},
]
return: ["null"]
return: ["any"]
category: "Event"
description: #"""
Removed the fields specified by the given paths from the root `event` object. Multiple fields can be specified.
Removes the field specified by the given path from the event object. If the field exists,
the field's value is returned by the delete operation, while `null` is returned if the field
doesn't exist.
"""#
examples: [
JeanMertz marked this conversation as resolved.
Show resolved Hide resolved
{
title: "Success"
input: {
"field1": 1
"field2": 2
"field3": 3
}
source: #"""
del(.field1, .field3)
"""#
source: "del(.field1)"
output: {
"field2": 2
}
},
{
title: "Rename field"
JeanMertz marked this conversation as resolved.
Show resolved Hide resolved
input: {
old_field: "please rename me"
}
source: ".new_field = del(.old_field)"
output: {
new_field: "please rename me"
}
},
]
}
122 changes: 88 additions & 34 deletions lib/remap-functions/src/del.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,47 +9,58 @@ impl Function for Del {
}

fn parameters(&self) -> &'static [Parameter] {
generate_param_list! {
accepts = |_| true,
required = false,
keywords = [
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16",
],
}
&[Parameter {
keyword: "path",
accepts: |_| true,
required: true,
}]
}

fn compile(&self, mut arguments: ArgumentList) -> Result<Box<dyn Expression>> {
let mut paths = vec![];
paths.push(arguments.required_path("1")?);

for i in 2..=16 {
if let Some(path) = arguments.optional_path(&format!("{}", i))? {
paths.push(path)
}
}
let path = arguments.required_path("path")?;

Ok(Box::new(DelFn { paths }))
Ok(Box::new(DelFn { path }))
}
}

#[derive(Debug, Clone)]
pub struct DelFn {
paths: Vec<Path>,
path: Path,
}

impl DelFn {
#[cfg(test)]
fn new(path: Path) -> Self {
Self { path }
}
}

impl Expression for DelFn {
fn execute(&self, _: &mut state::Program, object: &mut dyn Object) -> Result<Value> {
self.paths
.iter()
.try_for_each(|path| object.remove(path.as_ref(), false))?;

Ok(Value::Null)
// TODO: we're silencing the result of the `remove` call here, to make
// this function infallible.
//
// This isn't correct though, since, while deleting Vector log
// fields is infallible, deleting metric fields is not.
//
// For example, if you try to delete `.name` in a metric event, the
// call returns an error, since this is an infallible field.
//
// After some debating, we've decided to _silently ignore_ deletions
// of infallible fields for now, but we'll circle back to this in
// the near future to potentially improve this situation.
//
// see tracking issue: <TODO>
JeanMertz marked this conversation as resolved.
Show resolved Hide resolved
Ok(object
.remove(self.path.as_ref(), false)
.ok()
.flatten()
.unwrap_or(Value::Null))
}

fn type_def(&self, _: &state::Compiler) -> TypeDef {
TypeDef {
fallible: true,
kind: value::Kind::Null,
kind: value::Kind::all(),
..Default::default()
}
}
Expand All @@ -58,15 +69,58 @@ impl Expression for DelFn {
#[cfg(test)]
mod tests {
use super::*;
use crate::map;

test_type_def![static_type_def {
expr: |_| DelFn {
paths: vec![Path::from("foo")]
},
def: TypeDef {
fallible: true,
kind: value::Kind::Null,
..Default::default()
},
}];
#[test]
fn del() {
let cases = vec![
(
// String field exists
map!["exists": "value"],
Ok(value!("value")),
DelFn::new(Path::from("exists")),
),
(
// String field doesn't exist
map!["exists": "value"],
Ok(value!(null)),
DelFn::new(Path::from("does_not_exist")),
),
(
// Array field exists
map!["exists": value!([1, 2, 3])],
Ok(value!([1, 2, 3])),
DelFn::new(Path::from("exists")),
),
(
// Null field exists
map!["exists": value!(null)],
Ok(value!(null)),
DelFn::new(Path::from("exists")),
),
(
// Map field exists
map!["exists": map!["foo": "bar"]],
Ok(value!(map!["foo": "bar"])),
DelFn::new(Path::from("exists")),
),
(
// Integer field exists
map!["exists": 127],
JeanMertz marked this conversation as resolved.
Show resolved Hide resolved
Ok(value!(127)),
DelFn::new(Path::from("exists")),
),
];

let mut state = state::Program::default();

for (object, exp, func) in cases {
let mut object: Value = object.into();
let got = func
.execute(&mut state, &mut object)
.map_err(|e| format!("{:#}", anyhow::anyhow!(e)));

assert_eq!(got, exp);
}
}
}
8 changes: 4 additions & 4 deletions lib/remap-functions/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,6 @@ pub use uuid_v4::UuidV4;

pub fn all() -> Vec<Box<dyn remap::Function>> {
vec![
#[cfg(feature = "md5")]
Box::new(Md5),
#[cfg(feature = "sha1")]
Box::new(Sha1),
#[cfg(feature = "assert")]
Box::new(Assert),
#[cfg(feature = "ceil")]
Expand Down Expand Up @@ -280,6 +276,8 @@ pub fn all() -> Vec<Box<dyn remap::Function>> {
Box::new(Ipv6ToIpV4),
#[cfg(feature = "log")]
Box::new(Log),
#[cfg(feature = "md5")]
Box::new(Md5),
#[cfg(feature = "merge")]
Box::new(Merge),
#[cfg(feature = "now")]
Expand Down Expand Up @@ -312,6 +310,8 @@ pub fn all() -> Vec<Box<dyn remap::Function>> {
Box::new(Replace),
#[cfg(feature = "round")]
Box::new(Round),
#[cfg(feature = "sha1")]
Box::new(Sha1),
#[cfg(feature = "sha2")]
Box::new(Sha2),
#[cfg(feature = "sha3")]
Expand Down
2 changes: 1 addition & 1 deletion lib/remap-functions/src/only_fields.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ impl Expression for OnlyFieldsFn {
.iter()
.map(|path| (path, path.to_string()))
.filter(|(_, key)| paths.iter().find(|p| key.starts_with(p.as_str())).is_none())
.try_for_each(|(path, _)| object.remove(&path, true))?;
.try_for_each(|(path, _)| object.remove(&path, true).map(|_| ()))?;

Ok(Value::Null)
}
Expand Down
2 changes: 1 addition & 1 deletion lib/remap-lang/src/object.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,5 +55,5 @@ pub trait Object: std::fmt::Debug {
///
/// If `compact` is true, after deletion, if an empty object or array is
/// left behind, it should be removed as well.
fn remove(&mut self, path: &Path, compact: bool) -> Result<(), String>;
fn remove(&mut self, path: &Path, compact: bool) -> Result<Option<Value>, String>;
}
27 changes: 22 additions & 5 deletions lib/remap-lang/src/value/object.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@ impl Object for Value {
self.paths().map_err(|err| err.to_string())
}

fn remove(&mut self, path: &Path, compact: bool) -> Result<(), String> {
fn remove(&mut self, path: &Path, compact: bool) -> Result<Option<Value>, String> {
let value = self.get(path)?;
self.remove_by_path(path, compact);
Ok(())

Ok(value)
}
}

Expand Down Expand Up @@ -183,10 +185,18 @@ mod tests {
#[test]
fn object_remove() {
let cases = vec![
(
value!({foo: "bar"}),
vec![Field(Regular("baz".to_owned()))],
false,
None,
Some(value!({foo: "bar"})),
),
(
value!({foo: "bar"}),
vec![Field(Regular("foo".to_owned()))],
false,
Some(value!("bar")),
Some(value!({})),
),
(
Expand All @@ -196,30 +206,35 @@ mod tests {
Regular("foo".to_owned()),
])],
false,
Some(value!("bar")),
Some(value!({})),
),
(
value!({foo: "bar", baz: "qux"}),
vec![],
false,
Some(value!({foo: "bar", baz: "qux"})),
Some(value!({})),
),
(
value!({foo: "bar", baz: "qux"}),
vec![],
true,
Some(value!({foo: "bar", baz: "qux"})),
Some(value!({})),
),
(
value!({foo: [0]}),
vec![Field(Regular("foo".to_owned())), Index(0)],
false,
Some(value!(0)),
Some(value!({foo: []})),
),
(
value!({foo: [0]}),
vec![Field(Regular("foo".to_owned())), Index(0)],
true,
Some(value!(0)),
Some(value!({})),
),
(
Expand All @@ -230,6 +245,7 @@ mod tests {
Index(0),
],
false,
Some(value!(0)),
Some(value!({foo: {"bar baz": []}, bar: "baz"})),
),
(
Expand All @@ -240,15 +256,16 @@ mod tests {
Index(0),
],
true,
Some(value!(0)),
Some(value!({bar: "baz"})),
),
];

for (mut object, segments, compact, expect) in cases {
for (mut object, segments, compact, value, expect) in cases {
let path = Path::new_unchecked(segments);

assert_eq!(Object::remove(&mut object, &path, compact), Ok(()));
assert_eq!(Object::get(&object, &Path::root()), Ok(expect))
assert_eq!(Object::remove(&mut object, &path, compact), Ok(value));
assert_eq!(Object::get(&object, &Path::root()), Ok(expect));
}
}

Expand Down
Loading