✨ Flatten results of VRL scripts

Taskfile.yml

@@ -11,7 +11,8 @@ tasks:
   test:
     desc: "Run tests"
     cmds:
-      - go test -v ./...
+      - task: "test:go"
+      - task: "test:rust"
 
   doc:
     desc: "Generate documentation"
@@ -91,3 +92,26 @@ tasks:
     dir: ./web/components/code-editor
     cmds:
       - npm i
+
+  "test:go":
+    internal: true
+    cmds:
+      - go test ./...
+
+  "test:rust":
+    internal: true
+    cmds:
+      - task: "test:rust:filterdsl"
+      - task: "test:rust:vrl"
+
+  "test:rust:filterdsl":
+    internal: true
+    dir: ./internal/ffi/filterdsl/rust-crate
+    cmds:
+      - cargo test
+
+  "test:rust:vrl":
+    internal: true
+    dir: ./internal/ffi/vrl/rust-crate
+    cmds:
+      - cargo test

docker/flowg.dockerfile

@@ -50,6 +50,7 @@ COPY --from=sources-rust-filterdsl /src /workspace
 WORKDIR /workspace/internal/ffi/filterdsl/rust-crate
 
 RUN cargo build --release
+RUN cargo test
 
 ## VRL
 FROM rust:1.81-alpine3.20 AS builder-rust-vrl
@@ -59,6 +60,7 @@ COPY --from=sources-rust-vrl /src /workspace
 WORKDIR /workspace/internal/ffi/vrl/rust-crate
 
 RUN cargo build --release
+RUN cargo test
 
 ##############################
 ## BUILD JS DEPENDENCIES

internal/ffi/vrl/ffi_test.go

@@ -0,0 +1,33 @@
+package vrl_test
+
+import (
+	"reflect"
+	"testing"
+
+	"link-society.com/flowg/internal/ffi/vrl"
+)
+
+func TestProcessRecord(t *testing.T) {
+	input := map[string]string{}
+	script := `
+		.foo = "bar"
+		.bar.baz = [1, 2, 3, "a"]
+		.
+	`
+
+	output, err := vrl.ProcessRecord(input, script)
+	if err != nil {
+		t.Errorf("ProcessRecord() failed: %v", err)
+	}
+
+	expected := map[string]string{
+		"foo":       "bar",
+		"bar.baz.0": "1",
+		"bar.baz.1": "2",
+		"bar.baz.2": "3",
+		"bar.baz.3": "a",
+	}
+	if !reflect.DeepEqual(output, expected) {
+		t.Errorf("ProcessRecord() = %v, want %v", output, expected)
+	}
+}

internal/ffi/vrl/rust-crate/src/runner.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 
 use anyhow::{Context, Result};
 use vrl::prelude::*;
@@ -39,15 +39,136 @@ pub fn process_record(
   let vrl_result = compiled.program.resolve(&mut ctx)
     .context("Failed to execute VRL script")?;
 
+  let result = if let Value::Object(obj) = vrl_result {
+    flatten_obj(&obj)
+  } else {
+    HashMap::new()
+  };
+
+  Ok(result)
+}
+
+fn flatten_obj(obj: &BTreeMap<KeyString, Value>) -> HashMap<String, String> {
   let mut result = HashMap::new();
 
-  if let Value::Object(obj) = vrl_result {
-    for (key, value) in obj.iter() {
-      if let Some(s) = value.as_str() {
-        result.insert(key.to_string(), s.to_string());
-      }
-    }
+  for (key, value) in obj.iter() {
+    match value {
+      Value::Bytes(_) => {
+        if let Some(s) = value.as_str() {
+          result.insert(key.to_string(), s.to_string());
+        }
+        else {
+          result.insert(key.to_string(), "#ERROR#".to_string());
+        }
+      },
+      Value::Regex(r) => {
+        result.insert(key.to_string(), r.as_str().to_string());
+      },
+      Value::Timestamp(ts) => {
+        result.insert(key.to_string(), ts.to_string());
+      },
+      Value::Integer(i) => {
+        result.insert(key.to_string(), i.to_string());
+      },
+      Value::Float(f) => {
+        result.insert(key.to_string(), f.to_string());
+      },
+      Value::Boolean(b) => {
+        result.insert(key.to_string(), b.to_string());
+      },
+      Value::Object(obj) => {
+        let inner = flatten_obj(obj);
+
+        for (inner_key, inner_value) in inner.iter() {
+          result.insert(format!("{}.{}", key, inner_key), inner_value.to_string());
+        }
+      },
+      Value::Array(arr) => {
+        let inner = flatten_array(arr);
+
+        for (inner_key, inner_value) in inner.iter() {
+          result.insert(format!("{}.{}", key, inner_key), inner_value.to_string());
+        }
+      },
+      Value::Null => {
+        result.insert(key.to_string(), "".to_string());
+      },
+    };
   }
 
-  Ok(result)
+  result
+}
+
+fn flatten_array(array: &Vec<Value>) -> HashMap<String, String> {
+  let mut result = HashMap::new();
+
+  for (key, value) in array.iter().enumerate() {
+    match value {
+      Value::Bytes(_) => {
+        if let Some(s) = value.as_str() {
+          result.insert(key.to_string(), s.to_string());
+        }
+        else {
+          result.insert(key.to_string(), "#ERROR#".to_string());
+        }
+      },
+      Value::Regex(r) => {
+        result.insert(key.to_string(), r.as_str().to_string());
+      },
+      Value::Timestamp(ts) => {
+        result.insert(key.to_string(), ts.to_string());
+      },
+      Value::Integer(i) => {
+        result.insert(key.to_string(), i.to_string());
+      },
+      Value::Float(f) => {
+        result.insert(key.to_string(), f.to_string());
+      },
+      Value::Boolean(b) => {
+        result.insert(key.to_string(), b.to_string());
+      },
+      Value::Object(obj) => {
+        let inner = flatten_obj(obj);
+
+        for (inner_key, inner_value) in inner.iter() {
+          result.insert(format!("{}.{}", key, inner_key), inner_value.to_string());
+        }
+      },
+      Value::Array(arr) => {
+        let inner = flatten_array(arr);
+
+        for (inner_key, inner_value) in inner.iter() {
+          result.insert(format!("{}.{}", key, inner_key), inner_value.to_string());
+        }
+      },
+      Value::Null => {
+        result.insert(key.to_string(), "".to_string());
+      },
+    };
+  }
+
+  result
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn test_process_record() {
+    let input = HashMap::new();
+    let script = String::from(r#"
+      .foo = "x"
+      .bar.baz = [1, 2, 3, "a"]
+      .
+    "#);
+
+    let output = process_record(input, script).unwrap();
+
+    assert_eq!(output.get("foo"), Some(&"x".to_string()));
+    assert_eq!(output.get("bar.baz.0"), Some(&"1".to_string()));
+    assert_eq!(output.get("bar.baz.1"), Some(&"2".to_string()));
+    assert_eq!(output.get("bar.baz.2"), Some(&"3".to_string()));
+    assert_eq!(output.get("bar.baz.3"), Some(&"a".to_string()));
+  }
 }