Merge with main

Author: jorgehermo9

Cargo.lock

@@ -38,7 +38,7 @@ string_path = []
 datadog = ["datadog_filter", "datadog_grok", "datadog_search"]
 datadog_filter = ["datadog_search", "dep:regex", "dep:dyn-clone"]
 datadog_grok = ["value", "parsing", "dep:nom", "dep:peeking_take_while", "dep:serde_json", "dep:onig", "dep:lalrpop-util", "dep:thiserror", "dep:chrono", "dep:chrono-tz", "dep:percent-encoding"]
-datadog_search = ["dep:pest", "dep:pest_derive", "dep:itertools", "dep:once_cell", "dep:regex"]
+datadog_search = ["dep:pest", "dep:pest_derive", "dep:itertools", "dep:once_cell", "dep:regex", "dep:serde"]
 
 # Features that aren't used as often (default off)
 cli = ["stdlib", "dep:serde_json", "dep:thiserror", "dep:clap", "dep:exitcode", "dep:webbrowser", "dep:rustyline", "dep:prettytable-rs"]
@@ -142,11 +142,11 @@ hex = { version = "0.4", optional = true }
 hmac = { version = "0.12", optional = true }
 iana-time-zone = { version = "0.1", optional = true }
 idna = { version = "0.5", optional = true }
-indexmap = { version = "~2.4.0", default-features = false, features = ["std"], optional = true}
+indexmap = { version = "2", default-features = false, features = ["std"], optional = true}
 influxdb-line-protocol = { version = "2.0.0", optional = true }
 indoc = {version = "2", optional = true }
-itertools = { version = "0.13", default-features = false, optional = true }
-lalrpop-util = { version = "0.20", optional = true }
+itertools = { version = "0.13", default-features = false, features=["use_std"], optional = true }
+lalrpop-util = { version = "0.21", optional = true }
 linux-audit-parser = { version = "0.1.1", default-features = false, optional = true }
 mlua = { version = "0.9", default-features = false, features = ["lua54", "send", "vendored"], optional = true}
 nom = { version = "7", default-features = false, features = ["std"], optional = true }

Cargo.toml

@@ -160,6 +160,7 @@ criterion_group!(
               to_syslog_severity,
               to_unix_timestamp,
               truncate,
+              unflatten,
               unique,
               // TODO: Cannot pass a Path to bench_function
               //unnest
@@ -2789,6 +2790,31 @@ bench_function! {
     }
 }
 
+bench_function! {
+    unflatten => vrl::stdlib::Unflatten;
+
+    nested_map {
+        args: func_args![value: value!({"parent.child1": 1, "parent.child2": 2, key: "val"})],
+        want: Ok(value!({parent: {child1: 1, child2: 2}, key: "val"})),
+    }
+
+    map_and_array {
+        args: func_args![value: value!({
+            "parent.child1": [1, [2, 3]],
+            "parent.child2.grandchild1": 1,
+            "parent.child2.grandchild2": [1, [2, 3], 4],
+            "key": "val",
+        })],
+        want: Ok(value!({
+            "parent": {
+                "child1": [1, [2, 3]],
+                "child2": {"grandchild1": 1, "grandchild2": [1, [2, 3], 4]},
+            },
+            "key": "val",
+        })),
+    }
+}
+
 bench_function! {
     unique => vrl::stdlib::Unique;
 

benches/stdlib.rs

@@ -0,0 +1,3 @@
+Added `unflatten` function to inverse the result of the `flatten` function. This function is useful when you want to convert a flattened object back to its original form.
+
+authors: jorgehermo9

changelog.d/81.feature.md

@@ -0,0 +1,4 @@
+# object: { "a.b.c.d": 1, "a.b.c.e": 2, "a.b.f": 3, "a.g": 4 }
+# result: { "a.b.c.d": 1, "a.b.c.e": 2, "a.b.f": 3, "a.g": 4 }
+
+flatten(unflatten(.))

lib/tests/tests/functions/flatten/from_unflatten.vrl

@@ -0,0 +1,4 @@
+# object: { "a": { "b": { "c": { "d": 1, "e": 2 }, "f": 3 }, "g": 4 } }
+# result: { "a.b.c.d": 1, "a.b.c.e": 2, "a.b.f": 3, "a.g": 4 }
+
+flatten(.)

lib/tests/tests/functions/flatten/simple.vrl

@@ -0,0 +1,4 @@
+# object: { "a": { "b": { "c": { "d": 1, "e": 2 }, "f": 3 }, "g": 4 } }
+# result: { "a": { "b": { "c": { "d": 1, "e": 2 }, "f": 3 }, "g": 4 } }
+
+unflatten(flatten(.))

lib/tests/tests/functions/unflatten/from_flatten.vrl

@@ -0,0 +1,4 @@
+# object: { "a.b.c.d": 1, "a.b.c.e": 2, "a.b.f": 3, "a.g": 4 }
+# result: { "a": { "b": { "c": { "d": 1, "e": 2 }, "f": 3 }, "g": 4 } }
+
+unflatten(.)

lib/tests/tests/functions/unflatten/simple.vrl

@@ -1,10 +1,11 @@
 use once_cell::sync::Lazy;
 use regex::Regex;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
 use super::grammar::{unescape, DEFAULT_FIELD};
 
 /// This enum represents value comparisons that Queries might perform
-#[derive(Debug, Copy, Clone)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum Comparison {
     /// Greater than.
     Gt,
@@ -31,7 +32,7 @@ impl Comparison {
 /// This enum represents the values we might be using in a comparison, whether
 /// they are Strings, Numbers (currently only floating point numbers) or an
 /// Unbounded comparison with no terminating value.
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum ComparisonValue {
     Unbounded,
     String(String),
@@ -80,14 +81,14 @@ impl<T: AsRef<str>> From<T> for ComparisonValue {
 
 /// This enum represents the tokens in a range, including "greater than (or equal to)"
 /// for the left bracket, "less than (or equal to) in the right bracket, and range values.
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Range {
     Comparison(Comparison),
     Value(ComparisonValue),
 }
 
 /// This enum represents the AND or OR Boolean operations we might perform on QueryNodes.
-#[derive(Debug, Copy, Clone)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum BooleanType {
     And,
     Or,
@@ -133,7 +134,7 @@ impl BooleanBuilder {
 }
 
 /// QueryNodes represent specific search criteria to be enforced.
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum QueryNode {
     /// Match all documents.
     MatchAllDocs,
@@ -344,6 +345,29 @@ impl QueryNode {
     }
 }
 
+impl<'de> Deserialize<'de> for QueryNode {
+    fn deserialize<D>(deserializer: D) -> Result<QueryNode, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        use serde::de::Error;
+
+        let s = String::deserialize(deserializer)?;
+
+        s.parse::<QueryNode>()
+            .map_err(|e| D::Error::custom(e.to_string()))
+    }
+}
+
+impl Serialize for QueryNode {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(self.to_lucene().as_str())
+    }
+}
+
 /// Enum representing Lucene's concept of whether a node should occur.
 #[derive(Debug)]
 pub enum LuceneOccur {
@@ -364,3 +388,29 @@ static ESCAPE_RE: Lazy<Regex> = Lazy::new(|| Regex::new("^\"(.+)\"$").unwrap());
 fn escape_quotes<T: AsRef<str>>(value: T) -> String {
     ESCAPE_RE.replace_all(value.as_ref(), "$1").to_string()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn query_node_serializes_to_string() {
+        assert_eq!(
+            serde_json::to_string(&QueryNode::AttributeExists {
+                attr: "something".into()
+            })
+            .unwrap(),
+            r#""_exists_:something""#
+        );
+    }
+
+    #[test]
+    fn query_node_deserializes_from_string() {
+        assert_eq!(
+            serde_json::from_str::<QueryNode>(r#""_missing_:something_else""#).unwrap(),
+            QueryNode::AttributeMissing {
+                attr: "something_else".into()
+            }
+        );
+    }
+}

src/datadog/search/node.rs

@@ -204,6 +204,7 @@ cfg_if::cfg_if! {
         mod to_unix_timestamp;
         mod community_id;
         mod truncate;
+        mod unflatten;
         mod type_def;
         mod unique;
         mod unnest;
@@ -383,6 +384,7 @@ cfg_if::cfg_if! {
         pub use to_unix_timestamp::ToUnixTimestamp;
         pub use truncate::Truncate;
         pub use type_def::TypeDef;
+        pub use unflatten::Unflatten;
         pub use unique::Unique;
         pub use unnest::Unnest;
         pub use upcase::Upcase;
@@ -572,6 +574,7 @@ pub fn all() -> Vec<Box<dyn Function>> {
         Box::new(CommunityID),
         Box::new(Truncate),
         Box::new(TypeDef),
+        Box::new(Unflatten),
         Box::new(Unique),
         Box::new(Unnest),
         Box::new(Upcase),