misc: Create expression benchmark for default engine (#1220)

## What changes are proposed in this pull request?

#1192 added a makeshift benchmark test to measure the performance of the
new `ToJson` expression. This PR migrates this test to a dedicated
`expression_bench` for the default engine.

This PR only adds `ToJson` to `expression_bench`, but this can be
extended as necessary in future PRs.

## How was this change tested?

N/A - this PR adds a performance benchmark.

---------

Co-authored-by: Zach Schuermann <[email protected]>

Author: lbhm

kernel/Cargo.toml

@@ -152,3 +152,7 @@ tracing-subscriber = { version = "0.3", default-features = false, features = [
 [[bench]]
 name = "metadata_bench"
 harness = false
+
+[[bench]]
+name = "expression_bench"
+harness = false

kernel/benches/expression_bench.rs

@@ -0,0 +1,177 @@
+//! Benchmark for expression evaluation performance with the default engine.
+//!
+//! You can run this benchmark with `cargo bench --bench expression_bench`.
+//!
+//! To compare your changes vs. latest main, you can:
+//! ```bash
+//! # checkout baseline branch (upstream/main) and save as baseline
+//! git checkout main # or upstream/main, another branch, etc.
+//! cargo bench --bench expression_bench -- --save-baseline main
+//!
+//! # switch back to your changes, and compare against baseline
+//! git checkout your-branch
+//! cargo bench --bench expression_bench -- --baseline main
+//! ```
+
+use std::hint::black_box;
+use std::sync::Arc;
+
+use delta_kernel::arrow::array::{
+    ArrayRef, BooleanBuilder, Float64Builder, Int32Builder, StringBuilder, StructArray,
+};
+use delta_kernel::arrow::datatypes::{DataType, Field, Fields};
+use delta_kernel::engine::arrow_expression::evaluate_expression::to_json;
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+
+/// Creates a test struct array with realistic data for benchmarking.
+fn create_test_struct_array(num_rows: usize) -> StructArray {
+    let mut id_builder = Int32Builder::with_capacity(num_rows);
+    let mut name_builder = StringBuilder::with_capacity(num_rows, num_rows * 20);
+    let mut score_builder = Float64Builder::with_capacity(num_rows);
+    let mut active_builder = BooleanBuilder::with_capacity(num_rows);
+
+    for i in 0..num_rows {
+        id_builder.append_value(i as i32);
+        name_builder.append_value(format!("user_{i}"));
+        score_builder.append_value((i as f64) * 0.1 + 100.0);
+        active_builder.append_value(i % 3 != 0);
+    }
+
+    let fields = Fields::from(vec![
+        Arc::new(Field::new("id", DataType::Int32, false)),
+        Arc::new(Field::new("name", DataType::Utf8, false)),
+        Arc::new(Field::new("score", DataType::Float64, false)),
+        Arc::new(Field::new("active", DataType::Boolean, false)),
+    ]);
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(id_builder.finish()),
+        Arc::new(name_builder.finish()),
+        Arc::new(score_builder.finish()),
+        Arc::new(active_builder.finish()),
+    ];
+
+    StructArray::new(fields, arrays, None)
+}
+
+/// Creates a simple struct array with fewer fields for lightweight benchmarking.
+fn create_simple_struct_array(num_rows: usize) -> StructArray {
+    let mut id_builder = Int32Builder::with_capacity(num_rows);
+    let mut name_builder = StringBuilder::with_capacity(num_rows, num_rows * 10);
+
+    for i in 0..num_rows {
+        id_builder.append_value(i as i32);
+        name_builder.append_value(format!("item_{i}"));
+    }
+
+    let fields = Fields::from(vec![
+        Arc::new(Field::new("id", DataType::Int32, false)),
+        Arc::new(Field::new("name", DataType::Utf8, false)),
+    ]);
+
+    let arrays: Vec<ArrayRef> = vec![
+        Arc::new(id_builder.finish()),
+        Arc::new(name_builder.finish()),
+    ];
+
+    StructArray::new(fields, arrays, None)
+}
+
+/// Creates a nested struct array for complex JSON benchmarking.
+fn create_nested_struct_array(num_rows: usize) -> StructArray {
+    // Create inner struct
+    let mut inner_int_builder = Int32Builder::with_capacity(num_rows);
+    let mut inner_string_builder = StringBuilder::with_capacity(num_rows, num_rows * 15);
+
+    for i in 0..num_rows {
+        inner_int_builder.append_value(i as i32 * 10);
+        inner_string_builder.append_value(format!("inner_{i}"));
+    }
+
+    let inner_fields = Fields::from(vec![
+        Arc::new(Field::new("inner_int", DataType::Int32, true)),
+        Arc::new(Field::new("inner_string", DataType::Utf8, true)),
+    ]);
+
+    let inner_arrays: Vec<ArrayRef> = vec![
+        Arc::new(inner_int_builder.finish()),
+        Arc::new(inner_string_builder.finish()),
+    ];
+
+    let inner_struct = Arc::new(StructArray::new(inner_fields.clone(), inner_arrays, None));
+
+    // Create outer struct
+    let mut outer_id_builder = Int32Builder::with_capacity(num_rows);
+    for i in 0..num_rows {
+        outer_id_builder.append_value(i as i32);
+    }
+
+    let fields = Fields::from(vec![
+        Arc::new(Field::new("outer_id", DataType::Int32, false)),
+        Arc::new(Field::new(
+            "nested_struct",
+            DataType::Struct(inner_fields),
+            true,
+        )),
+    ]);
+
+    let arrays: Vec<ArrayRef> = vec![Arc::new(outer_id_builder.finish()), inner_struct];
+
+    StructArray::new(fields, arrays, None)
+}
+
+fn to_json_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("to_json");
+
+    // Test different sizes for scalability analysis
+    let test_sizes = [100, 1_000, 10_000, 100_000, 1_000_000];
+
+    for &size in &test_sizes {
+        group.throughput(Throughput::Elements(size as u64));
+
+        // Benchmark simple struct array
+        let simple_struct = create_simple_struct_array(size);
+        group.bench_with_input(
+            BenchmarkId::new("simple_struct", size),
+            &simple_struct,
+            |b, struct_array| {
+                b.iter(|| {
+                    let result = to_json(black_box(struct_array));
+                    black_box(result).unwrap()
+                })
+            },
+        );
+
+        // Benchmark complex struct array
+        let complex_struct = create_test_struct_array(size);
+        group.bench_with_input(
+            BenchmarkId::new("complex_struct", size),
+            &complex_struct,
+            |b, struct_array| {
+                b.iter(|| {
+                    let result = to_json(black_box(struct_array));
+                    black_box(result).unwrap()
+                })
+            },
+        );
+
+        // Benchmark nested struct array
+        let nested_struct = create_nested_struct_array(size);
+        group.bench_with_input(
+            BenchmarkId::new("nested_struct", size),
+            &nested_struct,
+            |b, struct_array| {
+                b.iter(|| {
+                    let result = to_json(black_box(struct_array));
+                    black_box(result).unwrap()
+                })
+            },
+        );
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, to_json_benchmark);
+criterion_main!(benches);

kernel/src/engine/arrow_expression/tests.rs

@@ -1,9 +1,8 @@
 use std::ops::{Add, Div, Mul, Sub};
 
 use crate::arrow::array::{
-    create_array, Array, ArrayRef, AsArray, BooleanArray, GenericStringArray, Int32Array,
-    Int32Builder, ListArray, MapArray, MapBuilder, MapFieldNames, StringArray, StringBuilder,
-    StructArray,
+    create_array, Array, ArrayRef, BooleanArray, GenericStringArray, Int32Array, Int32Builder,
+    ListArray, MapArray, MapBuilder, MapFieldNames, StringArray, StringBuilder, StructArray,
 };
 use crate::arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer};
 use crate::arrow::compute::kernels::cmp::{gt_eq, lt};
@@ -1109,76 +1108,3 @@ fn test_to_json_with_nested_struct() {
         r#"{"outer_int":200,"nested_struct":{"inner_string":"value"}}"#
     );
 }
-
-#[test]
-#[ignore]
-fn benchmark_to_json_performance() {
-    use std::time::Instant;
-
-    // Create a large test struct array for performance testing
-    fn create_large_test_struct_array(num_rows: usize) -> StructArray {
-        let mut id_builder = Int32Builder::with_capacity(num_rows);
-        let mut name_builder = StringBuilder::with_capacity(num_rows, num_rows * 20);
-        let mut score_builder = crate::arrow::array::Float64Builder::with_capacity(num_rows);
-        let mut active_builder = crate::arrow::array::BooleanBuilder::with_capacity(num_rows);
-
-        for i in 0..num_rows {
-            id_builder.append_value(i as i32);
-            name_builder.append_value(format!("user_{i}"));
-            score_builder.append_value((i as f64) * 0.1 + 100.0);
-            active_builder.append_value(i % 3 != 0);
-        }
-
-        let fields = Fields::from(vec![
-            Arc::new(Field::new("id", DataType::Int32, false)),
-            Arc::new(Field::new("name", DataType::Utf8, false)),
-            Arc::new(Field::new("score", DataType::Float64, false)),
-            Arc::new(Field::new("active", DataType::Boolean, false)),
-        ]);
-
-        let arrays: Vec<ArrayRef> = vec![
-            Arc::new(id_builder.finish()),
-            Arc::new(name_builder.finish()),
-            Arc::new(score_builder.finish()),
-            Arc::new(active_builder.finish()),
-        ];
-
-        StructArray::new(fields, arrays, None)
-    }
-
-    // Test with different sizes to measure performance characteristics
-    let test_sizes = [100, 1000, 5000, 100000];
-    for &size in &test_sizes {
-        let large_struct = create_large_test_struct_array(size);
-
-        let start = Instant::now();
-        let result = to_json(&large_struct).unwrap();
-        let duration = start.elapsed();
-
-        println!(
-            "to_json processed {} rows in {:?} ({:.2} μs/row)",
-            size,
-            duration,
-            duration.as_micros() as f64 / size as f64
-        );
-
-        // Verify correctness
-        assert_eq!(result.len(), size);
-        let string_array = result.as_string::<i32>();
-
-        // Check a few sample results
-        if size > 0 {
-            let first_json = string_array.value(0);
-            assert!(first_json.contains("\"id\":0"));
-            assert!(first_json.contains("\"name\":\"user_0\""));
-            assert!(first_json.contains("\"score\":100"));
-            assert!(first_json.contains("\"active\":false"));
-        }
-
-        if size > 1 {
-            let second_json = string_array.value(1);
-            assert!(second_json.contains("\"id\":1"));
-            assert!(second_json.contains("\"name\":\"user_1\""));
-        }
-    }
-}