diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71c1d84750..5b5e64630c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,7 +39,7 @@ jobs: uses: actions-rs/toolchain@v1 with: profile: default - toolchain: "1.85" + toolchain: "1.86" override: true - uses: Swatinem/rust-cache@v2 @@ -91,7 +91,7 @@ jobs: uses: actions-rs/toolchain@v1 with: profile: default - toolchain: "1.85" + toolchain: "1.86" override: true - uses: Swatinem/rust-cache@v2 @@ -132,7 +132,7 @@ jobs: uses: actions-rs/toolchain@v1 with: profile: default - toolchain: "1.85" + toolchain: "1.86" override: true - uses: Swatinem/rust-cache@v2 @@ -194,7 +194,7 @@ jobs: uses: actions-rs/toolchain@v1 with: profile: default - toolchain: "1.85" + toolchain: "1.86" override: true - uses: Swatinem/rust-cache@v2 @@ -241,7 +241,7 @@ jobs: uses: actions-rs/toolchain@v1 with: profile: default - toolchain: "1.85" + toolchain: "1.86" override: true - uses: Swatinem/rust-cache@v2 @@ -300,7 +300,7 @@ jobs: uses: actions-rs/toolchain@v1 with: profile: default - toolchain: "1.85" + toolchain: "1.86" override: true - uses: Swatinem/rust-cache@v2 diff --git a/Cargo.toml b/Cargo.toml index bcd16fb823..331d965c81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] authors = ["Qingping Hou "] -rust-version = "1.85" +rust-version = "1.86" keywords = ["deltalake", "delta", "datalake"] readme = "README.md" edition = "2021" @@ -26,32 +26,32 @@ debug = true debug = "line-tables-only" [workspace.dependencies] -delta_kernel = { version = "0.15.1", features = [ - "arrow-55", +delta_kernel = { version = "0.15.2", features = [ + "arrow-56", "default-engine-rustls", "internal-api", ] } # arrow -arrow = { version = "55.2.0" } -arrow-arith = { version = "55.2.0" } -arrow-array = { version = "55.2.0", features = ["chrono-tz"] } -arrow-buffer = { version = "55.2.0" } -arrow-cast = { version = "55.2.0" } -arrow-ipc = { version = "55.2.0" } -arrow-json = { version = "55.2.0" } -arrow-ord = { version = "55.2.0" } -arrow-row = { version = "55.2.0" } -arrow-schema = { version = "55.2.0" } -arrow-select = { version = "55.2.0" } +arrow = { version = "56.0.0" } +arrow-arith = { version = "56.0.0" } +arrow-array = { version = "56.0.0", features = ["chrono-tz"] } +arrow-buffer = { version = "56.0.0" } +arrow-cast = { version = "56.0.0" } +arrow-ipc = { version = "56.0.0" } +arrow-json = { version = "56.0.0" } +arrow-ord = { version = "56.0.0" } +arrow-row = { version = "56.0.0" } +arrow-schema = { version = "56.0.0" } +arrow-select = { version = "56.0.0" } object_store = { version = "0.12.1" } -parquet = { version = "55.2.0" } +parquet = { version = "56.0.0" } # datafusion -datafusion = "49.0.0" -datafusion-ffi = "49.0.0" -datafusion-proto = "49.0.0" +datafusion = "50.0.0" +datafusion-ffi = "50.0.0" +datafusion-proto = "50.0.0" # serde serde = { version = "1.0.194", features = ["derive"] } @@ -93,4 +93,4 @@ arro3 = "arro3" Arro3 = "Arro3" AKS = "AKS" # to avoid using 'type' as a field name. -tpe = "tpe" +tpe = "tpe" \ No newline at end of file diff --git a/crates/core/src/delta_datafusion/expr.rs b/crates/core/src/delta_datafusion/expr.rs index 57baec6095..b6cb446d74 100644 --- a/crates/core/src/delta_datafusion/expr.rs +++ b/crates/core/src/delta_datafusion/expr.rs @@ -52,7 +52,7 @@ use crate::{DeltaResult, DeltaTableError}; /// This struct is like Datafusion's MakeArray but ensures that `element` is used rather than `item /// as the field name within the list. -#[derive(Debug)] +#[derive(Debug, Hash, PartialEq, Eq)] struct MakeParquetArray { /// The actual upstream UDF, which we're just totally cheating and using actual: MakeArray, diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index 70b87d62e8..0c2cc7c0f3 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -389,6 +389,8 @@ pub(crate) fn get_null_of_arrow_type(t: &ArrowDataType) -> DeltaResult &dyn Any { + self + } + fn generate_next_batch(&mut self) -> DataFusionResult> { if self.current_index < self.data.len() { let batch = self.data[self.current_index].clone(); diff --git a/python/src/writer.rs b/python/src/writer.rs index 4193d76f21..109be666c0 100644 --- a/python/src/writer.rs +++ b/python/src/writer.rs @@ -1,4 +1,6 @@ //! This module contains helper functions to create a LazyTableProvider from an ArrowArrayStreamReader + +use std::any::Any; use std::fmt::{self}; use std::sync::{Arc, Mutex}; @@ -66,6 +68,10 @@ impl ArrowStreamBatchGenerator { } impl LazyBatchGenerator for ArrowStreamBatchGenerator { + fn as_any(&self) -> &dyn Any { + self + } + fn generate_next_batch( &mut self, ) -> deltalake::datafusion::error::Result> {