bump datafusion

delta-io · Feb 25, 2024 · fcb1825 · fcb1825
1 parent 7357098
commit fcb1825
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 15 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -46,12 +46,12 @@ object_store = { version = "0.9" }
 parquet = { version = "50" }
 
 # datafusion
-datafusion = { version = "35" }
-datafusion-expr = { version = "35" }
-datafusion-common = { version = "35" }
-datafusion-proto = { version = "35" }
-datafusion-sql = { version = "35" }
-datafusion-physical-expr = { version = "35" }
+datafusion = { version = "36" }
+datafusion-expr = { version = "36" }
+datafusion-common = { version = "36" }
+datafusion-proto = { version = "36" }
+datafusion-sql = { version = "36" }
+datafusion-physical-expr = { version = "36" }
 
 # serde
 serde = { version = "1.0.194", features = ["derive"] }

diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
@@ -97,7 +97,7 @@ reqwest = { version = "0.11.18", default-features = false, features = [
     "rustls-tls",
     "json",
 ], optional = true }
-sqlparser = { version = "0.41", optional = true }
+sqlparser = { version = "0.43.1", optional = true }
 
 [dev-dependencies]
 criterion = "0.5"

diff --git a/crates/core/src/data_catalog/unity/datafusion.rs b/crates/core/src/data_catalog/unity/datafusion.rs
@@ -6,7 +6,7 @@ use std::sync::Arc;
 
 use dashmap::DashMap;
 use datafusion::catalog::schema::SchemaProvider;
-use datafusion::catalog::{CatalogList, CatalogProvider};
+use datafusion::catalog::{CatalogProvider, CatalogProviderList};
 use datafusion::datasource::TableProvider;
 use tracing::error;
 
@@ -49,7 +49,7 @@ impl UnityCatalogList {
     }
 }
 
-impl CatalogList for UnityCatalogList {
+impl CatalogProviderList for UnityCatalogList {
     fn as_any(&self) -> &dyn Any {
         self
     }

diff --git a/crates/core/src/delta_datafusion/expr.rs b/crates/core/src/delta_datafusion/expr.rs
@@ -275,13 +275,18 @@ impl<'a> Display for SqlFormat<'a> {
                 datafusion_expr::GetFieldAccess::ListIndex { key } => {
                     write!(f, "{}[{}]", SqlFormat { expr }, SqlFormat { expr: key })
                 }
-                datafusion_expr::GetFieldAccess::ListRange { start, stop } => {
+                datafusion_expr::GetFieldAccess::ListRange {
+                    start,
+                    stop,
+                    stride,
+                } => {
                     write!(
                         f,
-                        "{}[{}:{}]",
+                        "{}[{}:{}:{}]",
                         SqlFormat { expr },
                         SqlFormat { expr: start },
-                        SqlFormat { expr: stop }
+                        SqlFormat { expr: stop },
+                        SqlFormat { expr: stride }
                     )
                 }
             },

diff --git a/crates/core/src/kernel/snapshot/log_data.rs b/crates/core/src/kernel/snapshot/log_data.rs
@@ -466,7 +466,7 @@ mod datafusion {
     use arrow_arith::aggregate::sum;
     use arrow_array::Int64Array;
     use arrow_schema::DataType as ArrowDataType;
-    use datafusion_common::scalar::ScalarValue;
+    use datafusion_common::scalar::{ScalarStructBuilder, ScalarValue};
     use datafusion_common::stats::{ColumnStatistics, Precision, Statistics};
     use datafusion_expr::AggregateFunction;
     use datafusion_physical_expr::aggregate::AggregateExpr;
@@ -549,7 +549,13 @@ mod datafusion {
                             _ => None,
                         })
                         .collect::<Option<Vec<_>>>()
-                        .map(|o| Precision::Exact(ScalarValue::Struct(Some(o), fields.clone())))
+                        .map(|o| {
+                            let mut s1 = ScalarStructBuilder::new();
+                            for (scalar_value, col) in o.iter().zip(fields.iter()) {
+                                s1 = s1.with_scalar(col, scalar_value.to_owned())
+                            }
+                            Precision::Exact(s1.build().unwrap())
+                        })
                         .unwrap_or(Precision::Absent);
                 }
                 _ => Precision::Absent,

diff --git a/crates/core/src/operations/convert_to_delta.rs b/crates/core/src/operations/convert_to_delta.rs
@@ -226,7 +226,7 @@ impl ConvertToDeltaBuilder {
     }
 
     /// Consume self into CreateBuilder with corresponding add actions, schemas and operation meta
-    async fn into_create_builder(mut self) -> Result<CreateBuilder, Error> {
+    async fn into_create_builder(self) -> Result<CreateBuilder, Error> {
         // Use the specified log store. If a log store is not provided, create a new store from the specified path.
         // Return an error if neither log store nor path is provided
         let log_store = if let Some(log_store) = self.log_store {