chore: update hdfs object store to 0.15 (#3681)

# Description
Update HDFS object store to 0.15. Brings some bug fixes and more
supported configs, and supports supplying a runtime for IO operations.

# Related Issue(s)
<!---
For example:

- closes #106
--->
Closes #3680 

# Documentation

https://github.com/datafusion-contrib/hdfs-native-object-store/releases/tag/v0.15.0
https://github.com/Kimahriman/hdfs-native/releases/tag/v0.12.0

Signed-off-by: Adam Binford <[email protected]>

Author: Kimahriman

crates/hdfs/Cargo.toml

@@ -13,7 +13,7 @@ rust-version.workspace = true
 
 [dependencies]
 deltalake-core = { version = "0.28.0", path = "../core"}
-hdfs-native-object-store = "0.14"
+hdfs-native-object-store = "0.15"
 
 # workspace dependecies
 object_store = { workspace = true }

crates/hdfs/src/lib.rs

@@ -1,12 +1,11 @@
 use std::sync::Arc;
 
 use deltalake_core::logstore::{
-    default_logstore, logstore_factories, DeltaIOStorageBackend, LogStore, LogStoreFactory,
-    StorageConfig,
+    default_logstore, logstore_factories, LogStore, LogStoreFactory, StorageConfig,
 };
 use deltalake_core::logstore::{object_store_factories, ObjectStoreFactory, ObjectStoreRef};
 use deltalake_core::{DeltaResult, Path};
-use hdfs_native_object_store::HdfsObjectStore;
+use hdfs_native_object_store::HdfsObjectStoreBuilder;
 use url::Url;
 
 #[derive(Clone, Default, Debug)]
@@ -18,15 +17,16 @@ impl ObjectStoreFactory for HdfsFactory {
         url: &Url,
         config: &StorageConfig,
     ) -> DeltaResult<(ObjectStoreRef, Path)> {
-        let mut store: ObjectStoreRef = Arc::new(HdfsObjectStore::with_config(
-            url.as_str(),
-            config.raw.clone(),
-        )?);
+        let mut builder = HdfsObjectStoreBuilder::new()
+            .with_url(url.as_str())
+            .with_config(&config.raw);
 
-        // HDFS doesn't have the spawnService, so we still wrap it in the old io storage backend (not as optimal though)
         if let Some(runtime) = &config.runtime {
-            store = Arc::new(DeltaIOStorageBackend::new(store, runtime.clone()));
-        };
+            builder = builder.with_io_runtime(runtime.get_handle());
+        }
+
+        let store = Arc::new(builder.build()?);
+
         let prefix = Path::parse(url.path())?;
         Ok((store, prefix))
     }