Custom Debug impl of T: FileSource

Author: friendlymatthew

datafusion/core/src/datasource/physical_plan/arrow_file.rs

@@ -37,7 +37,7 @@ use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore};
 
 /// Arrow configuration struct that is given to DataSourceExec
 /// Does not hold anything special, since [`FileScanConfig`] is sufficient for arrow
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct ArrowSource {
     metrics: ExecutionPlanMetricsSet,
     schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
@@ -114,6 +114,15 @@ impl FileSource for ArrowSource {
     }
 }
 
+impl std::fmt::Debug for ArrowSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} {{ ", self.file_type())?;
+        write!(f, "statistics={:?}, ", self.file_source_statistics())?;
+        write!(f, "config={:?}, ", self.config())?;
+        write!(f, " }}")
+    }
+}
+
 /// The struct arrow that implements `[FileOpener]` trait
 pub struct ArrowOpener {
     pub object_store: Arc<dyn ObjectStore>,

datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs

@@ -105,7 +105,7 @@ impl FileOpener for TestOpener {
 }
 
 /// A placeholder data source that accepts filter pushdown
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct TestSource {
     support: bool,
     predicate: Option<Arc<dyn PhysicalExpr>>,
@@ -234,6 +234,15 @@ impl FileSource for TestSource {
     }
 }
 
+impl std::fmt::Debug for TestSource {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} {{ ", self.file_type())?;
+        write!(f, "statistics={:?}, ", self.file_source_statistics())?;
+        write!(f, "config={:?}, ", self.config())?;
+        write!(f, " }}")
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct TestScanBuilder {
     support: bool,

datafusion/datasource-avro/src/source.rs

@@ -35,7 +35,7 @@ use datafusion_datasource::source::DataSource;
 use object_store::ObjectStore;
 
 /// AvroSource holds the extra configuration that is necessary for opening avro files
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct AvroSource {
     metrics: ExecutionPlanMetricsSet,
     schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
@@ -132,6 +132,15 @@ impl FileSource for AvroSource {
     }
 }
 
+impl std::fmt::Debug for AvroSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} {{ ", self.file_type())?;
+        write!(f, "statistics={:?}, ", self.file_source_statistics())?;
+        write!(f, "config={:?}, ", self.config())?;
+        write!(f, " }}")
+    }
+}
+
 mod private {
     use super::*;
 

datafusion/datasource-csv/src/source.rs

@@ -19,7 +19,6 @@
 
 use datafusion_datasource::schema_adapter::SchemaAdapterFactory;
 use std::any::Any;
-use std::fmt;
 use std::io::{Read, Seek, SeekFrom};
 use std::sync::Arc;
 use std::task::Poll;
@@ -83,7 +82,7 @@ use tokio::io::AsyncWriteExt;
 
 /// let exec = (DataSourceExec::from_data_source(source));
 /// ```
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct CsvSource {
     pub(crate) has_header: bool,
     delimiter: u8,
@@ -272,7 +271,11 @@ impl FileSource for CsvSource {
     fn file_type(&self) -> &str {
         "csv"
     }
-    fn fmt_extra(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt_extra(
+        &self,
+        t: DisplayFormatType,
+        f: &mut std::fmt::Formatter,
+    ) -> std::fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
                 write!(f, ", has_header={}", self.has_header)
@@ -296,6 +299,15 @@ impl FileSource for CsvSource {
     }
 }
 
+impl std::fmt::Debug for CsvSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} {{ ", self.file_type())?;
+        write!(f, "statistics={:?}, ", self.file_source_statistics())?;
+        write!(f, "config={:?}, ", self.config())?;
+        write!(f, " }}")
+    }
+}
+
 impl FileOpener for CsvOpener {
     /// Open a partitioned CSV file.
     ///

datafusion/datasource-json/src/source.rs

@@ -75,7 +75,7 @@ impl JsonOpener {
 }
 
 /// JsonSource holds the extra configuration that is necessary for [`JsonOpener`]
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct JsonSource {
     metrics: ExecutionPlanMetricsSet,
     schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
@@ -159,6 +159,15 @@ impl FileSource for JsonSource {
     }
 }
 
+impl std::fmt::Debug for JsonSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} {{ ", self.file_type())?;
+        write!(f, "statistics={:?}, ", self.file_source_statistics())?;
+        write!(f, "config={:?}, ", self.config())?;
+        write!(f, " }}")
+    }
+}
+
 impl FileOpener for JsonOpener {
     /// Open a partitioned NDJSON file.
     ///

datafusion/datasource-parquet/src/source.rs

@@ -17,8 +17,7 @@
 
 //! ParquetSource implementation for reading parquet files
 use std::any::Any;
-use std::fmt::Debug;
-use std::fmt::Formatter;
+use std::fmt::{self, Debug, Formatter};
 use std::sync::Arc;
 
 use crate::opener::build_pruning_predicates;
@@ -269,7 +268,7 @@ use object_store::ObjectStore;
 /// [`RecordBatch`]: arrow::record_batch::RecordBatch
 /// [`SchemaAdapter`]: datafusion_datasource::schema_adapter::SchemaAdapter
 /// [`ParquetMetadata`]: parquet::file::metadata::ParquetMetaData
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct ParquetSource {
     /// Options for reading Parquet files
     pub(crate) table_parquet_options: TableParquetOptions,
@@ -447,7 +446,7 @@ impl ParquetSource {
         // If the FileScanConfig.file_source() has a schema adapter factory, apply it
         if let Some(factory) = &self.schema_adapter_factory() {
             file_source.with_schema_adapter_factory(
-                Arc::<dyn SchemaAdapterFactory>::clone(&factory),
+                Arc::<dyn SchemaAdapterFactory>::clone(factory),
             )
         } else {
             Ok(file_source)
@@ -632,7 +631,7 @@ impl FileSource for ParquetSource {
         "parquet"
     }
 
-    fn fmt_extra(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+    fn fmt_extra(&self, t: DisplayFormatType, f: &mut Formatter) -> fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
                 let predicate_string = self
@@ -767,3 +766,12 @@ impl FileSource for ParquetSource {
         self.schema_adapter_factory.clone()
     }
 }
+
+impl Debug for ParquetSource {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{} {{ ", self.file_type())?;
+        write!(f, "statistics={:?}, ", self.file_source_statistics())?;
+        write!(f, "config={:?} ", self.config())?;
+        write!(f, " }}")
+    }
+}

datafusion/datasource/src/file_scan_config.rs

@@ -756,12 +756,6 @@ impl Debug for FileScanConfig {
         write!(f, "FileScanConfig {{")?;
         write!(f, "object_store_url={:?}, ", self.object_store_url)?;
 
-        // write!(
-        //     f,
-        //     "statistics={:?}, ",
-        //     self.file_source.file_source_statistics()
-        // )?;
-
         DisplayAs::fmt_as(self, DisplayFormatType::Verbose, f)?;
         write!(f, "}}")
     }

datafusion/datasource/src/test_util.rs

@@ -27,10 +27,11 @@ use arrow::datatypes::Schema;
 use datafusion_common::Result;
 use datafusion_physical_expr::{expressions::Column, PhysicalExpr};
 use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
+use datafusion_physical_plan::DisplayFormatType;
 use object_store::ObjectStore;
 
 /// Minimal [`crate::file::FileSource`] implementation for use in tests.
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub(crate) struct MockSource {
     metrics: ExecutionPlanMetricsSet,
     schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
@@ -99,6 +100,15 @@ impl FileSource for MockSource {
     }
 }
 
+impl std::fmt::Debug for MockSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} {{ ", self.file_type())?;
+        write!(f, "statistics={:?}, ", self.file_source_statistics())?;
+        <Self as DataSource>::fmt_as(self, DisplayFormatType::Verbose, f)?;
+        write!(f, " }}")
+    }
+}
+
 /// Create a column expression
 pub(crate) fn col(name: &str, schema: &Schema) -> Result<Arc<dyn PhysicalExpr>> {
     Ok(Arc::new(Column::new_with_schema(name, schema)?))