feat: allow extension DataSources to decode via a per-type registry

datafusion/datasource/Cargo.toml

@@ -39,6 +39,7 @@ default = ["compression"]
 # consumers that never serialize plans pay nothing. Mirrors the `proto` feature
 # on `datafusion-physical-plan`.
 proto = [
+    "dep:prost",
     "dep:datafusion-proto-models",
     "datafusion-physical-plan/proto",
 ]
@@ -75,6 +76,7 @@ log = { workspace = true }
 object_store = { workspace = true }
 parking_lot = { workspace = true }
 pin-project-lite = { workspace = true }
+prost = { workspace = true, optional = true }
 rand = { workspace = true }
 tempfile = { workspace = true, optional = true }
 tokio = { workspace = true }

datafusion/datasource/src/mod.rs

@@ -44,7 +44,7 @@ pub mod projection;
 /// Protobuf conversions for [`FileRange`], [`PartitionedFile`] and
 /// [`FileGroup`], gated on the `proto` feature.
 #[cfg(feature = "proto")]
-mod proto;
+pub mod proto;
 pub mod schema_adapter;
 pub mod sink;
 pub mod source;

datafusion/datasource/src/proto.rs

@@ -15,6 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Serialization support for data sources.
+//!
+//! Custom sources use the name-keyed decoder API in [`registry`](crate::proto::registry).
+//!
 //! Protobuf conversions for the file-scan leaf types owned by this crate:
 //! [`FileRange`], [`PartitionedFile`] and [`FileGroup`].
 //!
@@ -30,6 +34,14 @@
 //! the value, and the orphan rule allows it here because one side of each
 //! conversion is a type this crate owns.
 
+pub mod registry;
+
+pub use datafusion_proto_models::ProtoDecoderRegistry;
+pub use registry::{
+    ExtensionDataSourceFromProto, data_source_names, extension_data_source_node,
+    register_data_source,
+};
+
 use std::sync::Arc;
 
 use chrono::{TimeZone, Utc};