From ad42523ff637a5975c2bce6c4aaba91466bd2ec7 Mon Sep 17 00:00:00 2001 From: Vaibhave Sekar Date: Sun, 9 Aug 2026 06:07:32 +0000 Subject: [PATCH 01/20] fix: scope information schema to catalog --- datafusion/catalog/src/information_schema.rs | 164 +++++++----------- .../core/src/execution/session_state.rs | 27 +-- .../information_schema_multiple_catalogs.slt | 31 ++-- 3 files changed, 96 insertions(+), 126 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index d9ad7791af67c..d399f49e4ed7e 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -21,7 +21,7 @@ use crate::streaming::StreamingTable; use crate::table::TableFunction; -use crate::{CatalogProviderList, SchemaProvider, TableProvider}; +use crate::{CatalogProvider, SchemaProvider, TableProvider}; use arrow::array::builder::{BooleanBuilder, UInt8Builder}; use arrow::{ array::{StringBuilder, UInt64Builder}, @@ -79,11 +79,15 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { - /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` - pub fn new(catalog_list: Arc) -> Self { + /// Creates a new [`InformationSchemaProvider`] for the specified catalog. + pub fn new( + catalog_name: impl Into, + catalog: Arc, + ) -> Self { Self { config: InformationSchemaConfig { - catalog_list, + catalog_name: catalog_name.into(), + catalog, table_functions: HashMap::new(), }, } @@ -102,7 +106,8 @@ impl InformationSchemaProvider { #[derive(Clone, Debug)] struct InformationSchemaConfig { - catalog_list: Arc, + catalog_name: String, + catalog: Arc, table_functions: HashMap>, } @@ -114,54 +119,44 @@ impl InformationSchemaConfig { ) -> Result<(), DataFusionError> { // create a mem table with the names of tables - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table_type) = - schema.table_type(&table_name).await? - { - builder.add_table( - &catalog_name, - &schema_name, - &table_name, - table_type, - ); - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = self.catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table_type) = schema.table_type(&table_name).await? { + builder.add_table( + &self.catalog_name, + &schema_name, + &table_name, + table_type, + ); } } } } + } - // Add a final list for the information schema tables themselves - for table_name in INFORMATION_SCHEMA_TABLES { - builder.add_table( - &catalog_name, - INFORMATION_SCHEMA, - table_name, - TableType::View, - ); - } + // Add a final list for the information schema tables themselves + for table_name in INFORMATION_SCHEMA_TABLES { + builder.add_table( + &self.catalog_name, + INFORMATION_SCHEMA, + table_name, + TableType::View, + ); } Ok(()) } fn make_schemata(&self, builder: &mut InformationSchemataBuilder) { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA - && let Some(schema) = catalog.schema(&schema_name) - { - let schema_owner = schema.owner_name(); - builder.add_schemata(&catalog_name, &schema_name, schema_owner); - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA + && let Some(schema) = self.catalog.schema(&schema_name) + { + let schema_owner = schema.owner_name(); + builder.add_schemata(&self.catalog_name, &schema_name, schema_owner); } } } @@ -170,22 +165,18 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaViewBuilder, ) -> Result<(), DataFusionError> { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - builder.add_view( - &catalog_name, - &schema_name, - &table_name, - table.get_table_definition(), - ) - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = self.catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + builder.add_view( + &self.catalog_name, + &schema_name, + &table_name, + table.get_table_definition(), + ) } } } @@ -200,26 +191,22 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaColumnsBuilder, ) -> Result<(), DataFusionError> { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - for (field_position, field) in - table.schema().fields().iter().enumerate() - { - builder.add_column( - &catalog_name, - &schema_name, - &table_name, - field_position, - field, - ) - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = self.catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + for (field_position, field) in + table.schema().fields().iter().enumerate() + { + builder.add_column( + &self.catalog_name, + &schema_name, + &table_name, + field_position, + field, + ) } } } @@ -1564,7 +1551,8 @@ mod tests { #[tokio::test] async fn make_tables_uses_table_type() { let config = InformationSchemaConfig { - catalog_list: Arc::new(Fixture), + catalog_name: "acatalog".to_string(), + catalog: Arc::new(Fixture), table_functions: HashMap::new(), }; let mut builder = InformationSchemaTablesBuilder { @@ -1607,24 +1595,6 @@ mod tests { } } - impl CatalogProviderList for Fixture { - fn register_catalog( - &self, - _: String, - _: Arc, - ) -> Option> { - unimplemented!("not required for these tests") - } - - fn catalog_names(&self) -> Vec { - vec!["acatalog".to_string()] - } - - fn catalog(&self, _: &str) -> Option> { - Some(Arc::new(Self)) - } - } - impl CatalogProvider for Fixture { fn schema_names(&self) -> Vec { vec!["aschema".to_string()] diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index bfd38faacf816..782205dc9c003 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -369,26 +369,27 @@ impl SessionState { table_ref: impl Into, ) -> datafusion_common::Result> { let resolved_ref = self.resolve_table_ref(table_ref); + let catalog = self + .catalog_list + .catalog(&resolved_ref.catalog) + .ok_or_else(|| { + plan_datafusion_err!( + "failed to resolve catalog: {}", + resolved_ref.catalog + ) + })?; + if self.config.information_schema() && *resolved_ref.schema == *INFORMATION_SCHEMA { return Ok(Arc::new( - InformationSchemaProvider::new(Arc::clone(&self.catalog_list)) + InformationSchemaProvider::new(resolved_ref.catalog.to_string(), catalog) .with_table_functions(self.table_functions.clone()), )); } - self.catalog_list - .catalog(&resolved_ref.catalog) - .ok_or_else(|| { - plan_datafusion_err!( - "failed to resolve catalog: {}", - resolved_ref.catalog - ) - })? - .schema(&resolved_ref.schema) - .ok_or_else(|| { - plan_datafusion_err!("failed to resolve schema: {}", resolved_ref.schema) - }) + catalog.schema(&resolved_ref.schema).ok_or_else(|| { + plan_datafusion_err!("failed to resolve schema: {}", resolved_ref.schema) + }) } /// Add `analyzer_rule` to the end of the list of diff --git a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt index f57ecbc32a57a..8a204a1df0763 100644 --- a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt +++ b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt @@ -77,25 +77,24 @@ set datafusion.catalog.default_schema = my_other_schema; statement ok create table t3 as values(1); +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_other_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_other_catalog my_other_schema t3 + query TTTT rowsort SELECT * from information_schema.tables; ---- -datafusion information_schema columns VIEW -datafusion information_schema df_settings VIEW -datafusion information_schema parameters VIEW -datafusion information_schema routines VIEW -datafusion information_schema schemata VIEW -datafusion information_schema tables VIEW -datafusion information_schema views VIEW -my_catalog information_schema columns VIEW -my_catalog information_schema df_settings VIEW -my_catalog information_schema parameters VIEW -my_catalog information_schema routines VIEW -my_catalog information_schema schemata VIEW -my_catalog information_schema tables VIEW -my_catalog information_schema views VIEW -my_catalog my_schema t1 BASE TABLE -my_catalog my_schema t2 BASE TABLE my_other_catalog information_schema columns VIEW my_other_catalog information_schema df_settings VIEW my_other_catalog information_schema parameters VIEW From ea2d35f32fe9549135a64e1d1a0211a2c7ac86a1 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 9 Sep 2026 18:28:43 +0200 Subject: [PATCH 02/20] Add a 'system' catalog that provides a global information schema --- datafusion-cli/src/main.rs | 5 +- datafusion/catalog/src/information_schema.rs | 283 +++++++++++++----- datafusion/common/src/config.rs | 3 + datafusion/core/src/execution/context/mod.rs | 4 +- .../core/src/execution/session_state.rs | 112 +++++-- datafusion/execution/src/config.rs | 14 +- datafusion/sql/src/statement.rs | 70 +++-- 7 files changed, 366 insertions(+), 125 deletions(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index f82206a5bd184..f08cb6eeb37ef 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -370,8 +370,9 @@ fn get_session_config(args: &Args) -> Result { config_options.sql_parser.dialect = Dialect::Spark; } - let mut session_config = - SessionConfig::from(config_options).with_information_schema(true); + let mut session_config = SessionConfig::from(config_options) + .with_system_catalog(Some(String::from("system"))) + .with_information_schema(true); if args.reads_sql_from_stdin() { // When stdin carries the session's SQL — the REPL (including any rc diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index d399f49e4ed7e..9fd70cf9f7d0b 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -21,7 +21,7 @@ use crate::streaming::StreamingTable; use crate::table::TableFunction; -use crate::{CatalogProvider, SchemaProvider, TableProvider}; +use crate::{CatalogProviderList, SchemaProvider, TableProvider}; use arrow::array::builder::{BooleanBuilder, UInt8Builder}; use arrow::{ array::{StringBuilder, UInt64Builder}, @@ -79,15 +79,12 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { - /// Creates a new [`InformationSchemaProvider`] for the specified catalog. - pub fn new( - catalog_name: impl Into, - catalog: Arc, - ) -> Self { + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` + pub fn new(catalog_list: Arc) -> Self { Self { config: InformationSchemaConfig { - catalog_name: catalog_name.into(), - catalog, + system_catalog_name: None, + catalog_list, table_functions: HashMap::new(), }, } @@ -102,12 +99,17 @@ impl InformationSchemaProvider { self.config.table_functions = table_functions; self } + + pub fn with_system_catalog(mut self, catalog_name: String) -> Self { + self.config.system_catalog_name = Some(catalog_name); + self + } } #[derive(Clone, Debug)] struct InformationSchemaConfig { - catalog_name: String, - catalog: Arc, + system_catalog_name: Option, + catalog_list: Arc, table_functions: HashMap>, } @@ -119,45 +121,74 @@ impl InformationSchemaConfig { ) -> Result<(), DataFusionError> { // create a mem table with the names of tables - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = self.catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table_type) = schema.table_type(&table_name).await? { - builder.add_table( - &self.catalog_name, - &schema_name, - &table_name, - table_type, - ); + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table_type) = + schema.table_type(&table_name).await? + { + builder.add_table( + &catalog_name, + &schema_name, + &table_name, + table_type, + ); + } } } } } + + // Add a final list for the information schema tables themselves + for table_name in INFORMATION_SCHEMA_TABLES { + builder.add_table( + &catalog_name, + INFORMATION_SCHEMA, + table_name, + TableType::View, + ); + } } - // Add a final list for the information schema tables themselves - for table_name in INFORMATION_SCHEMA_TABLES { - builder.add_table( - &self.catalog_name, - INFORMATION_SCHEMA, - table_name, - TableType::View, - ); + if let Some(system_catalog_name) = &self.system_catalog_name { + for table_name in INFORMATION_SCHEMA_TABLES { + builder.add_table( + system_catalog_name, + INFORMATION_SCHEMA, + table_name, + TableType::View, + ); + } } Ok(()) } fn make_schemata(&self, builder: &mut InformationSchemataBuilder) { - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA - && let Some(schema) = self.catalog.schema(&schema_name) - { - let schema_owner = schema.owner_name(); - builder.add_schemata(&self.catalog_name, &schema_name, schema_owner); + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA + && let Some(schema) = catalog.schema(&schema_name) + { + let schema_owner = schema.owner_name(); + builder.add_schemata(&catalog_name, &schema_name, schema_owner); + } } + + // Add the information schema itself + builder.add_schemata(&catalog_name, INFORMATION_SCHEMA, None); + } + + // Add the system information schema + if let Some(system_catalog_name) = &self.system_catalog_name { + builder.add_schemata(system_catalog_name, INFORMATION_SCHEMA, None); } } @@ -165,24 +196,55 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaViewBuilder, ) -> Result<(), DataFusionError> { - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = self.catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - builder.add_view( - &self.catalog_name, - &schema_name, - &table_name, - table.get_table_definition(), - ) - } + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + Self::add_views( + builder, + &catalog_name, + &schema_name, + schema.as_ref(), + ) + .await?; } } + + // Add the information schema views themselves + Self::add_views(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; } } + // Add the system information schema + if let Some(system_catalog_name) = &self.system_catalog_name { + Self::add_views(builder, system_catalog_name, INFORMATION_SCHEMA, self) + .await?; + } + + Ok(()) + } + + async fn add_views( + builder: &mut InformationSchemaViewBuilder, + catalog_name: &str, + schema_name: &str, + schema: &dyn SchemaProvider, + ) -> Result<(), DataFusionError> { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? + && table.table_type() == TableType::View + { + builder.add_view( + catalog_name, + schema_name, + &table_name, + table.get_table_definition(), + ) + } + } Ok(()) } @@ -191,32 +253,60 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaColumnsBuilder, ) -> Result<(), DataFusionError> { - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = self.catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - for (field_position, field) in - table.schema().fields().iter().enumerate() - { - builder.add_column( - &self.catalog_name, - &schema_name, - &table_name, - field_position, - field, - ) - } - } + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + Self::add_columns( + builder, + &catalog_name, + &schema_name, + schema.as_ref(), + ) + .await?; } } } + + // Add the information schema view columns themselves + Self::add_columns(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; + } + + // Add the system information schema view columns + if let Some(system_catalog_name) = &self.system_catalog_name { + Self::add_columns(builder, system_catalog_name, INFORMATION_SCHEMA, self) + .await?; } Ok(()) } + async fn add_columns( + builder: &mut InformationSchemaColumnsBuilder, + catalog_name: &str, + schema_name: &str, + schema: &dyn SchemaProvider, + ) -> Result<(), DataFusionError> { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + for (field_position, field) in table.schema().fields().iter().enumerate() + { + builder.add_column( + catalog_name, + schema_name, + &table_name, + field_position, + field, + ) + } + } + } + Ok(()) + } + /// Construct the `information_schema.df_settings` virtual table fn make_df_settings( &self, @@ -562,6 +652,24 @@ fn remove_native_type_prefix(native_type: &NativeType) -> String { #[async_trait] impl SchemaProvider for InformationSchemaProvider { + fn table_names(&self) -> Vec { + self.config.table_names() + } + + async fn table( + &self, + name: &str, + ) -> Result>, DataFusionError> { + self.config.get_table(name) + } + + fn table_exist(&self, name: &str) -> bool { + self.config.table_exist(name) + } +} + +#[async_trait] +impl SchemaProvider for InformationSchemaConfig { fn table_names(&self) -> Vec { INFORMATION_SCHEMA_TABLES .iter() @@ -573,7 +681,20 @@ impl SchemaProvider for InformationSchemaProvider { &self, name: &str, ) -> Result>, DataFusionError> { - let config = self.config.clone(); + self.get_table(name) + } + + fn table_exist(&self, name: &str) -> bool { + INFORMATION_SCHEMA_TABLES.contains(&name.to_ascii_lowercase().as_str()) + } +} + +impl InformationSchemaConfig { + fn get_table( + &self, + name: &str, + ) -> Result>, DataFusionError> { + let config = self.clone(); let table: Arc = match name.to_ascii_lowercase().as_str() { TABLES => Arc::new(InformationSchemaTables::new(config)), COLUMNS => Arc::new(InformationSchemaColumns::new(config)), @@ -589,10 +710,6 @@ impl SchemaProvider for InformationSchemaProvider { StreamingTable::try_new(Arc::clone(table.schema()), vec![table]).unwrap(), ))) } - - fn table_exist(&self, name: &str) -> bool { - INFORMATION_SCHEMA_TABLES.contains(&name.to_ascii_lowercase().as_str()) - } } #[derive(Debug)] @@ -1551,8 +1668,8 @@ mod tests { #[tokio::test] async fn make_tables_uses_table_type() { let config = InformationSchemaConfig { - catalog_name: "acatalog".to_string(), - catalog: Arc::new(Fixture), + system_catalog_name: None, + catalog_list: Arc::new(Fixture), table_functions: HashMap::new(), }; let mut builder = InformationSchemaTablesBuilder { @@ -1595,6 +1712,24 @@ mod tests { } } + impl CatalogProviderList for Fixture { + fn register_catalog( + &self, + _: String, + _: Arc, + ) -> Option> { + unimplemented!("not required for these tests") + } + + fn catalog_names(&self) -> Vec { + vec!["acatalog".to_string()] + } + + fn catalog(&self, _: &str) -> Option> { + Some(Arc::new(Self)) + } + } + impl CatalogProvider for Fixture { fn schema_names(&self) -> Vec { vec!["aschema".to_string()] diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index f770cebf88f95..6f92cae3c42eb 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -235,6 +235,9 @@ config_namespace! { /// The default schema name - this impacts what SQL queries use if not specified pub default_schema: String, default = "public".to_string() + /// The system schema name + pub system_catalog: Option, default = None + /// Should DataFusion provide access to `information_schema` /// virtual tables for displaying schema information pub information_schema: bool, default = false diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index ff1ad25811440..1919a83bd7853 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -2764,7 +2764,7 @@ mod tests { async fn sql_create_catalog() -> Result<()> { // the information schema used to introduce cyclic Arcs let ctx = SessionContext::new_with_config( - SessionConfig::new().with_information_schema(true), + SessionConfig::new().with_information_schema(true).with_system_catalog(Some(String::from("system"))), ); // Create catalog @@ -2780,7 +2780,7 @@ mod tests { .await?; // Check table exists in schema - let results = ctx.sql("SELECT * FROM information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap(); + let results = ctx.sql("SELECT * FROM system.information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap(); assert_eq!(results[0].num_rows(), 1); Ok(()) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 869c3002a24d1..27114a89e1438 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -46,7 +46,7 @@ use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan}; use datafusion_common::tree_node::TreeNode; use datafusion_common::{ DFSchema, DataFusionError, ResolvedTableReference, TableReference, config_err, - exec_err, plan_datafusion_err, + exec_err, plan_datafusion_err, plan_err, }; use datafusion_execution::TaskContext; use datafusion_execution::config::SessionConfig; @@ -76,7 +76,9 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr; use datafusion_physical_optimizer::optimizer::PhysicalOptimizer; use datafusion_physical_plan::ExecutionPlan; use datafusion_physical_plan::operator_statistics::StatisticsRegistry; -use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session}; +use datafusion_session::{ + CatalogProvider, PhysicalOptimizerContext, PhysicalOptimizerRule, Session, +}; #[cfg(feature = "sql")] use datafusion_sql::{ parser::{DFParserBuilder, Statement}, @@ -404,28 +406,75 @@ impl SessionState { table_ref: impl Into, ) -> datafusion_common::Result> { let resolved_ref = self.resolve_table_ref(table_ref); - let catalog = self - .inner - .catalog_list - .catalog(&resolved_ref.catalog) - .ok_or_else(|| { - plan_datafusion_err!( - "failed to resolve catalog: {}", - resolved_ref.catalog - ) - })?; - - if self.config.information_schema() && *resolved_ref.schema == *INFORMATION_SCHEMA + let catalog_name = resolved_ref.catalog.as_ref(); + + if let Some(system_catalog) = self.inner.config.system_catalog() + && system_catalog == catalog_name { - return Ok(Arc::new( - InformationSchemaProvider::new(resolved_ref.catalog.to_string(), catalog) - .with_table_functions(self.table_functions.clone()), - )); + if *resolved_ref.schema == *INFORMATION_SCHEMA { + return Ok(Arc::new( + InformationSchemaProvider::new(Arc::clone(&self.inner.catalog_list)) + .with_table_functions(self.inner.table_functions.clone()) + .with_system_catalog(String::from(system_catalog)), + )); + } + } else { + let catalog = + self.inner + .catalog_list + .catalog(catalog_name) + .ok_or_else(|| { + plan_datafusion_err!("failed to resolve catalog: {catalog_name}") + })?; + + if self.inner.config.information_schema() + && *resolved_ref.schema == *INFORMATION_SCHEMA + { + #[derive(Debug)] + struct SingleCatalogList { + catalog_name: String, + catalog: Arc, + } + + impl CatalogProviderList for SingleCatalogList { + fn register_catalog( + &self, + _: String, + _: Arc, + ) -> Option> { + None + } + + fn catalog_names(&self) -> Vec { + vec![self.catalog_name.clone()] + } + + fn catalog(&self, name: &str) -> Option> { + if name == self.catalog_name { + Some(Arc::clone(&self.catalog)) + } else { + None + } + } + } + + let catalog_list = Arc::new(SingleCatalogList { + catalog_name: catalog_name.to_string(), + catalog, + }); + + return Ok(Arc::new( + InformationSchemaProvider::new(catalog_list) + .with_table_functions(self.inner.table_functions.clone()), + )); + } + + if let Some(schema) = catalog.schema(&resolved_ref.schema) { + return Ok(schema); + } } - catalog.schema(&resolved_ref.schema).ok_or_else(|| { - plan_datafusion_err!("failed to resolve schema: {}", resolved_ref.schema) - }) + plan_err!("failed to resolve schema: {}", resolved_ref.schema) } /// Add `analyzer_rule` to the end of the list of @@ -588,6 +637,27 @@ impl SessionState { statement, enable_ident_normalization, )?; + + let table_refs = if let Some(system_catalog) = self.config().system_catalog() { + table_refs + .iter() + .map(|table_ref| match table_ref { + TableReference::Partial { schema, table } + if schema.as_ref() == INFORMATION_SCHEMA => + { + TableReference::Full { + catalog: Arc::from(system_catalog), + schema: Arc::clone(schema), + table: Arc::clone(table), + } + } + r => r.clone(), + }) + .collect() + } else { + table_refs + }; + Ok(table_refs) } diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rs index efaedebdadb33..2d21772016ccd 100644 --- a/datafusion/execution/src/config.rs +++ b/datafusion/execution/src/config.rs @@ -219,6 +219,11 @@ impl SessionConfig { self.options.execution.target_partitions } + /// Is the information schema enabled? + pub fn system_catalog(&self) -> Option<&str> { + self.options.catalog.system_catalog.as_deref() + } + /// Is the information schema enabled? pub fn information_schema(&self) -> bool { self.options.catalog.information_schema @@ -285,7 +290,14 @@ impl SessionConfig { self } - /// Enables or disables the inclusion of `information_schema` virtual tables + /// Enables or disables the inclusion of the system virtual catalog + pub fn with_system_catalog(mut self, catalog_name: Option) -> Self { + self.options_mut().catalog.system_catalog = catalog_name; + self + } + + /// Enables or disables the inclusion of `information_schema` virtual tables in each user + /// defined catalog pub fn with_information_schema(mut self, enabled: bool) -> Self { self.options_mut().catalog.information_schema = enabled; self diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 1a9072212f2f3..543c63eeb9e49 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -16,6 +16,7 @@ // under the License. use std::collections::{BTreeMap, HashMap, HashSet}; +use std::ops::Deref; use std::path::Path; use std::str::FromStr; use std::sync::Arc; @@ -1637,14 +1638,16 @@ impl SqlToRel<'_, S> { /// Generate a logical plan from a "SHOW TABLES" query fn show_tables_to_plan(&self) -> Result { - if self.has_table("information_schema", "tables") { - let query = "SELECT * FROM information_schema.tables;"; - let mut rewrite = DFParser::parse_sql(query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 - } else { - plan_err!("SHOW TABLES is not supported unless information_schema is enabled") - } + let Some(tables_table_ref) = self.resolve_info_table("tables") else { + return plan_err!( + "SHOW TABLES is not supported unless information_schema is enabled" + ); + }; + + let query = format!("SELECT * FROM {tables_table_ref};"); + let mut rewrite = DFParser::parse_sql(&query)?; + assert_eq!(rewrite.len(), 1); + self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 } fn describe_table_to_plan(&self, table_name: ObjectName) -> Result { @@ -2120,11 +2123,11 @@ impl SqlToRel<'_, S> { } fn show_variable_to_plan(&self, variable: &[Ident]) -> Result { - if !self.has_table("information_schema", "df_settings") { + let Some(df_settings_table_ref) = self.resolve_info_table("df_settings") else { return plan_err!( "SHOW [VARIABLE] is not supported unless information_schema is enabled" ); - } + }; let verbose = variable .last() @@ -2139,7 +2142,7 @@ impl SqlToRel<'_, S> { } let variable = object_name_to_string(&ObjectName::from(variable_vec)); - let base_query = format!("SELECT {columns} FROM information_schema.df_settings"); + let base_query = format!("SELECT {columns} FROM {df_settings_table_ref}"); let query = if variable == "all" { // Add an ORDER BY so the output comes out in a consistent order format!("{base_query} ORDER BY name") @@ -2988,11 +2991,11 @@ impl SqlToRel<'_, S> { self.options.enable_ident_normalization, )?; - if !self.has_table("information_schema", "columns") { + let Some(columns_table_ref) = self.resolve_info_table("columns") else { return plan_err!( "SHOW COLUMNS is not supported unless information_schema is enabled" ); - } + }; // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(sql_table_name)?; @@ -3005,9 +3008,8 @@ impl SqlToRel<'_, S> { "table_catalog, table_schema, table_name, column_name, data_type, is_nullable" }; - let query = format!( - "SELECT {select_list} FROM information_schema.columns WHERE {where_clause}" - ); + let query = + format!("SELECT {select_list} FROM {columns_table_ref} WHERE {where_clause}"); let mut rewrite = DFParser::parse_sql(&query)?; assert_eq!(rewrite.len(), 1); @@ -3136,11 +3138,12 @@ FROM ( &self, sql_table_name: ObjectName, ) -> Result { - if !self.has_table("information_schema", "tables") { + let Some(tables_table_ref) = self.resolve_info_table("tables") else { return plan_err!( "SHOW CREATE TABLE is not supported unless information_schema is enabled" ); - } + }; + // Figure out the where clause let where_clause = object_name_to_qualifier( &sql_table_name, @@ -3152,7 +3155,7 @@ FROM ( let _ = self.context_provider.get_table_source(table_ref)?; let query = format!( - "SELECT table_catalog, table_schema, table_name, definition FROM information_schema.views WHERE {where_clause}" + "SELECT table_catalog, table_schema, table_name, definition FROM {tables_table_ref} WHERE {where_clause}" ); let mut rewrite = DFParser::parse_sql(&query)?; @@ -3161,14 +3164,31 @@ FROM ( } /// Return true if there is a table provider available for "schema.table" - fn has_table(&self, schema: &str, table: &str) -> bool { - let tables_reference = TableReference::Partial { - schema: schema.into(), - table: table.into(), + fn resolve_info_table(&self, table: &str) -> Option { + let table_reference = if let Some(system_catalog) = + &self.context_provider.options().catalog.system_catalog + { + TableReference::Full { + catalog: system_catalog.deref().into(), + schema: "information_schema".into(), + table: table.into(), + } + } else { + TableReference::Partial { + schema: "information_schema".into(), + table: table.into(), + } }; - self.context_provider - .get_table_source(tables_reference) + + if self + .context_provider + .get_table_source(table_reference.clone()) .is_ok() + { + Some(table_reference) + } else { + None + } } fn validate_transaction_kind( From db98eb912fbbc2e5dc51a85ccfd7881f80fd2fb6 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 30 Sep 2026 00:22:49 +0200 Subject: [PATCH 03/20] Fix resolution of partial information_schema table references --- .../core/src/execution/session_state.rs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 7959b171597e7..43cd46502c0bb 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -662,27 +662,6 @@ impl SessionState { statement, enable_ident_normalization, )?; - - let table_refs = if let Some(system_catalog) = self.config().system_catalog() { - table_refs - .iter() - .map(|table_ref| match table_ref { - TableReference::Partial { schema, table } - if schema.as_ref() == INFORMATION_SCHEMA => - { - TableReference::Full { - catalog: Arc::from(system_catalog), - schema: Arc::clone(schema), - table: Arc::clone(table), - } - } - r => r.clone(), - }) - .collect() - } else { - table_refs - }; - Ok(table_refs) } From 6da6d7224baa52c2f9a2a4a03a1913301583e33f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 30 Sep 2026 00:23:13 +0200 Subject: [PATCH 04/20] Formatting --- datafusion/core/src/execution/context/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index 1d7efe1ae9eaf..edb6361340144 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -2846,7 +2846,9 @@ mod tests { async fn sql_create_catalog() -> Result<()> { // the information schema used to introduce cyclic Arcs let ctx = SessionContext::new_with_config( - SessionConfig::new().with_information_schema(true).with_system_catalog(Some(String::from("system"))), + SessionConfig::new() + .with_information_schema(true) + .with_system_catalog(Some(String::from("system"))), ); // Create catalog From 1b1ed047d6ef3fed81ea1f37dad147a68b07ce08 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Fri, 2 Oct 2026 21:59:25 +0200 Subject: [PATCH 05/20] Update config documentation --- datafusion/common/src/config.rs | 2 +- docs/source/user-guide/configs.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 795cad6a5a9f7..2b22de90470fa 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -237,7 +237,7 @@ config_namespace! { /// The default schema name - this impacts what SQL queries use if not specified pub default_schema: String, default = "public".to_string() - /// The system schema name + /// The system schema name - when set a global system and `information_schema` catalog will be available pub system_catalog: Option, default = None /// Should DataFusion provide access to `information_schema` diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index ba8ea52bb0850..6537031c613a0 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -68,6 +68,7 @@ The following configuration settings are available: | datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | | datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | | datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | +| datafusion.catalog.system_catalog | NULL | The system schema name - when set a global system and `information_schema` catalog will be available | | datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | | datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | | datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | From 1735fc569a5134a1e8661e3a648026d9a08ac662 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 3 Oct 2026 00:17:37 +0200 Subject: [PATCH 06/20] Move information schema resolution to session state --- .../core/src/execution/session_state.rs | 35 +++++++++++-- datafusion/sql/src/resolve.rs | 50 ------------------- 2 files changed, 32 insertions(+), 53 deletions(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 43cd46502c0bb..31ce00f16b81d 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -37,7 +37,7 @@ use arrow_schema::DataType; use arrow_schema::FieldRef; use datafusion_catalog::MemoryCatalogProviderList; use datafusion_catalog::information_schema::{ - INFORMATION_SCHEMA, InformationSchemaProvider, + INFORMATION_SCHEMA, INFORMATION_SCHEMA_TABLES, InformationSchemaProvider, }; use datafusion_catalog::{TableFunction, TableFunctionImpl}; use datafusion_common::alias::AliasGenerator; @@ -95,7 +95,9 @@ use log::{debug, info}; use object_store::ObjectStore; #[cfg(feature = "sql")] use sqlparser::{ - ast::{Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias}, + ast::{ + Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias, Statement as SQLStatement, + }, dialect::dialect_from_str, }; use url::Url; @@ -671,7 +673,34 @@ impl SessionState { &self, statement: Statement, ) -> datafusion_common::Result { - let references = self.resolve_table_references(&statement)?; + let mut references = self.resolve_table_references(&statement)?; + let require_info_schema = match &statement { + Statement::Statement(s) => matches!( + s.as_ref(), + SQLStatement::ShowCreate { .. } + | SQLStatement::ShowTables { .. } + | SQLStatement::ShowColumns { .. } + | SQLStatement::ShowVariable { .. } + | SQLStatement::ShowFunctions { .. } + ), + _ => false, + }; + + if require_info_schema { + let info_tables = INFORMATION_SCHEMA_TABLES.iter(); + let info_table_refs: Vec<_> = if let Some(system_catalog) = + self.config().system_catalog() + { + info_tables + .map(|n| TableReference::full(system_catalog, INFORMATION_SCHEMA, *n)) + .collect() + } else { + info_tables + .map(|n| TableReference::partial(INFORMATION_SCHEMA, *n)) + .collect() + }; + references.extend(info_table_refs); + } let mut provider = SessionContextProvider { state: self, diff --git a/datafusion/sql/src/resolve.rs b/datafusion/sql/src/resolve.rs index b6b1a74849be0..9abc687a2c39b 100644 --- a/datafusion/sql/src/resolve.rs +++ b/datafusion/sql/src/resolve.rs @@ -25,28 +25,6 @@ use crate::planner::object_name_to_table_reference; use datafusion_common::TableReference; use sqlparser::ast::*; -// following constants are used in `resolve_table_references` -// and should be same as `datafusion/catalog/src/information_schema.rs` -const INFORMATION_SCHEMA: &str = "information_schema"; -const TABLES: &str = "tables"; -const VIEWS: &str = "views"; -const COLUMNS: &str = "columns"; -const DF_SETTINGS: &str = "df_settings"; -const SCHEMATA: &str = "schemata"; -const ROUTINES: &str = "routines"; -const PARAMETERS: &str = "parameters"; - -/// All information schema tables -const INFORMATION_SCHEMA_TABLES: &[&str] = &[ - TABLES, - VIEWS, - COLUMNS, - DF_SETTINGS, - SCHEMATA, - ROUTINES, - PARAMETERS, -]; - // Collect table/CTE references as `TableReference`s and normalize them during traversal. // This avoids a second normalization/conversion pass after visiting the AST. struct RelationVisitor { @@ -127,34 +105,6 @@ impl Visitor for RelationVisitor { self.insert_relation(obj_name)?; } - // SHOW statements will later be rewritten into a SELECT from the information_schema - let requires_information_schema = matches!( - statement, - Statement::ShowFunctions { .. } - | Statement::ShowVariable { .. } - | Statement::ShowStatus { .. } - | Statement::ShowVariables { .. } - | Statement::ShowCreate { .. } - | Statement::ShowColumns { .. } - | Statement::ShowTables { .. } - | Statement::ShowCollation { .. } - ); - if requires_information_schema { - for s in INFORMATION_SCHEMA_TABLES { - // Information schema references are synthesized here, so convert directly. - let obj = ObjectName::from(vec![ - Ident::new(INFORMATION_SCHEMA), - Ident::new(*s), - ]); - match object_name_to_table_reference(obj, self.enable_ident_normalization) - { - Ok(tbl_ref) => { - self.relations.insert(tbl_ref); - } - Err(e) => return ControlFlow::Break(e), - } - } - } ControlFlow::Continue(()) } } From 1e5b40e3c05cd1e6ab277e01a4d0b5a3f61a8473 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 3 Oct 2026 15:19:16 +0200 Subject: [PATCH 07/20] Revert information schema content changes --- datafusion/catalog/src/information_schema.rs | 21 +------------------ datafusion/sql/src/statement.rs | 2 +- .../test_files/information_schema.slt | 2 ++ 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 9fd70cf9f7d0b..30ed1a3753a98 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -181,14 +181,6 @@ impl InformationSchemaConfig { builder.add_schemata(&catalog_name, &schema_name, schema_owner); } } - - // Add the information schema itself - builder.add_schemata(&catalog_name, INFORMATION_SCHEMA, None); - } - - // Add the system information schema - if let Some(system_catalog_name) = &self.system_catalog_name { - builder.add_schemata(system_catalog_name, INFORMATION_SCHEMA, None); } } @@ -234,9 +226,7 @@ impl InformationSchemaConfig { schema: &dyn SchemaProvider, ) -> Result<(), DataFusionError> { for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? - && table.table_type() == TableType::View - { + if let Some(table) = schema.table(&table_name).await? { builder.add_view( catalog_name, schema_name, @@ -270,15 +260,6 @@ impl InformationSchemaConfig { } } } - - // Add the information schema view columns themselves - Self::add_columns(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; - } - - // Add the system information schema view columns - if let Some(system_catalog_name) = &self.system_catalog_name { - Self::add_columns(builder, system_catalog_name, INFORMATION_SCHEMA, self) - .await?; } Ok(()) diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index a88048a0cbcf2..75a7e6170fe6f 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -3146,7 +3146,7 @@ FROM ( &self, sql_table_name: ObjectName, ) -> Result { - let Some(tables_table_ref) = self.resolve_info_table("tables") else { + let Some(tables_table_ref) = self.resolve_info_table("views") else { return plan_err!( "SHOW CREATE TABLE is not supported unless information_schema is enabled" ); diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index f6633a165df18..b34c772f618fe 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -213,6 +213,7 @@ datafusion.catalog.has_header true datafusion.catalog.information_schema true datafusion.catalog.location NULL datafusion.catalog.newlines_in_values false +datafusion.catalog.system_catalog NULL datafusion.execution.batch_size 8192 datafusion.execution.coalesce_batches true datafusion.execution.collect_statistics true @@ -375,6 +376,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. +datafusion.catalog.system_catalog NULL The system schema name - when set a global system and `information_schema` catalog will be available datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. From f728f064c3b5f94b4183576280556a2538f1fe4d Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 20 Oct 2026 14:51:54 +0200 Subject: [PATCH 08/20] Make segregated information schema opt-in --- datafusion/catalog/src/information_schema.rs | 45 ++++ .../core/src/execution/session_state.rs | 118 ++++---- .../information_schema_multiple_catalogs.slt | 252 ++++++++++++++++++ 3 files changed, 355 insertions(+), 60 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index f466533cfa05f..7a2ad76bf1e17 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -44,6 +44,7 @@ use datafusion_expr::{TableType, Volatility}; use datafusion_physical_plan::SendableRecordBatchStream; use datafusion_physical_plan::stream::RecordBatchStreamAdapter; use datafusion_physical_plan::streaming::PartitionStream; +use datafusion_session::CatalogProvider; use itertools::Itertools; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; @@ -81,6 +82,50 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` + pub fn for_catalog(catalog_name: String, catalog: Arc) -> Self { + #[derive(Debug)] + struct SingleCatalogList { + catalog_name: String, + catalog: Arc, + } + + impl CatalogProviderList for SingleCatalogList { + fn register_catalog( + &self, + _: String, + _: Arc, + ) -> Option> { + None + } + + fn catalog_names(&self) -> Vec { + vec![self.catalog_name.clone()] + } + + fn catalog(&self, name: &str) -> Option> { + if name == self.catalog_name { + Some(Arc::clone(&self.catalog)) + } else { + None + } + } + } + + let catalog_list = Arc::new(SingleCatalogList { + catalog_name, + catalog, + }); + + Self { + config: InformationSchemaConfig { + system_catalog_name: None, + catalog_list, + table_functions: HashMap::new(), + }, + } + } + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` pub fn new(catalog_list: Arc) -> Self { Self { diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 09105dd953dbb..10f507d1d5dc4 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -423,73 +423,71 @@ impl SessionState { let resolved_ref = self.resolve_table_ref(table_ref); let catalog_name = resolved_ref.catalog.as_ref(); - if let Some(system_catalog) = self.inner.config.system_catalog() - && system_catalog == catalog_name + // Special case handling for `information_schema`. + if *resolved_ref.schema == *INFORMATION_SCHEMA + && self.inner.config.information_schema() { - if *resolved_ref.schema == *INFORMATION_SCHEMA { - return Ok(Arc::new( - InformationSchemaProvider::new(Arc::clone(&self.inner.catalog_list)) - .with_table_functions(self.inner.table_functions.clone()) - .with_system_catalog(String::from(system_catalog)), - )); - } - } else { - let catalog = - self.inner - .catalog_list - .catalog(catalog_name) - .ok_or_else(|| { - plan_datafusion_err!("failed to resolve catalog: {catalog_name}") - })?; - - if self.inner.config.information_schema() - && *resolved_ref.schema == *INFORMATION_SCHEMA - { - #[derive(Debug)] - struct SingleCatalogList { - catalog_name: String, - catalog: Arc, - } - - impl CatalogProviderList for SingleCatalogList { - fn register_catalog( - &self, - _: String, - _: Arc, - ) -> Option> { - None - } - - fn catalog_names(&self) -> Vec { - vec![self.catalog_name.clone()] - } - - fn catalog(&self, name: &str) -> Option> { - if name == self.catalog_name { - Some(Arc::clone(&self.catalog)) - } else { - None - } + match self.inner.config.system_catalog() { + Some(system_catalog) => { + if system_catalog == catalog_name { + // Global catalog configured, catalog name references it + // => information schema provider for full catalog list + return Ok(Arc::new( + InformationSchemaProvider::new(Arc::clone( + &self.inner.catalog_list, + )) + .with_table_functions(self.inner.table_functions.clone()) + .with_system_catalog(String::from(system_catalog)), + )); + } else { + // Global catalog configured, catalog name is other catalog + // => information schema provider for named catalog + let catalog = self.resolve_catalog(catalog_name)?; + return Ok(Arc::new( + InformationSchemaProvider::for_catalog( + catalog_name.to_string(), + catalog, + ) + .with_table_functions(self.inner.table_functions.clone()), + )); } } - - let catalog_list = Arc::new(SingleCatalogList { - catalog_name: catalog_name.to_string(), - catalog, - }); - - return Ok(Arc::new( - InformationSchemaProvider::new(catalog_list) + None => { + // Global catalog not configured + // => information schema provider for full catalog list + // Resolve the catalog (even though it's not needed for the InformationSchemaProvider) + // so references to non-existent catalogs report an error. + self.resolve_catalog(catalog_name)?; + return Ok(Arc::new( + InformationSchemaProvider::new(Arc::clone( + &self.inner.catalog_list, + )) .with_table_functions(self.inner.table_functions.clone()), - )); - } - - if let Some(schema) = catalog.schema(&resolved_ref.schema) { - return Ok(schema); + )); + } } } - plan_err!("failed to resolve schema: {}", resolved_ref.schema) + let catalog = self.resolve_catalog(catalog_name)?; + let Some(schema) = catalog.schema(&resolved_ref.schema) else { + return plan_err!("failed to resolve schema: {}", resolved_ref.schema); + }; + + Ok(schema) + } + + fn resolve_catalog( + &self, + catalog_name: &str, + ) -> Result, DataFusionError> { + let catalog = self + .inner + .catalog_list + .catalog(catalog_name) + .ok_or_else(|| { + plan_datafusion_err!("failed to resolve catalog: {catalog_name}") + })?; + Ok(catalog) } /// Add `analyzer_rule` to the end of the list of diff --git a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt index ae5a54902f23f..d4048a8b6b327 100644 --- a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt +++ b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt @@ -83,17 +83,71 @@ WHERE table_schema <> 'information_schema'; ---- my_catalog my_schema t1 my_catalog my_schema t2 +my_other_catalog my_other_schema t3 query TTT rowsort SELECT table_catalog, table_schema, table_name FROM my_other_catalog.information_schema.tables WHERE table_schema <> 'information_schema'; ---- +my_catalog my_schema t1 +my_catalog my_schema t2 my_other_catalog my_other_schema t3 query TTTT rowsort SELECT * from information_schema.tables; ---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE + +############ +# Configure a system catalog +# +# When `datafusion.catalog.system_catalog` is set, the `information_schema` +# of each catalog only describes the objects in that catalog. Only the +# `information_schema` of the system catalog describes all catalogs. +########### +statement ok +set datafusion.catalog.system_catalog = system; + +query TTTT rowsort +SELECT * FROM my_catalog.information_schema.tables; +---- +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE + +query TTTT rowsort +SELECT * FROM my_other_catalog.information_schema.tables; +---- my_other_catalog information_schema columns VIEW my_other_catalog information_schema df_settings VIEW my_other_catalog information_schema parameters VIEW @@ -103,6 +157,204 @@ my_other_catalog information_schema tables VIEW my_other_catalog information_schema views VIEW my_other_catalog my_other_schema t3 BASE TABLE +query TTTT rowsort +SELECT * FROM datafusion.information_schema.tables; +---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW + +# Unqualified `information_schema` resolves against the default catalog +query TTTT rowsort +SELECT * FROM information_schema.tables; +---- +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE + +query TTTT rowsort +SELECT * FROM system.information_schema.tables; +---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE +system information_schema columns VIEW +system information_schema df_settings VIEW +system information_schema parameters VIEW +system information_schema routines VIEW +system information_schema schemata VIEW +system information_schema tables VIEW +system information_schema views VIEW + +query TTTTTTT rowsort +SELECT * FROM my_catalog.information_schema.schemata; +---- +my_catalog my_schema NULL NULL NULL NULL NULL + +query TTTTTTT rowsort +SELECT * FROM my_other_catalog.information_schema.schemata; +---- +my_other_catalog my_other_schema NULL NULL NULL NULL NULL + +query TTTTTTT rowsort +SELECT * FROM system.information_schema.schemata; +---- +datafusion public NULL NULL NULL NULL NULL +my_catalog my_schema NULL NULL NULL NULL NULL +my_other_catalog my_other_schema NULL NULL NULL NULL NULL + +query TTTT rowsort +SELECT table_catalog, table_schema, table_name, column_name +FROM my_catalog.information_schema.columns; +---- +my_catalog my_schema t1 column1 +my_catalog my_schema t2 column1 + +query TTTT rowsort +SELECT table_catalog, table_schema, table_name, column_name +FROM my_other_catalog.information_schema.columns; +---- +my_other_catalog my_other_schema t3 column1 + +query TTTT rowsort +SELECT table_catalog, table_schema, table_name, column_name +FROM system.information_schema.columns; +---- +my_catalog my_schema t1 column1 +my_catalog my_schema t2 column1 +my_other_catalog my_other_schema t3 column1 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_catalog.information_schema.views +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM system.information_schema.views +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 +my_other_catalog my_other_schema t3 + +# SHOW statements use the system catalog's information schema +query TTTT rowsort +SHOW TABLES; +---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE +system information_schema columns VIEW +system information_schema df_settings VIEW +system information_schema parameters VIEW +system information_schema routines VIEW +system information_schema schemata VIEW +system information_schema tables VIEW +system information_schema views VIEW + +query TTTTTT +SHOW COLUMNS FROM my_catalog.my_schema.t1; +---- +my_catalog my_schema t1 column1 Int64 NO + +# The information schema of a catalog that does not exist cannot be queried +statement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not found +SELECT * FROM no_such_catalog.information_schema.tables; + +# A system catalog can be an existing catalog. Its information schema then +# describes all catalogs +statement ok +set datafusion.catalog.system_catalog = my_catalog; + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 +my_other_catalog my_other_schema t3 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_other_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_other_catalog my_other_schema t3 + +# Resetting the system catalog makes every information schema global again +statement ok +RESET datafusion.catalog.system_catalog; + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_other_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 +my_other_catalog my_other_schema t3 + +# Without a system catalog, the information schema of a catalog that does not +# exist cannot be queried either +statement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not found +SELECT * FROM no_such_catalog.information_schema.tables; + # Cleanup statement ok From b43795f3be627a20ce1b742d857bd447fe610114 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 20 Oct 2026 15:09:32 +0200 Subject: [PATCH 09/20] Add extra detail to `system_catalog` description --- datafusion/common/src/config.rs | 9 ++++++--- .../sqllogictest/test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 91c347cd6ea45..7efa7775f6325 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -238,13 +238,16 @@ config_namespace! { /// The default schema name - this impacts what SQL queries use if not specified pub default_schema: String, default = "public".to_string() - /// The system schema name - when set a global system and `information_schema` catalog will be available - pub system_catalog: Option, default = None - /// Should DataFusion provide access to `information_schema` /// virtual tables for displaying schema information pub information_schema: bool, default = false + /// The system schema name - when set together with `information_schema`, the + /// `information_schema` for each catalog will only describe objects contained in the + /// catalog itself. The global, cross-catalog `information_schema` is available under + /// the virtual catalog named by this option. + pub system_catalog: Option, default = None + /// Location scanned to load tables for `default` schema pub location: Option, default = None diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 7420463d3cc4d..4e8f9959540a7 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -383,7 +383,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.catalog.system_catalog NULL The system schema name - when set a global system and `information_schema` catalog will be available +datafusion.catalog.system_catalog NULL The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index bbc444e0af51b..cdd2246fbff8b 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -68,8 +68,8 @@ The following configuration settings are available: | datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | | datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | | datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | -| datafusion.catalog.system_catalog | NULL | The system schema name - when set a global system and `information_schema` catalog will be available | | datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | +| datafusion.catalog.system_catalog | NULL | The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. | | datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | | datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | | datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | From de365473fa974d81b6f3a3601727902a06f43268 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 20 Oct 2026 15:19:00 +0200 Subject: [PATCH 10/20] Correct `system_catalog` short description --- datafusion/common/src/config.rs | 2 +- datafusion/sqllogictest/test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 7efa7775f6325..6da0216f7e093 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -242,7 +242,7 @@ config_namespace! { /// virtual tables for displaying schema information pub information_schema: bool, default = false - /// The system schema name - when set together with `information_schema`, the + /// The system catalog name - when set together with `information_schema`, the /// `information_schema` for each catalog will only describe objects contained in the /// catalog itself. The global, cross-catalog `information_schema` is available under /// the virtual catalog named by this option. diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 4e8f9959540a7..03a1976306bf0 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -383,7 +383,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.catalog.system_catalog NULL The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. +datafusion.catalog.system_catalog NULL The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index cdd2246fbff8b..d13c650061ff4 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -69,7 +69,7 @@ The following configuration settings are available: | datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | | datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | | datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | -| datafusion.catalog.system_catalog | NULL | The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. | +| datafusion.catalog.system_catalog | NULL | The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. | | datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | | datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | | datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | From b40d5877efe8187a7c99667cc051b061603bb2f7 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 22:59:32 +0200 Subject: [PATCH 11/20] Fix 'explain show tables' --- .../core/src/execution/session_state.rs | 29 +++++++++++-------- .../test_files/information_schema.slt | 7 +++++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 10f507d1d5dc4..0debae505b6b5 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -102,6 +102,7 @@ use sqlparser::{ }; use url::Url; use uuid::Uuid; +use datafusion_sql::parser::ExplainStatement; /// `SessionState` contains all the necessary state to plan and execute queries, /// such as configuration, functions, and runtime environment. Please see the @@ -672,19 +673,8 @@ impl SessionState { statement: Statement, ) -> datafusion_common::Result { let mut references = self.resolve_table_references(&statement)?; - let require_info_schema = match &statement { - Statement::Statement(s) => matches!( - s.as_ref(), - SQLStatement::ShowCreate { .. } - | SQLStatement::ShowTables { .. } - | SQLStatement::ShowColumns { .. } - | SQLStatement::ShowVariable { .. } - | SQLStatement::ShowFunctions { .. } - ), - _ => false, - }; - if require_info_schema { + if Self::requires_information_schema(&statement) { let info_tables = INFORMATION_SCHEMA_TABLES.iter(); let info_table_refs: Vec<_> = if let Some(system_catalog) = self.config().system_catalog() @@ -721,6 +711,21 @@ impl SessionState { query.statement_to_plan(statement) } + fn requires_information_schema(statement: &Statement) -> bool { + match &statement { + Statement::Explain(ExplainStatement { statement, .. }) => Self::requires_information_schema(statement), + Statement::Statement(s) => matches!( + s.as_ref(), + SQLStatement::ShowCreate { .. } + | SQLStatement::ShowTables { .. } + | SQLStatement::ShowColumns { .. } + | SQLStatement::ShowVariable { .. } + | SQLStatement::ShowFunctions { .. } + ), + _ => false, + } + } + #[cfg(feature = "sql")] fn get_parser_options(&self) -> ParserOptions { let sql_parser_options = &self.config().options().sql_parser; diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 03a1976306bf0..e9d7e861fbb5c 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -653,6 +653,13 @@ datafusion information_schema tables VIEW datafusion information_schema views VIEW +query TT +EXPLAIN SHOW TABLES +---- +logical_plan TableScan: information_schema.tables projection=[table_catalog, table_schema, table_name, table_type] +physical_plan StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, table_type] + + # information_schema_show_tables_no_information_schema statement ok From 94c64e60def8170a392aa1b5d1d5d63a5cbb54db Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 23:00:15 +0200 Subject: [PATCH 12/20] Do not use partioned information schema in CLI by default --- datafusion-cli/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 16786bb3a8c1f..a3c3ad8ca9720 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -381,7 +381,6 @@ fn get_session_config(args: &Args) -> Result { } let mut session_config = SessionConfig::from(config_options) - .with_system_catalog(Some(String::from("system"))) .with_information_schema(true); if args.reads_sql_from_stdin() { From 4bc355381c27609b42c06890bb63abf842bc889d Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 23:06:30 +0200 Subject: [PATCH 13/20] Formatting --- datafusion-cli/src/main.rs | 4 ++-- datafusion/core/src/execution/session_state.rs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index a3c3ad8ca9720..d24f924bfb5bd 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -380,8 +380,8 @@ fn get_session_config(args: &Args) -> Result { config_options.sql_parser.dialect = Dialect::Spark; } - let mut session_config = SessionConfig::from(config_options) - .with_information_schema(true); + let mut session_config = + SessionConfig::from(config_options).with_information_schema(true); if args.reads_sql_from_stdin() { // When stdin carries the session's SQL — the REPL (including any rc diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 0debae505b6b5..9abf9e92ba0b2 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -89,6 +89,7 @@ use datafusion_sql::{ use async_trait::async_trait; use chrono::{DateTime, Utc}; +use datafusion_sql::parser::ExplainStatement; use futures::future::BoxFuture; use itertools::Itertools; use log::{debug, info}; @@ -102,7 +103,6 @@ use sqlparser::{ }; use url::Url; use uuid::Uuid; -use datafusion_sql::parser::ExplainStatement; /// `SessionState` contains all the necessary state to plan and execute queries, /// such as configuration, functions, and runtime environment. Please see the @@ -713,7 +713,9 @@ impl SessionState { fn requires_information_schema(statement: &Statement) -> bool { match &statement { - Statement::Explain(ExplainStatement { statement, .. }) => Self::requires_information_schema(statement), + Statement::Explain(ExplainStatement { statement, .. }) => { + Self::requires_information_schema(statement) + } Statement::Statement(s) => matches!( s.as_ref(), SQLStatement::ShowCreate { .. } From 2678ab791a93869c57cbfe5f5fd2d18ac88f44d9 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 23:35:17 +0200 Subject: [PATCH 14/20] Revert adding information schema views to information_schema.views --- datafusion/catalog/src/information_schema.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 7a2ad76bf1e17..97b55169e0e88 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -251,18 +251,9 @@ impl InformationSchemaConfig { .await?; } } - - // Add the information schema views themselves - Self::add_views(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; } } - // Add the system information schema - if let Some(system_catalog_name) = &self.system_catalog_name { - Self::add_views(builder, system_catalog_name, INFORMATION_SCHEMA, self) - .await?; - } - Ok(()) } From ccdb21698e1a87131400fb3e3ac85dabaf960211 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 24 Oct 2026 14:07:50 +0200 Subject: [PATCH 15/20] Resolve compile errors when `sql` feature is not enabled --- datafusion/core/src/execution/session_state.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 9abf9e92ba0b2..50034f42ec4a5 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -36,6 +36,7 @@ use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; use arrow_schema::DataType; use arrow_schema::FieldRef; use datafusion_catalog::MemoryCatalogProviderList; +#[cfg(feature = "sql")] use datafusion_catalog::information_schema::{ INFORMATION_SCHEMA, INFORMATION_SCHEMA_TABLES, InformationSchemaProvider, }; @@ -83,13 +84,12 @@ use datafusion_session::{ }; #[cfg(feature = "sql")] use datafusion_sql::{ - parser::{DFParserBuilder, Statement}, + parser::{DFParserBuilder, ExplainStatement, Statement}, planner::{ContextProvider, ParserOptions, PlannerContext, SqlToRel}, }; use async_trait::async_trait; use chrono::{DateTime, Utc}; -use datafusion_sql::parser::ExplainStatement; use futures::future::BoxFuture; use itertools::Itertools; use log::{debug, info}; @@ -711,6 +711,7 @@ impl SessionState { query.statement_to_plan(statement) } + #[cfg(feature = "sql")] fn requires_information_schema(statement: &Statement) -> bool { match &statement { Statement::Explain(ExplainStatement { statement, .. }) => { From 8dad5df44e749530b553fc3614f2d96f94555e1a Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 24 Oct 2026 22:02:20 +0200 Subject: [PATCH 16/20] Correctly resolve parameters and routines table references --- datafusion/sql/src/statement.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 609738a33988f..0ed504bffaae9 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -3029,6 +3029,15 @@ impl SqlToRel<'_, S> { &self, filter: Option, ) -> Result { + let (Some(parameters_table_ref), Some(routines_table_ref)) = ( + self.resolve_info_table("parameters"), + self.resolve_info_table("routines"), + ) else { + return plan_err!( + "SHOW FUNCTIONS is not supported unless information_schema is enabled" + ); + }; + let where_clause = if let Some(filter) = filter { match filter { ShowStatementFilter::Like(like) => { @@ -3082,7 +3091,7 @@ FROM ( data_type, rid FROM - information_schema.parameters + {parameters_table_ref} WHERE parameter_mode = 'OUT' ) o @@ -3097,7 +3106,7 @@ FROM ( data_type, rid FROM - information_schema.parameters + {parameters_table_ref} WHERE parameter_mode = 'IN' ) i @@ -3107,7 +3116,7 @@ FROM ( AND i.rid = o.rid GROUP BY 1, 2, o.rid ) as p - JOIN information_schema.routines r + JOIN {routines_table_ref} r ON p.function_name = r.routine_name AND r.function_type <> 'TABLE' @@ -3121,7 +3130,7 @@ FROM ( function_type, description, syntax_example - FROM information_schema.routines + FROM {routines_table_ref} WHERE function_type = 'TABLE' GROUP BY routine_name, data_type, function_type, description, syntax_example ) sc @@ -3137,7 +3146,7 @@ FROM ( &self, sql_table_name: ObjectName, ) -> Result { - let Some(tables_table_ref) = self.resolve_info_table("views") else { + let Some(views_table_ref) = self.resolve_info_table("views") else { return plan_err!( "SHOW CREATE TABLE is not supported unless information_schema is enabled" ); @@ -3154,7 +3163,7 @@ FROM ( let _ = self.context_provider.get_table_source(table_ref)?; let query = format!( - "SELECT table_catalog, table_schema, table_name, definition FROM {tables_table_ref} WHERE {where_clause}" + "SELECT table_catalog, table_schema, table_name, definition FROM {views_table_ref} WHERE {where_clause}" ); let mut rewrite = DFParser::parse_sql(&query)?; From 821136cdacddbdf7e99428b6fc858f11334084d9 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 24 Oct 2026 22:02:54 +0200 Subject: [PATCH 17/20] Another attempt to fix compilation issues when `sql` feature is not enabled --- datafusion/catalog/src/information_schema.rs | 16 ++++++++++------ datafusion/core/src/execution/session_state.rs | 3 +-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 97b55169e0e88..fde7020df8c60 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -82,8 +82,11 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { - /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` - pub fn for_catalog(catalog_name: String, catalog: Arc) -> Self { + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog` + pub fn new_from_catalog( + catalog_name: String, + catalog: Arc, + ) -> Self { #[derive(Debug)] struct SingleCatalogList { catalog_name: String, @@ -167,9 +170,10 @@ impl InformationSchemaConfig { builder: &mut InformationSchemaTablesBuilder, ) -> Result<(), DataFusionError> { // create a mem table with the names of tables + let catalog_names = self.catalog_list.catalog_names(); - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + for catalog_name in &catalog_names { + let catalog = self.catalog_list.catalog(catalog_name).unwrap(); for schema_name in catalog.schema_names() { if schema_name != INFORMATION_SCHEMA { @@ -180,7 +184,7 @@ impl InformationSchemaConfig { schema.table_type(&table_name).await? { builder.add_table( - &catalog_name, + catalog_name, &schema_name, &table_name, table_type, @@ -194,7 +198,7 @@ impl InformationSchemaConfig { // Add a final list for the information schema tables themselves for table_name in INFORMATION_SCHEMA_TABLES { builder.add_table( - &catalog_name, + catalog_name, INFORMATION_SCHEMA, table_name, TableType::View, diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 50034f42ec4a5..41f911a980e7b 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -36,7 +36,6 @@ use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; use arrow_schema::DataType; use arrow_schema::FieldRef; use datafusion_catalog::MemoryCatalogProviderList; -#[cfg(feature = "sql")] use datafusion_catalog::information_schema::{ INFORMATION_SCHEMA, INFORMATION_SCHEMA_TABLES, InformationSchemaProvider, }; @@ -445,7 +444,7 @@ impl SessionState { // => information schema provider for named catalog let catalog = self.resolve_catalog(catalog_name)?; return Ok(Arc::new( - InformationSchemaProvider::for_catalog( + InformationSchemaProvider::new_from_catalog( catalog_name.to_string(), catalog, ) From f9ee4544e871c9de5fbfd95b5179169b9cd655ac Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 24 Oct 2026 22:05:32 +0200 Subject: [PATCH 18/20] Avoid double listing of system catalog tables --- datafusion/catalog/src/information_schema.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index fde7020df8c60..ebaf142d916b9 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -206,7 +206,9 @@ impl InformationSchemaConfig { } } - if let Some(system_catalog_name) = &self.system_catalog_name { + if let Some(system_catalog_name) = &self.system_catalog_name + && !catalog_names.contains(system_catalog_name) + { for table_name in INFORMATION_SCHEMA_TABLES { builder.add_table( system_catalog_name, From 0dd5736bd6295d0ac30a519ade5f6c10daacae1f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 24 Oct 2026 22:17:36 +0200 Subject: [PATCH 19/20] Silence unused import warning when `sql` feature is disabled --- datafusion/core/src/execution/session_state.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 41f911a980e7b..7d87e2043d678 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -36,8 +36,10 @@ use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; use arrow_schema::DataType; use arrow_schema::FieldRef; use datafusion_catalog::MemoryCatalogProviderList; +#[cfg(feature = "sql")] +use datafusion_catalog::information_schema::INFORMATION_SCHEMA_TABLES; use datafusion_catalog::information_schema::{ - INFORMATION_SCHEMA, INFORMATION_SCHEMA_TABLES, InformationSchemaProvider, + INFORMATION_SCHEMA, InformationSchemaProvider, }; use datafusion_catalog::{TableFunction, TableFunctionImpl}; use datafusion_common::alias::AliasGenerator; From 21dafcef5f932efa6a6690edea40cd11c64a6d14 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Mon, 5 Oct 2026 17:59:37 +0200 Subject: [PATCH 20/20] Decouple 'show' statements from information schema tables --- datafusion-cli/src/exec.rs | 4 +- datafusion/catalog/src/information_schema.rs | 1419 ++++++----------- datafusion/common/src/config.rs | 9 +- datafusion/common/src/information_schema.rs | 1046 ++++++++++++ datafusion/common/src/lib.rs | 3 + datafusion/common/src/table_type.rs | 37 + .../core/src/execution/session_state.rs | 103 +- datafusion/core/src/physical_planner.rs | 189 ++- datafusion/execution/src/config.rs | 22 +- datafusion/expr/src/logical_plan/builder.rs | 67 +- datafusion/expr/src/logical_plan/display.rs | 69 +- datafusion/expr/src/logical_plan/mod.rs | 8 +- datafusion/expr/src/logical_plan/plan.rs | 133 +- datafusion/expr/src/logical_plan/tree_node.rs | 6 +- datafusion/expr/src/table_source.rs | 22 +- .../optimizer/src/common_subexpr_eliminate.rs | 2 +- datafusion/optimizer/src/decorrelate.rs | 2 +- .../optimizer/src/optimize_projections/mod.rs | 2 +- datafusion/optimizer/src/optimizer.rs | 2 +- datafusion/proto/src/logical_plan/mod.rs | 4 +- datafusion/sql/src/statement.rs | 295 +--- datafusion/sql/src/unparser/plan.rs | 2 +- .../runner.rs | 2 +- .../test_files/information_schema.slt | 43 +- .../information_schema_multiple_catalogs.slt | 115 +- .../src/logical_plan/producer/rel/mod.rs | 4 +- docs/source/user-guide/configs.md | 2 +- 27 files changed, 2032 insertions(+), 1580 deletions(-) create mode 100644 datafusion/common/src/information_schema.rs create mode 100644 datafusion/common/src/table_type.rs diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs index 288ce4b7351b6..f4cc3911be510 100644 --- a/datafusion-cli/src/exec.rs +++ b/datafusion-cli/src/exec.rs @@ -373,9 +373,7 @@ impl AdjustedPrintOptions { // all rows if matches!( plan, - LogicalPlan::Explain(_) - | LogicalPlan::DescribeTable(_) - | LogicalPlan::Analyze(_) + LogicalPlan::Explain(_) | LogicalPlan::Show(_) | LogicalPlan::Analyze(_) ) { self.inner.maxrows = MaxRows::Unlimited; } diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index ebaf142d916b9..4bcc6d75c2c56 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -22,34 +22,52 @@ use crate::streaming::StreamingTable; use crate::table::TableFunction; use crate::{CatalogProviderList, SchemaProvider, TableProvider}; -use arrow::array::builder::{BooleanBuilder, UInt8Builder}; -use arrow::{ - array::{StringBuilder, UInt64Builder}, - datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}, - record_batch::RecordBatch, -}; +use arrow::array::StringArray; +use arrow::array::builder::{ListBuilder, StringBuilder}; +use arrow::compute::kernels::filter::filter_record_batch; +use arrow::compute::like; +use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; use async_trait::async_trait; -use datafusion_common::DataFusionError; -use datafusion_common::config::{ConfigEntry, ConfigOptions}; +use datafusion_common::config::ConfigOptions; use datafusion_common::error::Result; +use datafusion_common::information_schema::{ + ColumnsDetail, InformationSchemaColumnsBuilder, InformationSchemaDfSettingsBuilder, + InformationSchemaParametersBuilder, InformationSchemaRoutinesBuilder, + InformationSchemaTablesBuilder, InformationSchemaViewBuilder, columns_schema, + df_settings_schema, parameters_schema, routines_schema, show_functions_schema, + tables_schema, views_schema, +}; +use datafusion_common::{ + DataFusionError, ResolvedTableReference, plan_datafusion_err, plan_err, +}; +// Re-exported (rather than just `use`d) because these were already part of +// this module's public API before it started sharing its schema/builder +// definitions with `datafusion_common::information_schema`. +pub use datafusion_common::information_schema::{ + InformationSchemataBuilder, schemata_schema, +}; use datafusion_common::types::{LogicalType, NativeType}; +use datafusion_datasource::memory::MemorySourceConfig; use datafusion_execution::TaskContext; use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::function::WindowUDFFieldArgs; use datafusion_expr::type_coercion::functions::fields_with_udf; use datafusion_expr::{ - AggregateUDF, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature, WindowUDF, + AggregateUDF, Documentation, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature, + WindowUDF, }; use datafusion_expr::{TableType, Volatility}; -use datafusion_physical_plan::SendableRecordBatchStream; use datafusion_physical_plan::stream::RecordBatchStreamAdapter; -use datafusion_physical_plan::streaming::PartitionStream; +use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use datafusion_physical_plan::{ExecutionPlan, SendableRecordBatchStream}; use datafusion_session::CatalogProvider; use itertools::Itertools; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; use std::sync::Arc; +/// Name of the `information_schema` schema pub const INFORMATION_SCHEMA: &str = "information_schema"; pub(crate) const TABLES: &str = "tables"; pub(crate) const VIEWS: &str = "views"; @@ -70,7 +88,8 @@ pub const INFORMATION_SCHEMA_TABLES: &[&str] = &[ PARAMETERS, ]; -/// Implements the `information_schema` virtual schema and tables +/// Implements the `information_schema` virtual schema and tables, and the +/// `SHOW` statements which are based on the same data. /// /// The underlying tables in the `information_schema` are created on /// demand. This means that if more tables are added to the underlying @@ -82,59 +101,13 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { - /// Creates a new [`InformationSchemaProvider`] for the provided `catalog` - pub fn new_from_catalog( - catalog_name: String, - catalog: Arc, - ) -> Self { - #[derive(Debug)] - struct SingleCatalogList { - catalog_name: String, - catalog: Arc, - } - - impl CatalogProviderList for SingleCatalogList { - fn register_catalog( - &self, - _: String, - _: Arc, - ) -> Option> { - None - } - - fn catalog_names(&self) -> Vec { - vec![self.catalog_name.clone()] - } - - fn catalog(&self, name: &str) -> Option> { - if name == self.catalog_name { - Some(Arc::clone(&self.catalog)) - } else { - None - } - } - } - - let catalog_list = Arc::new(SingleCatalogList { - catalog_name, - catalog, - }); - - Self { - config: InformationSchemaConfig { - system_catalog_name: None, - catalog_list, - table_functions: HashMap::new(), - }, - } - } - /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` pub fn new(catalog_list: Arc) -> Self { Self { config: InformationSchemaConfig { - system_catalog_name: None, + information_schema_enabled: true, catalog_list, + catalog_filter: None, table_functions: HashMap::new(), }, } @@ -150,51 +123,368 @@ impl InformationSchemaProvider { self } - pub fn with_system_catalog(mut self, catalog_name: String) -> Self { - self.config.system_catalog_name = Some(catalog_name); + /// Sets whether the `information_schema` is enabled. When disabled, its own + /// tables are not reported and cannot be described. + pub fn with_information_schema_enabled(mut self, enabled: bool) -> Self { + self.config.information_schema_enabled = enabled; + self + } + + /// Restricts the reported objects to the given catalog. When `None`, all + /// catalogs are reported. + pub fn with_catalog_filter(mut self, catalog_name: Option) -> Self { + self.config.catalog_filter = catalog_name; self } + + /// Creates an execution plan for the `SHOW TABLES` statement. + pub fn show_tables(&self) -> Result> { + Ok(Arc::new(StreamingTableExec::try_new( + SchemaRef::from(tables_schema()), + vec![Arc::new(InformationSchemaTables::new(self.config.clone()))], + None, + vec![], + false, + None, + )?)) + } + + /// Creates an execution plan for a `DESCRIBE ` statement, where + /// `schema` is the output schema of the query. + pub fn describe_schema(&self, schema: &Schema) -> Result> { + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Describe); + for (position, field) in schema.fields().iter().enumerate() { + builder.add_column("", "", "", position, field); + } + record_batch_to_exec(builder.finish()) + } + + /// Creates an execution plan for a `DESCRIBE ` or + /// `SHOW [FULL|EXTENDED] COLUMNS FROM
` statement. + pub async fn show_columns( + &self, + table_ref: &ResolvedTableReference, + detail: ColumnsDetail, + ) -> Result> { + let table = self.resolve_table(table_ref).await?; + + let mut builder = InformationSchemaColumnsBuilder::new(detail); + for (position, field) in table.schema().fields().iter().enumerate() { + builder.add_column( + &table_ref.catalog, + &table_ref.schema, + &table_ref.table, + position, + field, + ); + } + record_batch_to_exec(builder.finish()) + } + + /// Creates an execution plan for a `SHOW CREATE TABLE
` statement. + pub async fn show_create_table( + &self, + table_ref: &ResolvedTableReference, + ) -> Result> { + let table = self.resolve_table(table_ref).await?; + + let mut builder = InformationSchemaViewBuilder::new(); + builder.add_view( + table_ref.catalog.as_ref(), + table_ref.schema.as_ref(), + table_ref.table.as_ref(), + table.get_table_definition(), + ); + let batch = builder.finish(); + record_batch_to_exec(batch) + } + + async fn resolve_table( + &self, + table_ref: &ResolvedTableReference, + ) -> Result, DataFusionError> { + let not_found = || plan_datafusion_err!("table '{table_ref}' not found"); + + // Look up the catalog before the `information_schema` special case so that + // `information_schema` tables in a non-existent catalog are reported as not found. + let catalog = self + .config + .catalog_list + .catalog(&table_ref.catalog) + .ok_or_else(not_found)?; + + let table = if table_ref.schema.as_ref() == INFORMATION_SCHEMA + && self.config.information_schema_enabled + { + self.table(&table_ref.table).await? + } else { + catalog + .schema(&table_ref.schema) + .ok_or_else(not_found)? + .table(&table_ref.table) + .await? + } + .ok_or_else(not_found)?; + Ok(table) + } + + /// Creates an execution plan for a `SHOW ` statement, or for + /// `SHOW ALL` when `name` is `None`. + pub fn show_variables( + &self, + config_options: &ConfigOptions, + runtime_env: &Arc, + name: Option<&str>, + verbose: bool, + ) -> Result> { + let name = name.map(|n| { + if n == "timezone" || n == "time.zone" { + // we could introduce alias in OptionDefinition if this string matching thing grows + "datafusion.execution.time_zone" + } else { + n + } + }); + + if let Some(name) = name { + // Report unknown variables as an error rather than an empty result + let is_valid_variable = + config_options.entries().iter().any(|opt| opt.key == name); + let is_runtime_variable = name.starts_with("datafusion.runtime."); + if !is_valid_variable && !is_runtime_variable { + return plan_err!( + "'{name}' is not a variable which can be viewed with 'SHOW'" + ); + } + } + let mut builder = InformationSchemaDfSettingsBuilder::new(verbose); + self.config + .make_df_settings(config_options, runtime_env, name, &mut builder); + let batch = builder.finish(); + record_batch_to_exec(batch) + } + + /// Creates an execution plan for a `SHOW FUNCTIONS [LIKE ]` + /// statement. `filter` is the `LIKE` pattern. + pub fn show_functions( + &self, + udfs: &HashMap>, + udafs: &HashMap>, + udwfs: &HashMap>, + filter: Option<&str>, + ) -> Result> { + let mut builder = ShowFunctionsBuilder::new(); + + for (name, udf) in udfs { + for (arg_types, return_type) in get_udf_args_and_return_types(udf)? { + builder.add_overload( + name, + &arg_types, + return_type.as_deref(), + udf.documentation(), + "SCALAR", + ); + } + } + for (name, udaf) in udafs { + for (arg_types, return_type) in get_udaf_args_and_return_types(udaf)? { + builder.add_overload( + name, + &arg_types, + return_type.as_deref(), + udaf.documentation(), + "AGGREGATE", + ); + } + } + for (name, udwf) in udwfs { + for (arg_types, return_type) in get_udwf_args_and_return_types(udwf)? { + builder.add_overload( + name, + &arg_types, + return_type.as_deref(), + udwf.documentation(), + "WINDOW", + ); + } + } + // UDTFs have no signature or documentation + for name in self.config.table_functions.keys() { + builder.add_table_function(name); + } + + record_batch_to_exec(builder.finish(filter)?) + } +} + +/// Wraps a [`RecordBatch`] in a single partition in-memory [`ExecutionPlan`]. +fn record_batch_to_exec(record_batch: RecordBatch) -> Result> { + let schema = record_batch.schema(); + let partitions = vec![vec![record_batch]]; + let mem_exec = MemorySourceConfig::try_new_exec(&partitions, schema, None)?; + Ok(mem_exec) +} + +/// Builds `SHOW FUNCTIONS` rows: one per signature of each scalar, aggregate +/// and window function, plus one per table function. +struct ShowFunctionsBuilder { + function_names: StringBuilder, + return_types: StringBuilder, + parameters: ListBuilder, + parameter_types: ListBuilder, + function_types: StringBuilder, + descriptions: StringBuilder, + syntax_examples: StringBuilder, +} + +impl ShowFunctionsBuilder { + fn new() -> Self { + Self { + function_names: StringBuilder::new(), + return_types: StringBuilder::new(), + parameters: ListBuilder::new(StringBuilder::new()), + parameter_types: ListBuilder::new(StringBuilder::new()), + function_types: StringBuilder::new(), + descriptions: StringBuilder::new(), + syntax_examples: StringBuilder::new(), + } + } + + /// Append one row for a single function signature. + fn add_overload( + &mut self, + name: &str, + arg_types: &[String], + return_type: Option<&str>, + documentation: Option<&Documentation>, + function_type: &str, + ) { + self.function_names.append_value(name); + self.return_types.append_option(return_type); + + // Documented argument names are matched to the signature by position + let arg_names = documentation.and_then(|d| d.arguments.as_ref()); + for (position, type_name) in arg_types.iter().enumerate() { + let param_name = arg_names + .and_then(|args| args.get(position)) + .map(|(arg_name, _)| arg_name.as_str()); + self.parameters.values().append_option(param_name); + self.parameter_types.values().append_value(type_name); + } + self.parameters.append(true); + self.parameter_types.append(true); + + self.function_types.append_value(function_type); + self.descriptions + .append_option(documentation.map(|d| d.description.as_str())); + self.syntax_examples + .append_option(documentation.map(|d| d.syntax_example.as_str())); + } + + /// Append one row for a table function. + fn add_table_function(&mut self, name: &str) { + self.function_names.append_value(name); + self.return_types.append_value("TABLE"); + self.parameters.append(false); + self.parameter_types.append(false); + self.function_types.append_value("TABLE"); + self.descriptions.append_null(); + self.syntax_examples.append_null(); + } + + /// Finalize into a [`RecordBatch`], keeping only the functions whose name + /// matches the `LIKE` pattern `filter`. + fn finish(mut self, filter: Option<&str>) -> Result { + let batch = RecordBatch::try_new( + SchemaRef::from(show_functions_schema()), + vec![ + Arc::new(self.function_names.finish()), + Arc::new(self.return_types.finish()), + Arc::new(self.parameters.finish()), + Arc::new(self.parameter_types.finish()), + Arc::new(self.function_types.finish()), + Arc::new(self.descriptions.finish()), + Arc::new(self.syntax_examples.finish()), + ], + )?; + match filter { + Some(pattern) => { + let mask = like(batch.column(0), &StringArray::new_scalar(pattern))?; + Ok(filter_record_batch(&batch, &mask)?) + } + None => Ok(batch), + } + } } #[derive(Clone, Debug)] struct InformationSchemaConfig { - system_catalog_name: Option, + information_schema_enabled: bool, catalog_list: Arc, + catalog_filter: Option, table_functions: HashMap>, } impl InformationSchemaConfig { + /// The catalogs to report: only `catalog_filter` if set, otherwise all. + fn catalogs(&self) -> Result)>> { + match &self.catalog_filter { + Some(catalog_name) => { + let catalog = + self.catalog_list.catalog(catalog_name).ok_or_else(|| { + plan_datafusion_err!("catalog '{catalog_name}' not found") + })?; + Ok(vec![(catalog_name.clone(), catalog)]) + } + // Skip catalogs that were deregistered after `catalog_names` was called + None => Ok(self + .catalog_list + .catalog_names() + .into_iter() + .filter_map(|name| { + let catalog = self.catalog_list.catalog(&name)?; + Some((name, catalog)) + }) + .collect()), + } + } + /// Construct the `information_schema.tables` virtual table async fn make_tables( &self, builder: &mut InformationSchemaTablesBuilder, ) -> Result<(), DataFusionError> { - // create a mem table with the names of tables - let catalog_names = self.catalog_list.catalog_names(); - - for catalog_name in &catalog_names { - let catalog = self.catalog_list.catalog(catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table_type) = - schema.table_type(&table_name).await? - { - builder.add_table( - catalog_name, - &schema_name, - &table_name, - table_type, - ); - } + for (catalog_name, catalog) in self.catalogs()? { + self.add_tables(builder, &catalog_name, catalog.as_ref()) + .await?; + } + Ok(()) + } + + async fn add_tables( + &self, + builder: &mut InformationSchemaTablesBuilder, + catalog_name: &str, + catalog: &dyn CatalogProvider, + ) -> Result<(), DataFusionError> { + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table_type) = schema.table_type(&table_name).await? { + builder.add_table( + catalog_name, + &schema_name, + &table_name, + table_type, + ); } } } } - + } + if self.information_schema_enabled { // Add a final list for the information schema tables themselves for table_name in INFORMATION_SCHEMA_TABLES { builder.add_table( @@ -205,34 +495,30 @@ impl InformationSchemaConfig { ); } } + Ok(()) + } - if let Some(system_catalog_name) = &self.system_catalog_name - && !catalog_names.contains(system_catalog_name) - { - for table_name in INFORMATION_SCHEMA_TABLES { - builder.add_table( - system_catalog_name, - INFORMATION_SCHEMA, - table_name, - TableType::View, - ); - } + fn make_schemata( + &self, + builder: &mut InformationSchemataBuilder, + ) -> Result<(), DataFusionError> { + for (catalog_name, catalog) in self.catalogs()? { + Self::add_schemata(builder, &catalog_name, catalog.as_ref()); } - Ok(()) } - fn make_schemata(&self, builder: &mut InformationSchemataBuilder) { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA - && let Some(schema) = catalog.schema(&schema_name) - { - let schema_owner = schema.owner_name(); - builder.add_schemata(&catalog_name, &schema_name, schema_owner); - } + fn add_schemata( + builder: &mut InformationSchemataBuilder, + catalog_name: &str, + catalog: &dyn CatalogProvider, + ) { + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA + && let Some(schema) = catalog.schema(&schema_name) + { + let schema_owner = schema.owner_name(); + builder.add_schemata(catalog_name, &schema_name, schema_owner); } } } @@ -241,42 +527,32 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaViewBuilder, ) -> Result<(), DataFusionError> { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - Self::add_views( - builder, - &catalog_name, - &schema_name, - schema.as_ref(), - ) - .await?; - } - } - } + for (catalog_name, catalog) in self.catalogs()? { + Self::add_views(builder, &catalog_name, catalog.as_ref()).await?; } - Ok(()) } async fn add_views( builder: &mut InformationSchemaViewBuilder, catalog_name: &str, - schema_name: &str, - schema: &dyn SchemaProvider, + catalog: &dyn CatalogProvider, ) -> Result<(), DataFusionError> { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - builder.add_view( - catalog_name, - schema_name, - &table_name, - table.get_table_definition(), - ) + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + builder.add_view( + catalog_name, + &schema_name, + &table_name, + table.get_table_definition(), + ) + } + } + } } } Ok(()) @@ -287,63 +563,60 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaColumnsBuilder, ) -> Result<(), DataFusionError> { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - Self::add_columns( - builder, - &catalog_name, - &schema_name, - schema.as_ref(), - ) - .await?; - } - } - } + for (catalog_name, catalog) in self.catalogs()? { + Self::add_columns(builder, &catalog_name, catalog.as_ref()).await?; } - Ok(()) } async fn add_columns( builder: &mut InformationSchemaColumnsBuilder, catalog_name: &str, - schema_name: &str, - schema: &dyn SchemaProvider, + catalog: &dyn CatalogProvider, ) -> Result<(), DataFusionError> { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - for (field_position, field) in table.schema().fields().iter().enumerate() - { - builder.add_column( - catalog_name, - schema_name, - &table_name, - field_position, - field, - ) + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + for (field_position, field) in + table.schema().fields().iter().enumerate() + { + builder.add_column( + catalog_name, + &schema_name, + &table_name, + field_position, + field, + ) + } + } + } } } } Ok(()) } - /// Construct the `information_schema.df_settings` virtual table + /// Construct the `information_schema.df_settings` virtual table, restricted + /// to the setting `name` if given. All settings are ordered by name. fn make_df_settings( &self, config_options: &ConfigOptions, runtime_env: &Arc, + name: Option<&str>, builder: &mut InformationSchemaDfSettingsBuilder, ) { - for entry in config_options.entries() { - builder.add_setting(entry); + let mut entries = config_options.entries(); + entries.extend(runtime_env.config_entries()); + + match name { + Some(name) => entries.retain(|entry| entry.key == name), + None => entries.sort_unstable_by(|a, b| a.key.cmp(&b.key)), } - // Add runtime configuration entries - for entry in runtime_env.config_entries() { + + for entry in entries { builder.add_setting(entry); } } @@ -419,10 +692,7 @@ impl InformationSchemaConfig { } } - // Table functions (UDTFs) don't have scalar signatures; their return - // type is always a table, so emit a single row per UDTF with - // routine_type = "FUNCTION", function_type = "TABLE" and - // data_type = "TABLE". + // UDTFs have no signature; report a single `TABLE`-typed row for each for name in self.table_functions.keys() { builder.add_routine( catalog_name, @@ -537,13 +807,8 @@ impl InformationSchemaConfig { } } - // UDTFs deliberately do NOT appear in `information_schema.parameters`. - // A same-named scalar UDF (e.g. `generate_series` exists as both a - // scalar UDF in functions-nested and a UDTF in functions-table) would - // cross-join with a UDTF row keyed only by (name, rid) and produce - // spurious `TABLE`-typed variants of every scalar signature in - // SHOW FUNCTIONS. `show_functions_to_plan` sources UDTFs directly - // from `information_schema.routines` via a UNION branch instead. + // UDTFs are deliberately omitted: a scalar UDF with the same name (e.g. + // `generate_series`) would be indistinguishable from it here. Ok(()) } @@ -661,24 +926,6 @@ fn remove_native_type_prefix(native_type: &NativeType) -> String { #[async_trait] impl SchemaProvider for InformationSchemaProvider { - fn table_names(&self) -> Vec { - self.config.table_names() - } - - async fn table( - &self, - name: &str, - ) -> Result>, DataFusionError> { - self.config.get_table(name) - } - - fn table_exist(&self, name: &str) -> bool { - self.config.table_exist(name) - } -} - -#[async_trait] -impl SchemaProvider for InformationSchemaConfig { fn table_names(&self) -> Vec { INFORMATION_SCHEMA_TABLES .iter() @@ -690,7 +937,7 @@ impl SchemaProvider for InformationSchemaConfig { &self, name: &str, ) -> Result>, DataFusionError> { - self.get_table(name) + self.config.get_table(name) } fn table_exist(&self, name: &str) -> bool { @@ -729,23 +976,9 @@ struct InformationSchemaTables { impl InformationSchemaTables { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("table_catalog", DataType::Utf8, false), - Field::new("table_schema", DataType::Utf8, false), - Field::new("table_name", DataType::Utf8, false), - Field::new("table_type", DataType::Utf8, false), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaTablesBuilder { - InformationSchemaTablesBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - table_types: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + schema: Arc::new(tables_schema()), + config, } } } @@ -756,7 +989,7 @@ impl PartitionStream for InformationSchemaTables { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemaTablesBuilder::new(); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), @@ -769,50 +1002,6 @@ impl PartitionStream for InformationSchemaTables { } } -/// Builds the `information_schema.TABLE` table row by row -/// -/// Columns are based on -struct InformationSchemaTablesBuilder { - schema: SchemaRef, - catalog_names: StringBuilder, - schema_names: StringBuilder, - table_names: StringBuilder, - table_types: StringBuilder, -} - -impl InformationSchemaTablesBuilder { - fn add_table( - &mut self, - catalog_name: impl AsRef, - schema_name: impl AsRef, - table_name: impl AsRef, - table_type: TableType, - ) { - // Note: append_value is actually infallible. - self.catalog_names.append_value(catalog_name.as_ref()); - self.schema_names.append_value(schema_name.as_ref()); - self.table_names.append_value(table_name.as_ref()); - self.table_types.append_value(match table_type { - TableType::Base => "BASE TABLE", - TableType::View => "VIEW", - TableType::Temporary => "LOCAL TEMPORARY", - }); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_names.finish()), - Arc::new(self.schema_names.finish()), - Arc::new(self.table_names.finish()), - Arc::new(self.table_types.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemaViews { schema: SchemaRef, @@ -821,23 +1010,9 @@ struct InformationSchemaViews { impl InformationSchemaViews { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("table_catalog", DataType::Utf8, false), - Field::new("table_schema", DataType::Utf8, false), - Field::new("table_name", DataType::Utf8, false), - Field::new("definition", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaViewBuilder { - InformationSchemaViewBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - definitions: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + schema: Arc::new(views_schema()), + config, } } } @@ -848,7 +1023,7 @@ impl PartitionStream for InformationSchemaViews { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemaViewBuilder::new(); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), @@ -861,46 +1036,6 @@ impl PartitionStream for InformationSchemaViews { } } -/// Builds the `information_schema.VIEWS` table row by row -/// -/// Columns are based on -struct InformationSchemaViewBuilder { - schema: SchemaRef, - catalog_names: StringBuilder, - schema_names: StringBuilder, - table_names: StringBuilder, - definitions: StringBuilder, -} - -impl InformationSchemaViewBuilder { - fn add_view( - &mut self, - catalog_name: impl AsRef, - schema_name: impl AsRef, - table_name: impl AsRef, - definition: Option<&(impl AsRef + ?Sized)>, - ) { - // Note: append_value is actually infallible. - self.catalog_names.append_value(catalog_name.as_ref()); - self.schema_names.append_value(schema_name.as_ref()); - self.table_names.append_value(table_name.as_ref()); - self.definitions.append_option(definition.as_ref()); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_names.finish()), - Arc::new(self.schema_names.finish()), - Arc::new(self.table_names.finish()), - Arc::new(self.definitions.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemaColumns { schema: SchemaRef, @@ -909,50 +1044,9 @@ struct InformationSchemaColumns { impl InformationSchemaColumns { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("table_catalog", DataType::Utf8, false), - Field::new("table_schema", DataType::Utf8, false), - Field::new("table_name", DataType::Utf8, false), - Field::new("column_name", DataType::Utf8, false), - Field::new("ordinal_position", DataType::UInt64, false), - Field::new("column_default", DataType::Utf8, true), - Field::new("is_nullable", DataType::Utf8, false), - Field::new("data_type", DataType::Utf8, false), - Field::new("character_maximum_length", DataType::UInt64, true), - Field::new("character_octet_length", DataType::UInt64, true), - Field::new("numeric_precision", DataType::UInt64, true), - Field::new("numeric_precision_radix", DataType::UInt64, true), - Field::new("numeric_scale", DataType::UInt64, true), - Field::new("datetime_precision", DataType::UInt64, true), - Field::new("interval_type", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaColumnsBuilder { - // StringBuilder requires providing an initial capacity, so - // pick 10 here arbitrarily as this is not performance - // critical code and the number of tables is unavailable here. - let default_capacity = 10; - - InformationSchemaColumnsBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - column_names: StringBuilder::new(), - ordinal_positions: UInt64Builder::with_capacity(default_capacity), - column_defaults: StringBuilder::new(), - is_nullables: StringBuilder::new(), - data_types: StringBuilder::new(), - character_maximum_lengths: UInt64Builder::with_capacity(default_capacity), - character_octet_lengths: UInt64Builder::with_capacity(default_capacity), - numeric_precisions: UInt64Builder::with_capacity(default_capacity), - numeric_precision_radixes: UInt64Builder::with_capacity(default_capacity), - numeric_scales: UInt64Builder::with_capacity(default_capacity), - datetime_precisions: UInt64Builder::with_capacity(default_capacity), - interval_types: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + schema: Arc::new(columns_schema(ColumnsDetail::Full)), + config, } } } @@ -963,7 +1057,7 @@ impl PartitionStream for InformationSchemaColumns { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Full); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), @@ -976,281 +1070,19 @@ impl PartitionStream for InformationSchemaColumns { } } -/// Builds the `information_schema.COLUMNS` table row by row -/// -/// Columns are based on -struct InformationSchemaColumnsBuilder { - schema: SchemaRef, - catalog_names: StringBuilder, - schema_names: StringBuilder, - table_names: StringBuilder, - column_names: StringBuilder, - ordinal_positions: UInt64Builder, - column_defaults: StringBuilder, - is_nullables: StringBuilder, - data_types: StringBuilder, - character_maximum_lengths: UInt64Builder, - character_octet_lengths: UInt64Builder, - numeric_precisions: UInt64Builder, - numeric_precision_radixes: UInt64Builder, - numeric_scales: UInt64Builder, - datetime_precisions: UInt64Builder, - interval_types: StringBuilder, -} - -impl InformationSchemaColumnsBuilder { - fn add_column( - &mut self, - catalog_name: &str, - schema_name: &str, - table_name: &str, - field_position: usize, - field: &Field, - ) { - use DataType::*; - - // Note: append_value is actually infallible. - self.catalog_names.append_value(catalog_name); - self.schema_names.append_value(schema_name); - self.table_names.append_value(table_name); - - self.column_names.append_value(field.name()); - - self.ordinal_positions.append_value(field_position as u64); - - // DataFusion does not support column default values, so null - self.column_defaults.append_null(); - - // "YES if the column is possibly nullable, NO if it is known not nullable. " - let nullable_str = if field.is_nullable() { "YES" } else { "NO" }; - self.is_nullables.append_value(nullable_str); - - // "System supplied type" --> Use debug format of the datatype - self.data_types.append_value(field.data_type().to_string()); - - // "If data_type identifies a character or bit string type, the - // declared maximum length; null for all other data types or - // if no maximum length was declared." - // - // Arrow has no equivalent of VARCHAR(20), so we leave this as Null - let max_chars = None; - self.character_maximum_lengths.append_option(max_chars); - - // "Maximum length, in bytes, for binary data, character data, - // or text and image data." - let char_len: Option = match field.data_type() { - Utf8 | Binary => Some(i32::MAX as u64), - LargeBinary | LargeUtf8 => Some(i64::MAX as u64), - _ => None, - }; - self.character_octet_lengths.append_option(char_len); - - // numeric_precision: "If data_type identifies a numeric type, this column - // contains the (declared or implicit) precision of the type - // for this column. The precision indicates the number of - // significant digits. It can be expressed in decimal (base - // 10) or binary (base 2) terms, as specified in the column - // numeric_precision_radix. For all other data types, this - // column is null." - // - // numeric_radix: If data_type identifies a numeric type, this - // column indicates in which base the values in the columns - // numeric_precision and numeric_scale are expressed. The - // value is either 2 or 10. For all other data types, this - // column is null. - // - // numeric_scale: If data_type identifies an exact numeric - // type, this column contains the (declared or implicit) scale - // of the type for this column. The scale indicates the number - // of significant digits to the right of the decimal point. It - // can be expressed in decimal (base 10) or binary (base 2) - // terms, as specified in the column - // numeric_precision_radix. For all other data types, this - // column is null. - let (numeric_precision, numeric_radix, numeric_scale) = match field.data_type() { - Int8 | UInt8 => (Some(8), Some(2), None), - Int16 | UInt16 => (Some(16), Some(2), None), - Int32 | UInt32 => (Some(32), Some(2), None), - // From max value of 65504 as explained on - // https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Exponent_encoding - Float16 => (Some(15), Some(2), None), - // Numbers from postgres `real` type - Float32 => (Some(24), Some(2), None), - // Numbers from postgres `double` type - Float64 => (Some(24), Some(2), None), - Decimal128(precision, scale) => { - (Some(*precision as u64), Some(10), Some(*scale as u64)) - } - _ => (None, None, None), - }; - - self.numeric_precisions.append_option(numeric_precision); - self.numeric_precision_radixes.append_option(numeric_radix); - self.numeric_scales.append_option(numeric_scale); - - self.datetime_precisions.append_option(None); - self.interval_types.append_null(); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_names.finish()), - Arc::new(self.schema_names.finish()), - Arc::new(self.table_names.finish()), - Arc::new(self.column_names.finish()), - Arc::new(self.ordinal_positions.finish()), - Arc::new(self.column_defaults.finish()), - Arc::new(self.is_nullables.finish()), - Arc::new(self.data_types.finish()), - Arc::new(self.character_maximum_lengths.finish()), - Arc::new(self.character_octet_lengths.finish()), - Arc::new(self.numeric_precisions.finish()), - Arc::new(self.numeric_precision_radixes.finish()), - Arc::new(self.numeric_scales.finish()), - Arc::new(self.datetime_precisions.finish()), - Arc::new(self.interval_types.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemata { schema: SchemaRef, config: InformationSchemaConfig, } -/// The Arrow schema of [`information_schema.schemata`] rows. -/// -/// Useful for downstream catalog implementations that want to declare a -/// `TableProvider` for `schemata` before populating any rows via -/// [`InformationSchemataBuilder`]. -/// -/// Columns and nullability match -/// . -/// -/// [`information_schema.schemata`]: https://www.postgresql.org/docs/current/infoschema-schemata.html -pub fn schemata_schema() -> SchemaRef { - Arc::new(Schema::new(vec![ - Field::new("catalog_name", DataType::Utf8, false), - Field::new("schema_name", DataType::Utf8, false), - Field::new("schema_owner", DataType::Utf8, true), - Field::new("default_character_set_catalog", DataType::Utf8, true), - Field::new("default_character_set_schema", DataType::Utf8, true), - Field::new("default_character_set_name", DataType::Utf8, true), - Field::new("sql_path", DataType::Utf8, true), - ])) -} - impl InformationSchemata { fn new(config: InformationSchemaConfig) -> Self { Self { - schema: schemata_schema(), + schema: Arc::new(schemata_schema()), config, } } - - fn builder(&self) -> InformationSchemataBuilder { - InformationSchemataBuilder { - schema: Arc::clone(&self.schema), - catalog_name: StringBuilder::new(), - schema_name: StringBuilder::new(), - schema_owner: StringBuilder::new(), - default_character_set_catalog: StringBuilder::new(), - default_character_set_schema: StringBuilder::new(), - default_character_set_name: StringBuilder::new(), - sql_path: StringBuilder::new(), - } - } -} - -/// Builder that produces [`RecordBatch`] values matching the schema of -/// `information_schema.schemata` (see [`schemata_schema`]). -/// -/// Intended for downstream catalog implementations that need to emit -/// `schemata` rows from their own metadata source rather than going -/// through DataFusion's `InformationSchemaProvider`, which enumerates -/// schemas synchronously via `CatalogProviderList` and so is unsuitable -/// for catalog backends that resolve asynchronously. -#[derive(Debug)] -pub struct InformationSchemataBuilder { - schema: SchemaRef, - catalog_name: StringBuilder, - schema_name: StringBuilder, - schema_owner: StringBuilder, - default_character_set_catalog: StringBuilder, - default_character_set_schema: StringBuilder, - default_character_set_name: StringBuilder, - sql_path: StringBuilder, -} - -impl Default for InformationSchemataBuilder { - fn default() -> Self { - Self::new() - } -} - -impl InformationSchemataBuilder { - /// Construct an empty builder. - pub fn new() -> Self { - Self { - schema: schemata_schema(), - catalog_name: StringBuilder::new(), - schema_name: StringBuilder::new(), - schema_owner: StringBuilder::new(), - default_character_set_catalog: StringBuilder::new(), - default_character_set_schema: StringBuilder::new(), - default_character_set_name: StringBuilder::new(), - sql_path: StringBuilder::new(), - } - } - - /// Append one row to the builder. `schema_owner` is the optional SQL - /// schema owner; the three `default_character_set_*` columns and - /// `sql_path` are written as null (DataFusion does not model those - /// concepts; see the PostgreSQL docs link on [`schemata_schema`]). - pub fn add_schemata( - &mut self, - catalog_name: &str, - schema_name: &str, - schema_owner: Option<&str>, - ) { - self.catalog_name.append_value(catalog_name); - self.schema_name.append_value(schema_name); - match schema_owner { - Some(owner) => self.schema_owner.append_value(owner), - None => self.schema_owner.append_null(), - } - self.default_character_set_catalog.append_null(); - self.default_character_set_schema.append_null(); - self.default_character_set_name.append_null(); - self.sql_path.append_null(); - } - - /// Finalize the builder into a [`RecordBatch`]. - /// - /// Returns an error only if Arrow buffer construction fails, which - /// the builder's column-count and type invariants make unreachable - /// under normal use. The `Result` return type preserves room to add - /// validation in the future without a breaking API change. - pub fn finish(&mut self) -> Result { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_name.finish()), - Arc::new(self.schema_name.finish()), - Arc::new(self.schema_owner.finish()), - Arc::new(self.default_character_set_catalog.finish()), - Arc::new(self.default_character_set_schema.finish()), - Arc::new(self.default_character_set_name.finish()), - Arc::new(self.sql_path.finish()), - ], - ) - .map_err(DataFusionError::from) - } } impl PartitionStream for InformationSchemata { @@ -1259,13 +1091,13 @@ impl PartitionStream for InformationSchemata { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemataBuilder::new(); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), // TODO: Stream this futures::stream::once(async move { - config.make_schemata(&mut builder); + config.make_schemata(&mut builder)?; builder.finish() }), )) @@ -1280,21 +1112,9 @@ struct InformationSchemaDfSettings { impl InformationSchemaDfSettings { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("name", DataType::Utf8, false), - Field::new("value", DataType::Utf8, true), - Field::new("description", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaDfSettingsBuilder { - InformationSchemaDfSettingsBuilder { - names: StringBuilder::new(), - values: StringBuilder::new(), - descriptions: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + schema: Arc::new(df_settings_schema(true)), + config, } } } @@ -1306,7 +1126,7 @@ impl PartitionStream for InformationSchemaDfSettings { fn execute(&self, ctx: Arc) -> SendableRecordBatchStream { let config = self.config.clone(); - let mut builder = self.builder(); + let mut builder = InformationSchemaDfSettingsBuilder::new(true); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), // TODO: Stream this @@ -1316,6 +1136,7 @@ impl PartitionStream for InformationSchemaDfSettings { config.make_df_settings( ctx.session_config().options(), &runtime_env, + None, &mut builder, ); Ok(builder.finish()) @@ -1324,33 +1145,6 @@ impl PartitionStream for InformationSchemaDfSettings { } } -struct InformationSchemaDfSettingsBuilder { - schema: SchemaRef, - names: StringBuilder, - values: StringBuilder, - descriptions: StringBuilder, -} - -impl InformationSchemaDfSettingsBuilder { - fn add_setting(&mut self, entry: ConfigEntry) { - self.names.append_value(entry.key); - self.values.append_option(entry.value); - self.descriptions.append_value(entry.description); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.names.finish()), - Arc::new(self.values.finish()), - Arc::new(self.descriptions.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemaRoutines { schema: SchemaRef, @@ -1359,109 +1153,13 @@ struct InformationSchemaRoutines { impl InformationSchemaRoutines { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("specific_catalog", DataType::Utf8, false), - Field::new("specific_schema", DataType::Utf8, false), - Field::new("specific_name", DataType::Utf8, false), - Field::new("routine_catalog", DataType::Utf8, false), - Field::new("routine_schema", DataType::Utf8, false), - Field::new("routine_name", DataType::Utf8, false), - Field::new("routine_type", DataType::Utf8, false), - Field::new("is_deterministic", DataType::Boolean, true), - Field::new("data_type", DataType::Utf8, true), - Field::new("function_type", DataType::Utf8, true), - Field::new("description", DataType::Utf8, true), - Field::new("syntax_example", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaRoutinesBuilder { - InformationSchemaRoutinesBuilder { - schema: Arc::clone(&self.schema), - specific_catalog: StringBuilder::new(), - specific_schema: StringBuilder::new(), - specific_name: StringBuilder::new(), - routine_catalog: StringBuilder::new(), - routine_schema: StringBuilder::new(), - routine_name: StringBuilder::new(), - routine_type: StringBuilder::new(), - is_deterministic: BooleanBuilder::new(), - data_type: StringBuilder::new(), - function_type: StringBuilder::new(), - description: StringBuilder::new(), - syntax_example: StringBuilder::new(), + Self { + schema: Arc::new(routines_schema()), + config, } } } -struct InformationSchemaRoutinesBuilder { - schema: SchemaRef, - specific_catalog: StringBuilder, - specific_schema: StringBuilder, - specific_name: StringBuilder, - routine_catalog: StringBuilder, - routine_schema: StringBuilder, - routine_name: StringBuilder, - routine_type: StringBuilder, - is_deterministic: BooleanBuilder, - data_type: StringBuilder, - function_type: StringBuilder, - description: StringBuilder, - syntax_example: StringBuilder, -} - -impl InformationSchemaRoutinesBuilder { - #[expect(clippy::too_many_arguments)] - fn add_routine( - &mut self, - catalog_name: impl AsRef, - schema_name: impl AsRef, - routine_name: impl AsRef, - routine_type: impl AsRef, - is_deterministic: bool, - data_type: Option<&impl AsRef>, - function_type: impl AsRef, - description: Option>, - syntax_example: Option>, - ) { - self.specific_catalog.append_value(catalog_name.as_ref()); - self.specific_schema.append_value(schema_name.as_ref()); - self.specific_name.append_value(routine_name.as_ref()); - self.routine_catalog.append_value(catalog_name.as_ref()); - self.routine_schema.append_value(schema_name.as_ref()); - self.routine_name.append_value(routine_name.as_ref()); - self.routine_type.append_value(routine_type.as_ref()); - self.is_deterministic.append_value(is_deterministic); - self.data_type.append_option(data_type.as_ref()); - self.function_type.append_value(function_type.as_ref()); - self.description.append_option(description); - self.syntax_example.append_option(syntax_example); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.specific_catalog.finish()), - Arc::new(self.specific_schema.finish()), - Arc::new(self.specific_name.finish()), - Arc::new(self.routine_catalog.finish()), - Arc::new(self.routine_schema.finish()), - Arc::new(self.routine_name.finish()), - Arc::new(self.routine_type.finish()), - Arc::new(self.is_deterministic.finish()), - Arc::new(self.data_type.finish()), - Arc::new(self.function_type.finish()), - Arc::new(self.description.finish()), - Arc::new(self.syntax_example.finish()), - ], - ) - .unwrap() - } -} - impl PartitionStream for InformationSchemaRoutines { fn schema(&self) -> &SchemaRef { &self.schema @@ -1469,7 +1167,7 @@ impl PartitionStream for InformationSchemaRoutines { fn execute(&self, ctx: Arc) -> SendableRecordBatchStream { let config = self.config.clone(); - let mut builder = self.builder(); + let mut builder = InformationSchemaRoutinesBuilder::new(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), futures::stream::once(async move { @@ -1494,106 +1192,13 @@ struct InformationSchemaParameters { impl InformationSchemaParameters { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("specific_catalog", DataType::Utf8, false), - Field::new("specific_schema", DataType::Utf8, false), - Field::new("specific_name", DataType::Utf8, false), - Field::new("ordinal_position", DataType::UInt64, false), - Field::new("parameter_mode", DataType::Utf8, false), - Field::new("parameter_name", DataType::Utf8, true), - Field::new("data_type", DataType::Utf8, false), - Field::new("parameter_default", DataType::Utf8, true), - Field::new("is_variadic", DataType::Boolean, false), - // `rid` (short for `routine id`) is used to differentiate parameters from different signatures - // (It serves as the group-by key when generating the `SHOW FUNCTIONS` query). - // For example, the following signatures have different `rid` values: - // - `datetrunc(Utf8, Timestamp(Microsecond, Some("+TZ"))) -> Timestamp(Microsecond, Some("+TZ"))` - // - `datetrunc(Utf8View, Timestamp(Nanosecond, None)) -> Timestamp(Nanosecond, None)` - Field::new("rid", DataType::UInt8, false), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaParametersBuilder { - InformationSchemaParametersBuilder { - schema: Arc::clone(&self.schema), - specific_catalog: StringBuilder::new(), - specific_schema: StringBuilder::new(), - specific_name: StringBuilder::new(), - ordinal_position: UInt64Builder::new(), - parameter_mode: StringBuilder::new(), - parameter_name: StringBuilder::new(), - data_type: StringBuilder::new(), - parameter_default: StringBuilder::new(), - is_variadic: BooleanBuilder::new(), - rid: UInt8Builder::new(), + Self { + schema: Arc::new(parameters_schema()), + config, } } } -struct InformationSchemaParametersBuilder { - schema: SchemaRef, - specific_catalog: StringBuilder, - specific_schema: StringBuilder, - specific_name: StringBuilder, - ordinal_position: UInt64Builder, - parameter_mode: StringBuilder, - parameter_name: StringBuilder, - data_type: StringBuilder, - parameter_default: StringBuilder, - is_variadic: BooleanBuilder, - rid: UInt8Builder, -} - -impl InformationSchemaParametersBuilder { - #[expect(clippy::too_many_arguments)] - fn add_parameter( - &mut self, - specific_catalog: impl AsRef, - specific_schema: impl AsRef, - specific_name: impl AsRef, - ordinal_position: u64, - parameter_mode: impl AsRef, - parameter_name: Option<&(impl AsRef + ?Sized)>, - data_type: impl AsRef, - parameter_default: Option>, - is_variadic: bool, - rid: u8, - ) { - self.specific_catalog - .append_value(specific_catalog.as_ref()); - self.specific_schema.append_value(specific_schema.as_ref()); - self.specific_name.append_value(specific_name.as_ref()); - self.ordinal_position.append_value(ordinal_position); - self.parameter_mode.append_value(parameter_mode.as_ref()); - self.parameter_name.append_option(parameter_name.as_ref()); - self.data_type.append_value(data_type.as_ref()); - self.parameter_default.append_option(parameter_default); - self.is_variadic.append_value(is_variadic); - self.rid.append_value(rid); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.specific_catalog.finish()), - Arc::new(self.specific_schema.finish()), - Arc::new(self.specific_name.finish()), - Arc::new(self.ordinal_position.finish()), - Arc::new(self.parameter_mode.finish()), - Arc::new(self.parameter_name.finish()), - Arc::new(self.data_type.finish()), - Arc::new(self.parameter_default.finish()), - Arc::new(self.is_variadic.finish()), - Arc::new(self.rid.finish()), - ], - ) - .unwrap() - } -} - impl PartitionStream for InformationSchemaParameters { fn schema(&self) -> &SchemaRef { &self.schema @@ -1601,7 +1206,7 @@ impl PartitionStream for InformationSchemaParameters { fn execute(&self, ctx: Arc) -> SendableRecordBatchStream { let config = self.config.clone(); - let mut builder = self.builder(); + let mut builder = InformationSchemaParametersBuilder::new(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), futures::stream::once(async move { @@ -1627,74 +1232,26 @@ mod tests { use datafusion_common::ScalarValue; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; - #[test] - fn schemata_builder_emits_canonical_schema_and_rows() { - // Construct via `Default` so the test exercises both `new()` (via - // the `Default` impl) and the public column-layout contract. - let mut builder = InformationSchemataBuilder::default(); - builder.add_schemata("cat", "schema_one", Some("alice")); - builder.add_schemata("cat", "schema_two", None); - let batch = builder.finish().expect("finish should not fail"); - - assert_eq!(batch.schema(), schemata_schema()); - assert_eq!(batch.num_rows(), 2); - - let col = |name: &str| { - batch - .column_by_name(name) - .unwrap_or_else(|| panic!("missing column {name}")) - }; - let string_col = |name: &str| { - col(name) - .as_any() - .downcast_ref::() - .unwrap_or_else(|| panic!("{name} should be a StringArray")) - }; - - let catalog = string_col("catalog_name"); - assert_eq!(catalog.value(0), "cat"); - assert_eq!(catalog.value(1), "cat"); - - let schema = string_col("schema_name"); - assert_eq!(schema.value(0), "schema_one"); - assert_eq!(schema.value(1), "schema_two"); - - let owner = string_col("schema_owner"); - assert_eq!(owner.value(0), "alice"); - assert!(owner.is_null(1)); - - // The three character-set columns and sql_path are unconditionally - // null — they exist for SQL-standard column-layout compatibility. - for name in [ - "default_character_set_catalog", - "default_character_set_schema", - "default_character_set_name", - "sql_path", - ] { - let c = string_col(name); - assert!(c.is_null(0), "{name} row 0 should be null"); - assert!(c.is_null(1), "{name} row 1 should be null"); - } - } - #[tokio::test] async fn make_tables_uses_table_type() { let config = InformationSchemaConfig { - system_catalog_name: None, + information_schema_enabled: true, catalog_list: Arc::new(Fixture), + catalog_filter: None, table_functions: HashMap::new(), }; - let mut builder = InformationSchemaTablesBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - table_types: StringBuilder::new(), - schema: Arc::new(Schema::empty()), - }; + let mut builder = InformationSchemaTablesBuilder::new(); assert!(config.make_tables(&mut builder).await.is_ok()); - assert_eq!("BASE TABLE", builder.table_types.finish().value(0)); + let batch = builder.finish(); + let table_types = batch + .column_by_name("table_type") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!("BASE TABLE", table_types.value(0)); } // UDF diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 8309518b91c2a..a44f3da74b91f 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -242,11 +242,10 @@ config_namespace! { /// virtual tables for displaying schema information pub information_schema: bool, default = false - /// The system catalog name - when set together with `information_schema`, the - /// `information_schema` for each catalog will only describe objects contained in the - /// catalog itself. The global, cross-catalog `information_schema` is available under - /// the virtual catalog named by this option. - pub system_catalog: Option, default = None + /// When set together with `information_schema`, the `information_schema` + /// of each catalog only describes the objects in that catalog. When not + /// set, the `information_schema` of each catalog describes all catalogs. + pub partition_information_schema: bool, default = false /// Location scanned to load tables for `default` schema pub location: Option, default = None diff --git a/datafusion/common/src/information_schema.rs b/datafusion/common/src/information_schema.rs new file mode 100644 index 0000000000000..f25755d2e3589 --- /dev/null +++ b/datafusion/common/src/information_schema.rs @@ -0,0 +1,1046 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Schema and [`RecordBatch`] builders for the `information_schema` virtual +//! tables and `SHOW` statements. + +use crate::Result; +use crate::TableType; +use crate::config::ConfigEntry; +use arrow::array::ArrayRef; +use arrow::array::builder::{BooleanBuilder, StringBuilder, UInt8Builder, UInt64Builder}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; +use std::sync::Arc; + +/// Returns the Arrow schema of `information_schema.tables` rows. +/// +/// Columns are based on . +pub fn tables_schema() -> Schema { + Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("table_type", DataType::Utf8, false), + ]) +} + +/// Builds `information_schema.tables` rows. +#[derive(Debug)] +pub struct InformationSchemaTablesBuilder { + schema: SchemaRef, + catalog_names: StringBuilder, + schema_names: StringBuilder, + table_names: StringBuilder, + table_types: StringBuilder, +} + +impl Default for InformationSchemaTablesBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaTablesBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(tables_schema()), + catalog_names: StringBuilder::new(), + schema_names: StringBuilder::new(), + table_names: StringBuilder::new(), + table_types: StringBuilder::new(), + } + } + + /// Append one row. + pub fn add_table( + &mut self, + catalog_name: impl AsRef, + schema_name: impl AsRef, + table_name: impl AsRef, + table_type: TableType, + ) { + // Note: append_value is actually infallible. + self.catalog_names.append_value(catalog_name.as_ref()); + self.schema_names.append_value(schema_name.as_ref()); + self.table_names.append_value(table_name.as_ref()); + self.table_types.append_value(match table_type { + TableType::Base => "BASE TABLE", + TableType::View => "VIEW", + TableType::Temporary => "LOCAL TEMPORARY", + }); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.catalog_names.finish()), + Arc::new(self.schema_names.finish()), + Arc::new(self.table_names.finish()), + Arc::new(self.table_types.finish()), + ], + ) + .unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.views` rows. +/// +/// Columns are based on . +pub fn views_schema() -> Schema { + Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("definition", DataType::Utf8, true), + ]) +} + +/// Builds `information_schema.views` rows. +#[derive(Debug)] +pub struct InformationSchemaViewBuilder { + schema: SchemaRef, + catalog_names: StringBuilder, + schema_names: StringBuilder, + table_names: StringBuilder, + definitions: StringBuilder, +} + +impl Default for InformationSchemaViewBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaViewBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(views_schema()), + catalog_names: StringBuilder::new(), + schema_names: StringBuilder::new(), + table_names: StringBuilder::new(), + definitions: StringBuilder::new(), + } + } + + /// Append one row. `definition` is the statement that created the table, if known. + pub fn add_view( + &mut self, + catalog_name: impl AsRef, + schema_name: impl AsRef, + table_name: impl AsRef, + definition: Option<&(impl AsRef + ?Sized)>, + ) { + // Note: append_value is actually infallible. + self.catalog_names.append_value(catalog_name.as_ref()); + self.schema_names.append_value(schema_name.as_ref()); + self.table_names.append_value(table_name.as_ref()); + self.definitions.append_option(definition.as_ref()); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.catalog_names.finish()), + Arc::new(self.schema_names.finish()), + Arc::new(self.table_names.finish()), + Arc::new(self.definitions.finish()), + ], + ) + .unwrap() + } +} + +/// Which subset of the `information_schema.columns` columns to produce. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ColumnsDetail { + /// `column_name`, `data_type` and `is_nullable`, as output by `DESCRIBE`. + Describe, + /// [`Self::Describe`] plus the table's catalog, schema and name, as output + /// by `SHOW COLUMNS`. + Basic, + /// All columns, as output by `SHOW FULL COLUMNS` and `information_schema.columns`. + Full, +} + +/// Returns the Arrow schema of `information_schema.columns` rows at the +/// given [`ColumnsDetail`] level. +/// +/// The `Full` columns are based on +/// . +pub fn columns_schema(detail: ColumnsDetail) -> Schema { + match detail { + ColumnsDetail::Describe => Schema::new(vec![ + Field::new("column_name", DataType::Utf8, false), + Field::new("data_type", DataType::Utf8, false), + Field::new("is_nullable", DataType::Utf8, false), + ]), + ColumnsDetail::Basic => Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("column_name", DataType::Utf8, false), + Field::new("data_type", DataType::Utf8, false), + Field::new("is_nullable", DataType::Utf8, false), + ]), + ColumnsDetail::Full => Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("column_name", DataType::Utf8, false), + Field::new("ordinal_position", DataType::UInt64, false), + Field::new("column_default", DataType::Utf8, true), + Field::new("is_nullable", DataType::Utf8, false), + Field::new("data_type", DataType::Utf8, false), + Field::new("character_maximum_length", DataType::UInt64, true), + Field::new("character_octet_length", DataType::UInt64, true), + Field::new("numeric_precision", DataType::UInt64, true), + Field::new("numeric_precision_radix", DataType::UInt64, true), + Field::new("numeric_scale", DataType::UInt64, true), + Field::new("datetime_precision", DataType::UInt64, true), + Field::new("interval_type", DataType::Utf8, true), + ]), + } +} + +/// Builds `information_schema.columns` rows at a given [`ColumnsDetail`] level. +#[derive(Debug)] +pub struct InformationSchemaColumnsBuilder { + schema: SchemaRef, + detail: ColumnsDetail, + catalog_names: Option, + schema_names: Option, + table_names: Option, + column_names: StringBuilder, + ordinal_positions: Option, + column_defaults: Option, + is_nullables: StringBuilder, + data_types: StringBuilder, + character_maximum_lengths: Option, + character_octet_lengths: Option, + numeric_precisions: Option, + numeric_precision_radixes: Option, + numeric_scales: Option, + datetime_precisions: Option, + interval_types: Option, +} + +impl InformationSchemaColumnsBuilder { + /// Construct an empty builder that will only retain the columns needed + /// for `detail`. + pub fn new(detail: ColumnsDetail) -> Self { + // StringBuilder requires providing an initial capacity, so + // pick 10 here arbitrarily as this is not performance + // critical code and the number of tables is unavailable here. + let default_capacity = 10; + + let with_table_qualifiers = + matches!(detail, ColumnsDetail::Basic | ColumnsDetail::Full); + let full = detail == ColumnsDetail::Full; + + Self { + schema: Arc::new(columns_schema(detail)), + detail, + catalog_names: with_table_qualifiers.then(StringBuilder::new), + schema_names: with_table_qualifiers.then(StringBuilder::new), + table_names: with_table_qualifiers.then(StringBuilder::new), + column_names: StringBuilder::new(), + ordinal_positions: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + column_defaults: full.then(StringBuilder::new), + is_nullables: StringBuilder::new(), + data_types: StringBuilder::new(), + character_maximum_lengths: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + character_octet_lengths: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + numeric_precisions: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + numeric_precision_radixes: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + numeric_scales: full.then(|| UInt64Builder::with_capacity(default_capacity)), + datetime_precisions: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + interval_types: full.then(StringBuilder::new), + } + } + + /// Append one row describing `field`. Values not needed at this builder's + /// detail level are discarded. + pub fn add_column( + &mut self, + catalog_name: &str, + schema_name: &str, + table_name: &str, + field_position: usize, + field: &Field, + ) { + use DataType::*; + + // Note: append_value is actually infallible. + if let Some(b) = &mut self.catalog_names { + b.append_value(catalog_name); + } + if let Some(b) = &mut self.schema_names { + b.append_value(schema_name); + } + if let Some(b) = &mut self.table_names { + b.append_value(table_name); + } + + self.column_names.append_value(field.name()); + + if let Some(b) = &mut self.ordinal_positions { + b.append_value(field_position as u64); + } + + // DataFusion does not support column default values, so null + if let Some(b) = &mut self.column_defaults { + b.append_null(); + } + + // "YES if the column is possibly nullable, NO if it is known not nullable. " + let nullable_str = if field.is_nullable() { "YES" } else { "NO" }; + self.is_nullables.append_value(nullable_str); + + // "System supplied type" --> Use display format of the datatype + self.data_types.append_value(field.data_type().to_string()); + + // "If data_type identifies a character or bit string type, the + // declared maximum length; null for all other data types or + // if no maximum length was declared." + // + // Arrow has no equivalent of VARCHAR(20), so we leave this as Null + if let Some(b) = &mut self.character_maximum_lengths { + b.append_option(None); + } + + // "Maximum length, in bytes, for binary data, character data, + // or text and image data." + if let Some(b) = &mut self.character_octet_lengths { + let char_len: Option = match field.data_type() { + Utf8 | Binary => Some(i32::MAX as u64), + LargeBinary | LargeUtf8 => Some(i64::MAX as u64), + _ => None, + }; + b.append_option(char_len); + } + + // numeric_precision: "If data_type identifies a numeric type, this column + // contains the (declared or implicit) precision of the type + // for this column. The precision indicates the number of + // significant digits. It can be expressed in decimal (base + // 10) or binary (base 2) terms, as specified in the column + // numeric_precision_radix. For all other data types, this + // column is null." + // + // numeric_radix: If data_type identifies a numeric type, this + // column indicates in which base the values in the columns + // numeric_precision and numeric_scale are expressed. The + // value is either 2 or 10. For all other data types, this + // column is null. + // + // numeric_scale: If data_type identifies an exact numeric + // type, this column contains the (declared or implicit) scale + // of the type for this column. The scale indicates the number + // of significant digits to the right of the decimal point. It + // can be expressed in decimal (base 10) or binary (base 2) + // terms, as specified in the column + // numeric_precision_radix. For all other data types, this + // column is null. + if let (Some(precisions), Some(radixes), Some(scales)) = ( + &mut self.numeric_precisions, + &mut self.numeric_precision_radixes, + &mut self.numeric_scales, + ) { + let (numeric_precision, numeric_radix, numeric_scale) = + match field.data_type() { + Int8 | UInt8 => (Some(8), Some(2), None), + Int16 | UInt16 => (Some(16), Some(2), None), + Int32 | UInt32 => (Some(32), Some(2), None), + // From max value of 65504 as explained on + // https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Exponent_encoding + Float16 => (Some(15), Some(2), None), + // Numbers from postgres `real` type + Float32 => (Some(24), Some(2), None), + // Numbers from postgres `double` type + Float64 => (Some(24), Some(2), None), + Decimal128(precision, scale) => { + (Some(*precision as u64), Some(10), Some(*scale as u64)) + } + _ => (None, None, None), + }; + + precisions.append_option(numeric_precision); + radixes.append_option(numeric_radix); + scales.append_option(numeric_scale); + } + + if let Some(b) = &mut self.datetime_precisions { + b.append_option(None); + } + if let Some(b) = &mut self.interval_types { + b.append_null(); + } + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + let mut columns: Vec = Vec::new(); + if self.detail != ColumnsDetail::Describe { + columns.push(Arc::new(self.catalog_names.as_mut().unwrap().finish())); + columns.push(Arc::new(self.schema_names.as_mut().unwrap().finish())); + columns.push(Arc::new(self.table_names.as_mut().unwrap().finish())); + } + columns.push(Arc::new(self.column_names.finish())); + if self.detail == ColumnsDetail::Full { + columns.push(Arc::new(self.ordinal_positions.as_mut().unwrap().finish())); + columns.push(Arc::new(self.column_defaults.as_mut().unwrap().finish())); + } + if self.detail == ColumnsDetail::Full { + columns.push(Arc::new(self.is_nullables.finish())); + columns.push(Arc::new(self.data_types.finish())); + } else { + // `Describe`/`Basic` list `data_type` before `is_nullable`, + // unlike the SQL-standard `information_schema.columns` order + // used by `Full`. + columns.push(Arc::new(self.data_types.finish())); + columns.push(Arc::new(self.is_nullables.finish())); + } + if self.detail == ColumnsDetail::Full { + columns.push(Arc::new( + self.character_maximum_lengths.as_mut().unwrap().finish(), + )); + columns.push(Arc::new( + self.character_octet_lengths.as_mut().unwrap().finish(), + )); + columns.push(Arc::new(self.numeric_precisions.as_mut().unwrap().finish())); + columns.push(Arc::new( + self.numeric_precision_radixes.as_mut().unwrap().finish(), + )); + columns.push(Arc::new(self.numeric_scales.as_mut().unwrap().finish())); + columns.push(Arc::new( + self.datetime_precisions.as_mut().unwrap().finish(), + )); + columns.push(Arc::new(self.interval_types.as_mut().unwrap().finish())); + } + RecordBatch::try_new(Arc::clone(&self.schema), columns).unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.schemata` rows. +/// +/// Columns and nullability match +/// . +pub fn schemata_schema() -> Schema { + Schema::new(vec![ + Field::new("catalog_name", DataType::Utf8, false), + Field::new("schema_name", DataType::Utf8, false), + Field::new("schema_owner", DataType::Utf8, true), + Field::new("default_character_set_catalog", DataType::Utf8, true), + Field::new("default_character_set_schema", DataType::Utf8, true), + Field::new("default_character_set_name", DataType::Utf8, true), + Field::new("sql_path", DataType::Utf8, true), + ]) +} + +/// Builds `information_schema.schemata` rows. +#[derive(Debug)] +pub struct InformationSchemataBuilder { + schema: SchemaRef, + catalog_name: StringBuilder, + schema_name: StringBuilder, + schema_owner: StringBuilder, + default_character_set_catalog: StringBuilder, + default_character_set_schema: StringBuilder, + default_character_set_name: StringBuilder, + sql_path: StringBuilder, +} + +impl Default for InformationSchemataBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemataBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(schemata_schema()), + catalog_name: StringBuilder::new(), + schema_name: StringBuilder::new(), + schema_owner: StringBuilder::new(), + default_character_set_catalog: StringBuilder::new(), + default_character_set_schema: StringBuilder::new(), + default_character_set_name: StringBuilder::new(), + sql_path: StringBuilder::new(), + } + } + + /// Append one row. The `default_character_set_*` and `sql_path` columns are + /// always null. + pub fn add_schemata( + &mut self, + catalog_name: &str, + schema_name: &str, + schema_owner: Option<&str>, + ) { + self.catalog_name.append_value(catalog_name); + self.schema_name.append_value(schema_name); + match schema_owner { + Some(owner) => self.schema_owner.append_value(owner), + None => self.schema_owner.append_null(), + } + self.default_character_set_catalog.append_null(); + self.default_character_set_schema.append_null(); + self.default_character_set_name.append_null(); + self.sql_path.append_null(); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> Result { + Ok(RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.catalog_name.finish()), + Arc::new(self.schema_name.finish()), + Arc::new(self.schema_owner.finish()), + Arc::new(self.default_character_set_catalog.finish()), + Arc::new(self.default_character_set_schema.finish()), + Arc::new(self.default_character_set_name.finish()), + Arc::new(self.sql_path.finish()), + ], + )?) + } +} + +/// Returns the Arrow schema of `information_schema.df_settings` rows. +/// The `description` column is only included when `verbose` is set. +pub fn df_settings_schema(verbose: bool) -> Schema { + let mut fields = vec![ + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), + ]; + if verbose { + fields.push(Field::new("description", DataType::Utf8, true)); + } + Schema::new(fields) +} + +/// Builds `information_schema.df_settings` rows. +#[derive(Debug)] +pub struct InformationSchemaDfSettingsBuilder { + schema: SchemaRef, + names: StringBuilder, + values: StringBuilder, + descriptions: Option, +} + +impl InformationSchemaDfSettingsBuilder { + /// Construct an empty builder producing the columns of + /// [`df_settings_schema`]`(verbose)`. + pub fn new(verbose: bool) -> Self { + Self { + schema: Arc::new(df_settings_schema(verbose)), + names: StringBuilder::new(), + values: StringBuilder::new(), + descriptions: verbose.then(StringBuilder::new), + } + } + + /// Append one row. + pub fn add_setting(&mut self, entry: ConfigEntry) { + self.names.append_value(entry.key); + self.values.append_option(entry.value); + if let Some(descriptions) = &mut self.descriptions { + descriptions.append_value(entry.description); + } + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + let mut columns: Vec = vec![ + Arc::new(self.names.finish()), + Arc::new(self.values.finish()), + ]; + if let Some(descriptions) = &mut self.descriptions { + columns.push(Arc::new(descriptions.finish())); + } + RecordBatch::try_new(Arc::clone(&self.schema), columns).unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.routines` rows. +pub fn routines_schema() -> Schema { + Schema::new(vec![ + Field::new("specific_catalog", DataType::Utf8, false), + Field::new("specific_schema", DataType::Utf8, false), + Field::new("specific_name", DataType::Utf8, false), + Field::new("routine_catalog", DataType::Utf8, false), + Field::new("routine_schema", DataType::Utf8, false), + Field::new("routine_name", DataType::Utf8, false), + Field::new("routine_type", DataType::Utf8, false), + Field::new("is_deterministic", DataType::Boolean, true), + Field::new("data_type", DataType::Utf8, true), + Field::new("function_type", DataType::Utf8, true), + Field::new("description", DataType::Utf8, true), + Field::new("syntax_example", DataType::Utf8, true), + ]) +} + +/// Builds `information_schema.routines` rows. +#[derive(Debug)] +pub struct InformationSchemaRoutinesBuilder { + schema: SchemaRef, + specific_catalog: StringBuilder, + specific_schema: StringBuilder, + specific_name: StringBuilder, + routine_catalog: StringBuilder, + routine_schema: StringBuilder, + routine_name: StringBuilder, + routine_type: StringBuilder, + is_deterministic: BooleanBuilder, + data_type: StringBuilder, + function_type: StringBuilder, + description: StringBuilder, + syntax_example: StringBuilder, +} + +impl Default for InformationSchemaRoutinesBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaRoutinesBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(routines_schema()), + specific_catalog: StringBuilder::new(), + specific_schema: StringBuilder::new(), + specific_name: StringBuilder::new(), + routine_catalog: StringBuilder::new(), + routine_schema: StringBuilder::new(), + routine_name: StringBuilder::new(), + routine_type: StringBuilder::new(), + is_deterministic: BooleanBuilder::new(), + data_type: StringBuilder::new(), + function_type: StringBuilder::new(), + description: StringBuilder::new(), + syntax_example: StringBuilder::new(), + } + } + + /// Append one row. + #[expect(clippy::too_many_arguments)] + pub fn add_routine( + &mut self, + catalog_name: impl AsRef, + schema_name: impl AsRef, + routine_name: impl AsRef, + routine_type: impl AsRef, + is_deterministic: bool, + data_type: Option<&impl AsRef>, + function_type: impl AsRef, + description: Option>, + syntax_example: Option>, + ) { + self.specific_catalog.append_value(catalog_name.as_ref()); + self.specific_schema.append_value(schema_name.as_ref()); + self.specific_name.append_value(routine_name.as_ref()); + self.routine_catalog.append_value(catalog_name.as_ref()); + self.routine_schema.append_value(schema_name.as_ref()); + self.routine_name.append_value(routine_name.as_ref()); + self.routine_type.append_value(routine_type.as_ref()); + self.is_deterministic.append_value(is_deterministic); + self.data_type.append_option(data_type.as_ref()); + self.function_type.append_value(function_type.as_ref()); + self.description.append_option(description); + self.syntax_example.append_option(syntax_example); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.specific_catalog.finish()), + Arc::new(self.specific_schema.finish()), + Arc::new(self.specific_name.finish()), + Arc::new(self.routine_catalog.finish()), + Arc::new(self.routine_schema.finish()), + Arc::new(self.routine_name.finish()), + Arc::new(self.routine_type.finish()), + Arc::new(self.is_deterministic.finish()), + Arc::new(self.data_type.finish()), + Arc::new(self.function_type.finish()), + Arc::new(self.description.finish()), + Arc::new(self.syntax_example.finish()), + ], + ) + .unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.parameters` rows. +pub fn parameters_schema() -> Schema { + Schema::new(vec![ + Field::new("specific_catalog", DataType::Utf8, false), + Field::new("specific_schema", DataType::Utf8, false), + Field::new("specific_name", DataType::Utf8, false), + Field::new("ordinal_position", DataType::UInt64, false), + Field::new("parameter_mode", DataType::Utf8, false), + Field::new("parameter_name", DataType::Utf8, true), + Field::new("data_type", DataType::Utf8, false), + Field::new("parameter_default", DataType::Utf8, true), + Field::new("is_variadic", DataType::Boolean, false), + // `rid` (short for `routine id`) is used to differentiate parameters from different signatures + // (It serves as the group-by key when generating the `SHOW FUNCTIONS` query). + // For example, the following signatures have different `rid` values: + // - `datetrunc(Utf8, Timestamp(Microsecond, Some("+TZ"))) -> Timestamp(Microsecond, Some("+TZ"))` + // - `datetrunc(Utf8View, Timestamp(Nanosecond, None)) -> Timestamp(Nanosecond, None)` + Field::new("rid", DataType::UInt8, false), + ]) +} + +/// Builds `information_schema.parameters` rows. +#[derive(Debug)] +pub struct InformationSchemaParametersBuilder { + schema: SchemaRef, + specific_catalog: StringBuilder, + specific_schema: StringBuilder, + specific_name: StringBuilder, + ordinal_position: UInt64Builder, + parameter_mode: StringBuilder, + parameter_name: StringBuilder, + data_type: StringBuilder, + parameter_default: StringBuilder, + is_variadic: BooleanBuilder, + rid: UInt8Builder, +} + +impl Default for InformationSchemaParametersBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaParametersBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(parameters_schema()), + specific_catalog: StringBuilder::new(), + specific_schema: StringBuilder::new(), + specific_name: StringBuilder::new(), + ordinal_position: UInt64Builder::new(), + parameter_mode: StringBuilder::new(), + parameter_name: StringBuilder::new(), + data_type: StringBuilder::new(), + parameter_default: StringBuilder::new(), + is_variadic: BooleanBuilder::new(), + rid: UInt8Builder::new(), + } + } + + /// Append one row. + #[expect(clippy::too_many_arguments)] + pub fn add_parameter( + &mut self, + specific_catalog: impl AsRef, + specific_schema: impl AsRef, + specific_name: impl AsRef, + ordinal_position: u64, + parameter_mode: impl AsRef, + parameter_name: Option<&(impl AsRef + ?Sized)>, + data_type: impl AsRef, + parameter_default: Option>, + is_variadic: bool, + rid: u8, + ) { + self.specific_catalog + .append_value(specific_catalog.as_ref()); + self.specific_schema.append_value(specific_schema.as_ref()); + self.specific_name.append_value(specific_name.as_ref()); + self.ordinal_position.append_value(ordinal_position); + self.parameter_mode.append_value(parameter_mode.as_ref()); + self.parameter_name.append_option(parameter_name.as_ref()); + self.data_type.append_value(data_type.as_ref()); + self.parameter_default.append_option(parameter_default); + self.is_variadic.append_value(is_variadic); + self.rid.append_value(rid); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.specific_catalog.finish()), + Arc::new(self.specific_schema.finish()), + Arc::new(self.specific_name.finish()), + Arc::new(self.ordinal_position.finish()), + Arc::new(self.parameter_mode.finish()), + Arc::new(self.parameter_name.finish()), + Arc::new(self.data_type.finish()), + Arc::new(self.parameter_default.finish()), + Arc::new(self.is_variadic.finish()), + Arc::new(self.rid.finish()), + ], + ) + .unwrap() + } +} + +/// Returns the Arrow schema of `SHOW FUNCTIONS` output, with one row per function signature: +/// +/// - `function_name` +/// - `return_type` +/// - `parameters`: parameter names, in order +/// - `parameter_types`: parameter types, in order +/// - `function_type`: `SCALAR`, `AGGREGATE`, `WINDOW` or `TABLE` +/// - `description` and `syntax_example`: from the function's documentation +/// +/// Unlike the other schemas in this module, this does not correspond to an +/// `information_schema` table. +pub fn show_functions_schema() -> Schema { + Schema::new(vec![ + Field::new("function_name", DataType::Utf8, false), + Field::new("return_type", DataType::Utf8, true), + Field::new("parameters", DataType::new_list(DataType::Utf8, true), true), + Field::new( + "parameter_types", + DataType::new_list(DataType::Utf8, true), + true, + ), + Field::new("function_type", DataType::Utf8, true), + Field::new("description", DataType::Utf8, true), + Field::new("syntax_example", DataType::Utf8, true), + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Array, StringArray}; + + #[test] + fn schemata_builder_emits_canonical_schema_and_rows() { + // Construct via `Default` so the test exercises both `new()` (via + // the `Default` impl) and the public column-layout contract. + let mut builder = InformationSchemataBuilder::default(); + builder.add_schemata("cat", "schema_one", Some("alice")); + builder.add_schemata("cat", "schema_two", None); + let batch = builder.finish().expect("finish should not fail"); + + assert_eq!(batch.schema().as_ref(), &schemata_schema()); + assert_eq!(batch.num_rows(), 2); + + let col = |name: &str| { + batch + .column_by_name(name) + .unwrap_or_else(|| panic!("missing column {name}")) + }; + let string_col = |name: &str| { + col(name) + .as_any() + .downcast_ref::() + .unwrap_or_else(|| panic!("{name} should be a StringArray")) + }; + + let catalog = string_col("catalog_name"); + assert_eq!(catalog.value(0), "cat"); + assert_eq!(catalog.value(1), "cat"); + + let schema = string_col("schema_name"); + assert_eq!(schema.value(0), "schema_one"); + assert_eq!(schema.value(1), "schema_two"); + + let owner = string_col("schema_owner"); + assert_eq!(owner.value(0), "alice"); + assert!(owner.is_null(1)); + + // The three character-set columns and sql_path are unconditionally + // null — they exist for SQL-standard column-layout compatibility. + for name in [ + "default_character_set_catalog", + "default_character_set_schema", + "default_character_set_name", + "sql_path", + ] { + let c = string_col(name); + assert!(c.is_null(0), "{name} row 0 should be null"); + assert!(c.is_null(1), "{name} row 1 should be null"); + } + } + + #[test] + fn tables_builder_renders_table_type_as_sql_string() { + let mut builder = InformationSchemaTablesBuilder::new(); + builder.add_table("cat", "schema", "t", TableType::Base); + let batch = builder.finish(); + + assert_eq!(batch.schema().as_ref(), &tables_schema()); + let table_types = batch + .column_by_name("table_type") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(table_types.value(0), "BASE TABLE"); + } + + #[test] + fn columns_builder_describe_retains_only_describe_columns() { + let field = Field::new("c", DataType::Int32, true); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Describe); + builder.add_column("cat", "schema", "t", 0, &field); + let batch = builder.finish(); + + assert_eq!( + batch.schema().as_ref(), + &columns_schema(ColumnsDetail::Describe) + ); + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(), + vec!["column_name", "data_type", "is_nullable"] + ); + assert_eq!( + batch + .column_by_name("column_name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "c" + ); + } + + #[test] + fn columns_builder_basic_adds_table_qualifiers_but_not_full_metadata() { + let field = Field::new("c", DataType::Int32, true); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Basic); + builder.add_column("cat", "schema", "t", 0, &field); + let batch = builder.finish(); + + assert_eq!( + batch.schema().as_ref(), + &columns_schema(ColumnsDetail::Basic) + ); + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(), + vec![ + "table_catalog", + "table_schema", + "table_name", + "column_name", + "data_type", + "is_nullable", + ] + ); + assert_eq!( + batch + .column_by_name("table_catalog") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "cat" + ); + } + + #[test] + fn columns_builder_full_retains_every_column_in_standard_order() { + let field = Field::new("c", DataType::Decimal128(10, 2), false); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Full); + builder.add_column("cat", "schema", "t", 3, &field); + let batch = builder.finish(); + + assert_eq!( + batch.schema().as_ref(), + &columns_schema(ColumnsDetail::Full) + ); + assert_eq!(batch.num_columns(), 15); + // `Full` keeps the SQL-standard order (`is_nullable` before + // `data_type`), unlike `Describe`/`Basic`. + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(), + vec![ + "table_catalog", + "table_schema", + "table_name", + "column_name", + "ordinal_position", + "column_default", + "is_nullable", + "data_type", + "character_maximum_length", + "character_octet_length", + "numeric_precision", + "numeric_precision_radix", + "numeric_scale", + "datetime_precision", + "interval_type", + ] + ); + assert_eq!( + batch + .column_by_name("ordinal_position") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 3 + ); + assert_eq!( + batch + .column_by_name("numeric_precision") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 10 + ); + } +} diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 2eebfe4963057..d07cb85b19cce 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -33,6 +33,7 @@ mod param_value; mod partitioning; mod schema_reference; mod table_reference; +mod table_type; mod unnest; pub mod alias; @@ -49,6 +50,7 @@ pub mod file_options; pub mod format; pub mod hash_utils; pub mod heap_size; +pub mod information_schema; pub mod instant; pub mod metadata; pub mod nested_struct; @@ -99,6 +101,7 @@ pub use schema_reference::SchemaReference; pub use spans::{Location, Span, Spans}; pub use stats::{ColumnStatistics, Statistics}; pub use table_reference::{ResolvedTableReference, TableReference}; +pub use table_type::TableType; pub use unnest::{NullHandling, RecursionUnnestOption, UnnestOptions}; pub use utils::project_schema; diff --git a/datafusion/common/src/table_type.rs b/datafusion/common/src/table_type.rs new file mode 100644 index 0000000000000..5adad946f7fc2 --- /dev/null +++ b/datafusion/common/src/table_type.rs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// The type of a table for metadata/catalog purposes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TableType { + /// An ordinary physical table. + Base, + /// A non-materialized table that itself uses a query internally to provide data. + View, + /// A transient table. + Temporary, +} + +impl std::fmt::Display for TableType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TableType::Base => write!(f, "Base"), + TableType::View => write!(f, "View"), + TableType::Temporary => write!(f, "Temporary"), + } + } +} diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 7d87e2043d678..4240935747532 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -36,8 +36,6 @@ use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; use arrow_schema::DataType; use arrow_schema::FieldRef; use datafusion_catalog::MemoryCatalogProviderList; -#[cfg(feature = "sql")] -use datafusion_catalog::information_schema::INFORMATION_SCHEMA_TABLES; use datafusion_catalog::information_schema::{ INFORMATION_SCHEMA, InformationSchemaProvider, }; @@ -85,7 +83,7 @@ use datafusion_session::{ }; #[cfg(feature = "sql")] use datafusion_sql::{ - parser::{DFParserBuilder, ExplainStatement, Statement}, + parser::{DFParserBuilder, Statement}, planner::{ContextProvider, ParserOptions, PlannerContext, SqlToRel}, }; @@ -97,9 +95,7 @@ use log::{debug, info}; use object_store::ObjectStore; #[cfg(feature = "sql")] use sqlparser::{ - ast::{ - Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias, Statement as SQLStatement, - }, + ast::{Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias}, dialect::dialect_from_str, }; use url::Url; @@ -429,45 +425,26 @@ impl SessionState { if *resolved_ref.schema == *INFORMATION_SCHEMA && self.inner.config.information_schema() { - match self.inner.config.system_catalog() { - Some(system_catalog) => { - if system_catalog == catalog_name { - // Global catalog configured, catalog name references it - // => information schema provider for full catalog list - return Ok(Arc::new( - InformationSchemaProvider::new(Arc::clone( - &self.inner.catalog_list, - )) - .with_table_functions(self.inner.table_functions.clone()) - .with_system_catalog(String::from(system_catalog)), - )); - } else { - // Global catalog configured, catalog name is other catalog - // => information schema provider for named catalog - let catalog = self.resolve_catalog(catalog_name)?; - return Ok(Arc::new( - InformationSchemaProvider::new_from_catalog( - catalog_name.to_string(), - catalog, - ) - .with_table_functions(self.inner.table_functions.clone()), - )); - } - } - None => { - // Global catalog not configured - // => information schema provider for full catalog list - // Resolve the catalog (even though it's not needed for the InformationSchemaProvider) - // so references to non-existent catalogs report an error. - self.resolve_catalog(catalog_name)?; - return Ok(Arc::new( - InformationSchemaProvider::new(Arc::clone( - &self.inner.catalog_list, - )) - .with_table_functions(self.inner.table_functions.clone()), - )); - } - } + // Resolve catalog to ensure it exists + let _ = self.resolve_catalog(catalog_name)?; + + // Create the global information schema + let information_schema_provider = + InformationSchemaProvider::new(Arc::clone(self.catalog_list())) + .with_information_schema_enabled(true) + .with_table_functions(self.inner.table_functions.clone()); + + // If partitioned information schema is enabled, restrict the information schema to + // the containing catalog + let information_schema_provider = + if self.config().partition_information_schema() { + information_schema_provider + .with_catalog_filter(Some(catalog_name.to_string())) + } else { + information_schema_provider + }; + + return Ok(Arc::new(information_schema_provider)); } let catalog = self.resolve_catalog(catalog_name)?; @@ -673,23 +650,7 @@ impl SessionState { &self, statement: Statement, ) -> datafusion_common::Result { - let mut references = self.resolve_table_references(&statement)?; - - if Self::requires_information_schema(&statement) { - let info_tables = INFORMATION_SCHEMA_TABLES.iter(); - let info_table_refs: Vec<_> = if let Some(system_catalog) = - self.config().system_catalog() - { - info_tables - .map(|n| TableReference::full(system_catalog, INFORMATION_SCHEMA, *n)) - .collect() - } else { - info_tables - .map(|n| TableReference::partial(INFORMATION_SCHEMA, *n)) - .collect() - }; - references.extend(info_table_refs); - } + let references = self.resolve_table_references(&statement)?; let mut provider = SessionContextProvider { state: self, @@ -712,24 +673,6 @@ impl SessionState { query.statement_to_plan(statement) } - #[cfg(feature = "sql")] - fn requires_information_schema(statement: &Statement) -> bool { - match &statement { - Statement::Explain(ExplainStatement { statement, .. }) => { - Self::requires_information_schema(statement) - } - Statement::Statement(s) => matches!( - s.as_ref(), - SQLStatement::ShowCreate { .. } - | SQLStatement::ShowTables { .. } - | SQLStatement::ShowColumns { .. } - | SQLStatement::ShowVariable { .. } - | SQLStatement::ShowFunctions { .. } - ), - _ => false, - } - } - #[cfg(feature = "sql")] fn get_parser_options(&self) -> ParserOptions { let sql_parser_options = &self.config().options().sql_parser; diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index d1e5054cc80da..0b3416e092120 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -27,6 +27,7 @@ use crate::datasource::physical_plan::{FileOutputMode, FileSinkConfig}; use crate::datasource::{DefaultTableSource, source_as_provider}; use crate::error::{DataFusionError, Result}; use crate::execution::context::ExecutionProps; +use crate::execution::session_state::SessionState; use crate::logical_expr::utils::generate_sort_key; use crate::logical_expr::{ Aggregate, EmptyRelation, Join, Projection, Sort, TableScan, Unnest, Values, Window, @@ -60,11 +61,12 @@ use crate::physical_plan::{ }; use crate::schema_equivalence::schema_satisfied_by; -use arrow::array::{ArrayRef, RecordBatch, UInt64Array, builder::StringBuilder}; +use arrow::array::{ArrayRef, RecordBatch, UInt64Array}; use arrow::compute::SortOptions; -use arrow::datatypes::Schema; +use arrow::datatypes::{DataType, Schema}; use arrow_schema::Field; use datafusion_catalog::ScanArgs; +use datafusion_catalog::information_schema::InformationSchemaProvider; use datafusion_common::Column; use datafusion_common::HashMap as DFHashMap; use datafusion_common::config::ConfigOptions; @@ -93,10 +95,10 @@ use datafusion_expr::physical_planning_context::{ }; use datafusion_expr::utils::{expr_to_columns, split_conjunction}; use datafusion_expr::{ - Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension, - FetchType, Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCtePlan, - MaterializedCteScan, Operator, RecursiveQuery, SkipType, StringifiedPlan, - WindowFrame, WindowFrameBound, WriteOp, + Analyze, BinaryExpr, DmlStatement, Explain, ExplainFormat, Extension, FetchType, + Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCtePlan, + MaterializedCteScan, Operator, RecursiveQuery, Show, ShowKind, SkipType, + StringifiedPlan, WindowFrame, WindowFrameBound, WriteOp, }; use datafusion_physical_expr::aggregate::{ AggregateFunctionExpr, LoweredAggregate, LoweredAggregateBuilder, @@ -725,11 +727,11 @@ impl DefaultPhysicalPlanner { async fn map_logical_node_to_physical( &self, node: &LogicalPlan, - session_state: &dyn Session, + session: &dyn Session, planning_ctx: &PhysicalPlanningContext, children: ChildrenContainer, ) -> Result> { - let execution_props = session_state.execution_props(); + let execution_props = session.execution_props(); let exec_node: Arc = match node { // Leaves (no children) LogicalPlan::TableScan(scan) => { @@ -758,7 +760,7 @@ impl DefaultPhysicalPlanner { .with_limit(*fetch) .with_skip(*offset) .with_statistics_requests(&stats_requests); - let res = source.scan_with_args(session_state, opts).await?; + let res = source.scan_with_args(session, opts).await?; Arc::clone(res.plan()) } else { let mut maybe_plan = None; @@ -768,7 +770,7 @@ impl DefaultPhysicalPlanner { } maybe_plan = planner - .plan_table_scan(self, scan, session_state, planning_ctx) + .plan_table_scan(self, scan, session, planning_ctx) .await?; } @@ -813,12 +815,71 @@ impl DefaultPhysicalPlanner { produce_one_row: true, schema, }) => Arc::new(PlaceholderRowExec::new(Arc::clone(schema.inner()))), - LogicalPlan::DescribeTable(DescribeTable { - schema, - output_schema, + LogicalPlan::Show(Show { + kind: ShowKind::Tables, + .. + }) => create_information_schema_provider(session).show_tables()?, + LogicalPlan::Show(Show { + kind: ShowKind::Describe { schema }, + .. + }) => create_information_schema_provider(session).describe_schema(schema)?, + LogicalPlan::Show(Show { + kind: ShowKind::Columns { table_ref, detail }, + .. + }) => { + let catalog_options = &session.config_options().catalog; + let resolved = table_ref.clone().resolve( + &catalog_options.default_catalog, + &catalog_options.default_schema, + ); + create_information_schema_provider(session) + .show_columns(&resolved, *detail) + .await? + } + LogicalPlan::Show(Show { + kind: ShowKind::CreateTable { table_ref }, + .. + }) => { + let catalog_options = &session.config_options().catalog; + let resolved = table_ref.clone().resolve( + &catalog_options.default_catalog, + &catalog_options.default_schema, + ); + + create_information_schema_provider(session) + .show_create_table(&resolved) + .await? + } + LogicalPlan::Show(Show { + kind: ShowKind::Variables { name, verbose }, + .. + }) => create_information_schema_provider(session).show_variables( + session.config_options(), + session.runtime_env(), + name.as_deref(), + *verbose, + )?, + LogicalPlan::Show(Show { + kind: ShowKind::Functions { filter }, + .. }) => { - let output_schema = Arc::clone(output_schema.inner()); - self.plan_describe(&Arc::clone(schema), output_schema)? + // Table functions (UDTFs) are only reachable through the + // concrete `SessionState` (the `Session` trait has no + // accessor for them); a custom `Session` implementor will + // simply see no `TABLE`-typed routines in `SHOW FUNCTIONS`. + let table_functions = session + .as_any() + .downcast_ref::() + .map(|state| state.table_functions().clone()) + .unwrap_or_default(); + create_information_schema_provider(session) + .with_table_functions(table_functions) + .show_functions( + session.scalar_functions(), + session.aggregate_functions(), + session.window_functions(), + filter.as_deref(), + )? } LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => { @@ -827,7 +888,7 @@ impl DefaultPhysicalPlanner { scan.id.as_u64(), &scan.name, Arc::clone(scan.schema.inner()), - session_state.config().target_partitions(), + session.config().target_partitions(), )) } @@ -852,7 +913,7 @@ impl DefaultPhysicalPlanner { // the column name rather than column name + explicit data type. let table_partition_cols = partition_by .iter() - .map(|s| (s.to_string(), arrow::datatypes::DataType::Null)) + .map(|s| (s.to_string(), DataType::Null)) .collect::>(); let keep_partition_by_columns = match source_option_tuples @@ -860,7 +921,7 @@ impl DefaultPhysicalPlanner { .map(|v| v.trim()) { None => { - session_state + session .config() .options() .execution @@ -897,8 +958,8 @@ impl DefaultPhysicalPlanner { .map(|(k, v)| (k.clone(), v.clone())) .collect(); - let sink_format = file_type_to_format(file_type)? - .create(session_state, &format_options)?; + let sink_format = + file_type_to_format(file_type)?.create(session, &format_options)?; // Determine extension based on format extension and compression let file_extension = match sink_format.compression_type() { @@ -927,7 +988,7 @@ impl DefaultPhysicalPlanner { sink_format .create_writer_physical_plan( input_exec, - session_state, + session, config, ordering.map(Into::into), ) @@ -942,7 +1003,7 @@ impl DefaultPhysicalPlanner { let input_exec = children.one()?; provider .table_provider - .insert_into(session_state, input_exec, *insert_op) + .insert_into(session, input_exec, *insert_op) .await? } else { return exec_err!( @@ -969,7 +1030,7 @@ impl DefaultPhysicalPlanner { extract_dml_filters(input, table_name, &allowed_refs)?; provider .table_provider - .delete_from(session_state, filters) + .delete_from(session, filters) .await .map_err(|e| { e.context(format!( @@ -1007,7 +1068,7 @@ impl DefaultPhysicalPlanner { let assignments = extract_update_assignments(input)?; provider .table_provider - .update(session_state, assignments, filters) + .update(session, assignments, filters) .await .map_err(|e| { e.context(format!( @@ -1031,7 +1092,7 @@ impl DefaultPhysicalPlanner { if let Some(provider) = target.downcast_ref::() { provider .table_provider - .truncate(session_state) + .truncate(session) .await .map_err(|e| { e.context(format!( @@ -1060,7 +1121,7 @@ impl DefaultPhysicalPlanner { ); provider .merge_into( - session_state, + session, input_exec, merge_schema, merge_op.on.clone(), @@ -1127,8 +1188,8 @@ impl DefaultPhysicalPlanner { }) .collect::>>()?; - let can_repartition = session_state.config().target_partitions() > 1 - && session_state.config().repartition_window_functions(); + let can_repartition = session.config().target_partitions() > 1 + && session.config().repartition_window_functions(); let uses_bounded_memory = window_expr.iter().all(|e| e.uses_bounded_memory()); @@ -1151,7 +1212,7 @@ impl DefaultPhysicalPlanner { } LogicalPlan::Aggregate(aggregate) => self.plan_aggregate( aggregate, - session_state, + session, planning_ctx, execution_props, children, @@ -1188,7 +1249,7 @@ impl DefaultPhysicalPlanner { Arc::clone(&runtime_expr[0]), physical_input, ) - .with_batch_size(session_state.config().batch_size()) + .with_batch_size(session.config().batch_size()) .build()? } PlanAsyncExpr::Async( @@ -1208,7 +1269,7 @@ impl DefaultPhysicalPlanner { .apply_projection(Some( (0..input.schema().fields().len()).collect::>(), ))? - .with_batch_size(session_state.config().batch_size()) + .with_batch_size(session.config().batch_size()) .build()? } _ => { @@ -1218,7 +1279,7 @@ impl DefaultPhysicalPlanner { } }; - let selectivity = session_state + let selectivity = session .config() .options() .optimizer @@ -1600,12 +1661,12 @@ impl DefaultPhysicalPlanner { }; let prefer_hash_join = - session_state.config_options().optimizer.prefer_hash_join; + session.config_options().optimizer.prefer_hash_join; // Null-aware joins are pinned to CollectLeft hash joins (see // `HashJoinExec::null_aware`): never repartition them, and // never route them to the sort-merge path below. - let can_repartition_join = session_state.config().target_partitions() > 1 - && session_state.config().repartition_joins() + let can_repartition_join = session.config().target_partitions() > 1 + && session.config().repartition_joins() && !*null_aware; // Only `HashJoinExec` implements null-aware semantics, and it @@ -1626,7 +1687,7 @@ impl DefaultPhysicalPlanner { Arc::new(CrossJoinExec::new(physical_left, physical_right)) } else if num_range_filters == 1 && total_filters == 1 - && session_state + && session .config_options() .optimizer .enable_piecewise_merge_join @@ -1731,7 +1792,7 @@ impl DefaultPhysicalPlanner { (on_left, on_right), op, *join_type, - session_state.config().target_partitions(), + session.config().target_partitions(), )?) } else { // there is no equal join condition, use the nested loop join @@ -1743,8 +1804,8 @@ impl DefaultPhysicalPlanner { None, )?) } - } else if session_state.config().target_partitions() > 1 - && session_state.config().repartition_joins() + } else if session.config().target_partitions() > 1 + && session.config().repartition_joins() && !prefer_hash_join && !*null_aware // Null-aware joins (e.g. `NOT IN` with a nullable subquery) must @@ -1900,7 +1961,7 @@ impl DefaultPhysicalPlanner { node.as_ref(), &logical_input, &children, - session_state, + session, planning_ctx, ) .await?; @@ -2228,6 +2289,15 @@ impl DefaultPhysicalPlanner { } } +fn create_information_schema_provider( + session: &dyn Session, +) -> InformationSchemaProvider { + // Do not set a catalog filter. `SHOW` statements describe all catalogs, + // regardless of `SessionConfig::partition_information_schema` + let provider = InformationSchemaProvider::new(session.catalog_list()); + provider.with_information_schema_enabled(session.config().information_schema()) +} + /// Expand and align a GROUPING SET expression. /// (see ) /// @@ -2680,7 +2750,7 @@ fn extract_dml_filters( // Plans without filter information LogicalPlan::EmptyRelation(_) | LogicalPlan::Values(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Explain(_) | LogicalPlan::Analyze(_) | LogicalPlan::Distinct(_) @@ -3367,43 +3437,6 @@ impl DefaultPhysicalPlanner { Ok(new_plan) } - // return an record_batch which describes a table's schema. - fn plan_describe( - &self, - table_schema: &Arc, - output_schema: Arc, - ) -> Result> { - let mut column_names = StringBuilder::new(); - let mut data_types = StringBuilder::new(); - let mut is_nullables = StringBuilder::new(); - for field in table_schema.fields() { - column_names.append_value(field.name()); - - // "System supplied type" --> Use debug format of the datatype - let data_type = field.data_type(); - data_types.append_value(format!("{data_type}")); - - // "YES if the column is possibly nullable, NO if it is known not nullable. " - let nullable_str = if field.is_nullable() { "YES" } else { "NO" }; - is_nullables.append_value(nullable_str); - } - - let record_batch = RecordBatch::try_new( - output_schema, - vec![ - Arc::new(column_names.finish()), - Arc::new(data_types.finish()), - Arc::new(is_nullables.finish()), - ], - )?; - - let schema = record_batch.schema(); - let partitions = vec![vec![record_batch]]; - let projection = None; - let mem_exec = MemorySourceConfig::try_new_exec(&partitions, schema, projection)?; - Ok(mem_exec) - } - /// Build physical plans for scalar subqueries and assign each an ordinal /// `SubqueryIndex`. Returns the links (plan + index) and a map from logical /// `Subquery` to its index. diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rs index a3de48e42ec3e..ae705b78b8f93 100644 --- a/datafusion/execution/src/config.rs +++ b/datafusion/execution/src/config.rs @@ -227,13 +227,6 @@ impl SessionConfig { self.options.execution.target_partitions } - /// Get [`system_catalog`] - /// - /// [`system_catalog`]: datafusion_common::config::CatalogOptions::system_catalog - pub fn system_catalog(&self) -> Option<&str> { - self.options.catalog.system_catalog.as_deref() - } - /// Get [`information_schema`] /// /// [`information_schema`]: datafusion_common::config::CatalogOptions::information_schema @@ -241,6 +234,13 @@ impl SessionConfig { self.options.catalog.information_schema } + /// Get [`partition_information_schema`] + /// + /// [`partition_information_schema`]: datafusion_common::config::CatalogOptions::partition_information_schema + pub fn partition_information_schema(&self) -> bool { + self.options.catalog.partition_information_schema + } + /// Get [`create_default_catalog_and_schema`] /// /// [`create_default_catalog_and_schema`]: datafusion_common::config::CatalogOptions::create_default_catalog_and_schema @@ -319,11 +319,11 @@ impl SessionConfig { self } - /// Set [`system_catalog`] + /// Set [`partition_information_schema`] /// - /// [`system_catalog`]: datafusion_common::config::CatalogOptions::system_catalog - pub fn with_system_catalog(mut self, catalog_name: Option) -> Self { - self.options_mut().catalog.system_catalog = catalog_name; + /// [`partition_information_schema`]: datafusion_common::config::CatalogOptions::partition_information_schema + pub fn with_partition_information_schema(mut self, partition: bool) -> Self { + self.options_mut().catalog.partition_information_schema = partition; self } diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index fe97ab8752835..84767f201b6a6 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -33,8 +33,8 @@ use crate::expr_rewriter::{ use crate::logical_plan::{ Aggregate, Analyze, AsOfJoin, AsOfMatch, Distinct, DistinctOn, EmptyRelation, Explain, Filter, Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, - PlanType, Prepare, Projection, Repartition, Sort, SubqueryAlias, TableScanBuilder, - Union, Unnest, Values, Window, + PlanType, Prepare, Projection, Repartition, Show, ShowKind, Sort, SubqueryAlias, + TableScanBuilder, Union, Unnest, Values, Window, }; use crate::select_expr::SelectExpr; use crate::utils::{ @@ -54,6 +54,7 @@ use arrow::compute::can_cast_types; use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use datafusion_common::display::ToStringifiedPlan; use datafusion_common::file_options::file_type::FileType; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::metadata::FieldMetadata; use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, FunctionalDependencies, NullEquality, @@ -264,6 +265,68 @@ impl LogicalPlanBuilder { })) } + /// Create a `DESCRIBE ` plan, where `schema` is the output schema of the query. + pub fn describe(schema: Arc) -> Result { + Self::show( + ShowKind::Describe { schema }, + datafusion_common::information_schema::columns_schema( + ColumnsDetail::Describe, + ), + ) + } + + /// Create a `SHOW TABLES` plan. + pub fn show_tables() -> Result { + Self::show( + ShowKind::Tables, + datafusion_common::information_schema::tables_schema(), + ) + } + + /// Create a `DESCRIBE
` or `SHOW [FULL|EXTENDED] COLUMNS FROM
` plan. + pub fn show_columns( + table_ref: TableReference, + detail: ColumnsDetail, + ) -> Result { + Self::show( + ShowKind::Columns { table_ref, detail }, + datafusion_common::information_schema::columns_schema(detail), + ) + } + + /// Create a `SHOW CREATE TABLE
` plan. + pub fn show_create_table(table_ref: TableReference) -> Result { + Self::show( + ShowKind::CreateTable { table_ref }, + datafusion_common::information_schema::views_schema(), + ) + } + + /// Create a `SHOW ` / `SHOW ALL` plan. `name` is `None` for + /// `SHOW ALL`. + pub fn show_variables(name: Option, verbose: bool) -> Result { + Self::show( + ShowKind::Variables { name, verbose }, + datafusion_common::information_schema::df_settings_schema(verbose), + ) + } + + /// Create a `SHOW FUNCTIONS [LIKE ]` plan. + pub fn show_functions(filter: Option) -> Result { + Self::show( + ShowKind::Functions { filter }, + datafusion_common::information_schema::show_functions_schema(), + ) + } + + /// Shared helper for the `SHOW`-class factory methods above. + fn show(kind: ShowKind, output_schema: Schema) -> Result { + Ok(Self::new(LogicalPlan::Show(Show { + kind, + output_schema: Arc::new(DFSchema::try_from(output_schema)?), + }))) + } + /// Convert a regular plan into a recursive query. /// `is_distinct` indicates whether the recursive term should be de-duplicated (`UNION`) after each iteration or not (`UNION ALL`). pub fn to_recursive_query( diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rs index 80ab6098920a5..0c7178f50c9b4 100644 --- a/datafusion/expr/src/logical_plan/display.rs +++ b/datafusion/expr/src/logical_plan/display.rs @@ -21,16 +21,17 @@ use std::collections::HashMap; use std::fmt; use crate::{ - Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter, - Join, Limit, LogicalPlan, MaterializedCte, MaterializedCtePlan, MaterializedCteScan, - Partitioning, Projection, RecursiveQuery, Repartition, Sort, Subquery, SubqueryAlias, - TableProviderFilterPushDown, TableScan, Unnest, Values, Window, WriteOp, - expr_vec_fmt, + Aggregate, AsOfJoin, Distinct, DistinctOn, DmlStatement, Expr, Filter, Join, Limit, + LogicalPlan, MaterializedCte, MaterializedCtePlan, MaterializedCteScan, Partitioning, + Projection, RecursiveQuery, Repartition, Show, ShowKind, Sort, Subquery, + SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest, Values, Window, + WriteOp, expr_vec_fmt, }; use crate::dml::CopyTo; use arrow::datatypes::Schema; use datafusion_common::display::GraphvizBuilder; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::tree_node::{TreeNodeRecursion, TreeNodeVisitor}; use datafusion_common::{Column, DataFusionError, internal_datafusion_err}; use serde::Serialize; @@ -725,11 +726,59 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> { "Name": name, ) } - LogicalPlan::DescribeTable(DescribeTable { .. }) => { - pg_fields!( - "Node Type": "DescribeTable" - ) - } + LogicalPlan::Show(Show { kind, .. }) => match kind { + ShowKind::Describe { .. } => { + pg_fields!( + "Node Type": "Describe" + ) + } + ShowKind::Tables => { + pg_fields!( + "Node Type": "ShowTables" + ) + } + ShowKind::Columns { + table_ref, + detail: ColumnsDetail::Describe, + } => { + pg_fields!( + "Node Type": "DescribeTable", + "Relation Name": table_ref.table() + ) + } + ShowKind::Columns { table_ref, detail } => { + pg_fields!( + "Node Type": "ShowColumns", + "Relation Name": table_ref.table(), + "Detail": format!("{detail:?}") + ) + } + ShowKind::CreateTable { table_ref } => { + pg_fields!( + "Node Type": "ShowCreateTable", + "Relation Name": table_ref.table() + ) + } + ShowKind::Variables { name, verbose } => { + let mut object = pg_fields!( + "Node Type": "ShowVariables", + "Verbose": verbose + ); + if let Some(name) = name { + object.push("Variable", serde_json::Value::String(name.clone())); + } + object + } + ShowKind::Functions { filter } => { + let mut object = pg_fields!( + "Node Type": "ShowFunctions" + ); + if let Some(filter) = filter { + object.push("Filter", serde_json::Value::String(filter.clone())); + } + object + } + }, LogicalPlan::Unnest(Unnest { input: plan, list_type_columns: list_col_indices, diff --git a/datafusion/expr/src/logical_plan/mod.rs b/datafusion/expr/src/logical_plan/mod.rs index 794747c733f21..6ceed2764eb90 100644 --- a/datafusion/expr/src/logical_plan/mod.rs +++ b/datafusion/expr/src/logical_plan/mod.rs @@ -43,10 +43,10 @@ pub use dml::{ WriteOp, }; pub use plan::{ - Aggregate, Analyze, AsOfJoin, AsOfMatch, ColumnUnnestList, DescribeTable, Distinct, - DistinctOn, EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter, - Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, - Projection, RangePartitioning, RecursiveQuery, Repartition, SkipType, Sort, + Aggregate, Analyze, AsOfJoin, AsOfMatch, ColumnUnnestList, Distinct, DistinctOn, + EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter, Join, + JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, Projection, + RangePartitioning, RecursiveQuery, Repartition, Show, ShowKind, SkipType, Sort, StringifiedPlan, Subquery, SubqueryAlias, TableScan, TableScanBuilder, ToStringifiedPlan, Union, Unnest, Values, Window, projection_schema, }; diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 477e9c73d5356..d0496b36c060e 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -61,6 +61,7 @@ use arrow::compute::SortOptions; use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef}; use datafusion_common::cse::{NormalizeEq, Normalizeable}; use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType}; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::metadata::check_metadata_with_storage_equal; use datafusion_common::tree_node::{ Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion, @@ -70,7 +71,8 @@ use datafusion_common::{ FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result, ScalarValue, Spans, SplitPoint, TableReference, UnnestOptions, aggregate_functional_dependencies, assert_eq_or_internal_err, assert_or_internal_err, - internal_err, plan_datafusion_err, plan_err, validate_range_split_points, + information_schema, internal_err, plan_datafusion_err, plan_err, + validate_range_split_points, }; use itertools::Itertools as _; @@ -354,9 +356,10 @@ pub enum LogicalPlan { Ddl(DdlStatement), /// `COPY TO` for writing plan results to files Copy(CopyTo), - /// Describe the schema of the table. This is used to implement the - /// SQL `DESCRIBE` command from MySQL. - DescribeTable(DescribeTable), + /// A `SHOW`-class SQL statement: `SHOW TABLES`, `SHOW COLUMNS`, + /// `SHOW FUNCTIONS`, `SHOW `, `SHOW CREATE TABLE`, and + /// `DESCRIBE` (MySQL's alias for `SHOW COLUMNS FROM`). + Show(Show), /// Unnest a column that contains a nested list type such as an /// ARRAY. This is used to implement SQL `UNNEST` Unnest(Unnest), @@ -427,9 +430,7 @@ impl LogicalPlan { LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => cte.schema(), LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => &scan.schema, LogicalPlan::Union(Union { schema, .. }) => schema, - LogicalPlan::DescribeTable(DescribeTable { output_schema, .. }) => { - output_schema - } + LogicalPlan::Show(Show { output_schema, .. }) => output_schema, LogicalPlan::Dml(DmlStatement { output_schema, .. }) => output_schema, LogicalPlan::Copy(CopyTo { output_schema, .. }) => output_schema, LogicalPlan::Ddl(ddl) => ddl.schema(), @@ -466,11 +467,7 @@ impl LogicalPlan { /// Returns the (fixed) output schema for `DESCRIBE` plans pub fn describe_schema() -> Schema { - Schema::new(vec![ - Field::new("column_name", DataType::Utf8, false), - Field::new("data_type", DataType::Utf8, false), - Field::new("is_nullable", DataType::Utf8, false), - ]) + information_schema::columns_schema(ColumnsDetail::Describe) } /// Returns all expressions (non-recursively) evaluated by the current @@ -572,7 +569,7 @@ impl LogicalPlan { LogicalPlan::TableScan { .. } | LogicalPlan::EmptyRelation { .. } | LogicalPlan::Values { .. } - | LogicalPlan::DescribeTable(_) => vec![], + | LogicalPlan::Show(_) => vec![], } } @@ -700,7 +697,7 @@ impl LogicalPlan { | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) | LogicalPlan::Ddl(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Unnest(_) => Ok(None), } } @@ -883,7 +880,7 @@ impl LogicalPlan { LogicalPlan::TableScan(_) => Ok(self), LogicalPlan::EmptyRelation(_) => Ok(self), LogicalPlan::Statement(_) => Ok(self), - LogicalPlan::DescribeTable(_) => Ok(self), + LogicalPlan::Show(_) => Ok(self), LogicalPlan::Unnest(Unnest { input, exec_columns, @@ -1365,7 +1362,7 @@ impl LogicalPlan { LogicalPlan::EmptyRelation(_) | LogicalPlan::Ddl(_) | LogicalPlan::Statement(_) - | LogicalPlan::DescribeTable(_) => { + | LogicalPlan::Show(_) => { // All of these plan types have no inputs / exprs so should not be called self.assert_no_expressions(expr)?; self.assert_no_inputs(inputs)?; @@ -1629,7 +1626,7 @@ impl LogicalPlan { | LogicalPlan::Analyze(_) | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Statement(_) | LogicalPlan::Extension(_) | LogicalPlan::MaterializedCte(_) => None, @@ -1707,7 +1704,7 @@ impl LogicalPlan { | LogicalPlan::Analyze(_) | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Statement(_) | LogicalPlan::Extension(_) | LogicalPlan::MaterializedCte(_) => 0, @@ -1748,7 +1745,7 @@ impl LogicalPlan { LogicalPlan::Dml(_) => Ok(None), LogicalPlan::Ddl(_) => Ok(None), LogicalPlan::Copy(_) => Ok(None), - LogicalPlan::DescribeTable(_) => Ok(None), + LogicalPlan::Show(_) => Ok(None), LogicalPlan::Unnest(_) => Ok(None), LogicalPlan::RecursiveQuery(_) => Ok(None), } @@ -1788,7 +1785,7 @@ impl LogicalPlan { LogicalPlan::Dml(_) => Ok(None), LogicalPlan::Ddl(_) => Ok(None), LogicalPlan::Copy(_) => Ok(None), - LogicalPlan::DescribeTable(_) => Ok(None), + LogicalPlan::Show(_) => Ok(None), LogicalPlan::Unnest(_) => Ok(None), LogicalPlan::RecursiveQuery(_) => Ok(None), } @@ -2540,9 +2537,18 @@ impl LogicalPlan { )) => { write!(f, "MaterializedCteScan: name={name}") } - LogicalPlan::DescribeTable(DescribeTable { .. }) => { - write!(f, "DescribeTable") - } + LogicalPlan::Show(Show { kind, .. }) => match kind { + ShowKind::Describe { .. } + | ShowKind::Columns { + detail: ColumnsDetail::Describe, + .. + } => write!(f, "DescribeTable"), + ShowKind::Tables => write!(f, "ShowTables"), + ShowKind::Columns { .. } => write!(f, "ShowColumns"), + ShowKind::CreateTable { .. } => write!(f, "ShowCreateTable"), + ShowKind::Variables { .. } => write!(f, "ShowVariables"), + ShowKind::Functions { .. } => write!(f, "ShowFunctions"), + }, LogicalPlan::Unnest(Unnest { input: plan, list_type_columns: list_col_indices, @@ -3783,9 +3789,10 @@ impl PartialOrd for Union { } } -/// Describe the schema of table +/// A `SHOW`-class SQL statement: `SHOW TABLES`, `SHOW COLUMNS`, `SHOW +/// FUNCTIONS`, `SHOW `, `SHOW CREATE TABLE`, and `DESCRIBE`. /// -/// # Example output: +/// # Example output (`DESCRIBE`): /// /// ```sql /// > describe traces; @@ -3806,16 +3813,52 @@ impl PartialOrd for Union { /// +--------------------+-----------------------------+-------------+ /// ``` #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct DescribeTable { - /// Table schema - pub schema: Arc, - /// schema of describe table output +pub struct Show { + /// Which `SHOW`-class statement this is, and any parameters it carries + pub kind: ShowKind, + /// schema of the output produced by this statement pub output_schema: DFSchemaRef, } +/// The specific `SHOW`-class statement carried by a [`Show`] plan. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ShowKind { + /// `DESCRIBE ` + Describe { + /// The schema to describe + schema: Arc, + }, + /// `SHOW TABLES` + Tables, + /// `SHOW [FULL|EXTENDED] COLUMNS FROM
` and `DESCRIBE
` + Columns { + /// Table to describe + table_ref: TableReference, + /// How detailed the output should be + detail: ColumnsDetail, + }, + /// `SHOW CREATE TABLE
` + CreateTable { + /// Table to describe + table_ref: TableReference, + }, + /// `SHOW ` / `SHOW ALL` + Variables { + /// The name of the variable to show, or `None` if all variables should be shown. + name: Option, + /// Whether to include the `description` column (`SHOW ... VERBOSE`) + verbose: bool, + }, + /// `SHOW FUNCTIONS [LIKE ]` + Functions { + /// The `LIKE` pattern to filter function names by, if any + filter: Option, + }, +} + // Manual implementation of `PartialOrd`, returning none since there are no comparable types in -// `DescribeTable`. This allows `LogicalPlan` to derive `PartialOrd`. -impl PartialOrd for DescribeTable { +// `Show`. This allows `LogicalPlan` to derive `PartialOrd`. +impl PartialOrd for Show { fn partial_cmp(&self, _other: &Self) -> Option { // There is no relevant comparison for schemas None @@ -6919,21 +6962,25 @@ mod tests { schema: Arc::new(DFSchema::empty()), }); - let describe_table = LogicalPlan::DescribeTable(DescribeTable { - schema: Arc::new(Schema::new(vec![Field::new( - "foo", - DataType::Int32, - false, - )])), + let describe_table = LogicalPlan::Show(Show { + kind: ShowKind::Describe { + schema: Arc::new(Schema::new(vec![Field::new( + "foo", + DataType::Int32, + false, + )])), + }, output_schema: DFSchemaRef::new(DFSchema::empty()), }); - let describe_table_clone = LogicalPlan::DescribeTable(DescribeTable { - schema: Arc::new(Schema::new(vec![Field::new( - "foo", - DataType::Int32, - false, - )])), + let describe_table_clone = LogicalPlan::Show(Show { + kind: ShowKind::Describe { + schema: Arc::new(Schema::new(vec![Field::new( + "foo", + DataType::Int32, + false, + )])), + }, output_schema: DFSchemaRef::new(DFSchema::empty()), }); diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rs index a4739b30247d9..64e50d79a94be 100644 --- a/datafusion/expr/src/logical_plan/tree_node.rs +++ b/datafusion/expr/src/logical_plan/tree_node.rs @@ -409,7 +409,7 @@ impl TreeNode for LogicalPlan { | LogicalPlan::EmptyRelation { .. } | LogicalPlan::Values { .. } | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) - | LogicalPlan::DescribeTable(_) => Transformed::no(self), + | LogicalPlan::Show(_) => Transformed::no(self), }) } } @@ -546,7 +546,7 @@ impl LogicalPlan { | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) => Ok(TreeNodeRecursion::Continue), + | LogicalPlan::Show(_) => Ok(TreeNodeRecursion::Continue), } } @@ -834,7 +834,7 @@ impl LogicalPlan { | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) => Transformed::no(self), + | LogicalPlan::Show(_) => Transformed::no(self), }) } diff --git a/datafusion/expr/src/table_source.rs b/datafusion/expr/src/table_source.rs index f870b5e7ea949..a9a92c0901acd 100644 --- a/datafusion/expr/src/table_source.rs +++ b/datafusion/expr/src/table_source.rs @@ -20,6 +20,7 @@ use crate::{Expr, LogicalPlan}; use arrow::datatypes::SchemaRef; +pub use datafusion_common::TableType; use datafusion_common::{Constraints, Result}; use std::{any::Any, borrow::Cow}; @@ -50,27 +51,6 @@ pub enum TableProviderFilterPushDown { Exact, } -/// Indicates the type of this table for metadata/catalog purposes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TableType { - /// An ordinary physical table. - Base, - /// A non-materialized table that itself uses a query internally to provide data. - View, - /// A transient table. - Temporary, -} - -impl std::fmt::Display for TableType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - TableType::Base => write!(f, "Base"), - TableType::View => write!(f, "View"), - TableType::Temporary => write!(f, "Temporary"), - } - } -} - /// Planning time information about a table. /// /// This trait is used during logical query planning and optimizations, and diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index 2c5a55cc65f85..0f72413c873f2 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -580,7 +580,7 @@ impl OptimizerRule for CommonSubexprEliminate { | LogicalPlan::Explain(_) | LogicalPlan::Analyze(_) | LogicalPlan::Statement(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Distinct(_) | LogicalPlan::Extension(_) | LogicalPlan::Dml(_) diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs index 6253c90ebb235..7c07047aaaf48 100644 --- a/datafusion/optimizer/src/decorrelate.rs +++ b/datafusion/optimizer/src/decorrelate.rs @@ -363,7 +363,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) => { + | LogicalPlan::Show(_) => { if holds_outer_reference(&plan) { Ok(self.stop_pull_up(plan)) } else { diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index 6cf404771302f..2048d84c1feef 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -388,7 +388,7 @@ fn optimize_projections( LogicalPlan::EmptyRelation(_) | LogicalPlan::Values(_) | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) - | LogicalPlan::DescribeTable(_) => { + | LogicalPlan::Show(_) => { // These operators have no inputs, so stop the optimization process. return Ok(Transformed::no(plan)); } diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index e3bf8eff17253..489f54e0a52aa 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -481,7 +481,7 @@ fn map_children_mut Result>( | LogicalPlan::EmptyRelation { .. } | LogicalPlan::Values { .. } | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Ddl(DdlStatement::CreateExternalTable(_)) | LogicalPlan::Ddl(DdlStatement::CreateCatalogSchema(_)) | LogicalPlan::Ddl(DdlStatement::CreateCatalog(_)) diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs index 9037fb8772396..16142c91936bb 100644 --- a/datafusion/proto/src/logical_plan/mod.rs +++ b/datafusion/proto/src/logical_plan/mod.rs @@ -2408,8 +2408,8 @@ impl AsLogicalPlan for LogicalPlanNode { ))), }) } - LogicalPlan::DescribeTable(_) => Err(proto_error( - "LogicalPlan serde is not yet implemented for DescribeTable", + LogicalPlan::Show(_) => Err(proto_error( + "LogicalPlan serde is not yet implemented for Show", )), LogicalPlan::MaterializedCte(_) => { not_impl_err!( diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 0ed504bffaae9..5fcbd59653401 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -16,23 +16,21 @@ // under the License. use std::collections::{BTreeMap, HashMap, HashSet}; -use std::ops::Deref; use std::path::Path; use std::str::FromStr; use std::sync::Arc; use crate::parser::{ - CopyToSource, CopyToStatement, CreateExternalCatalog, CreateExternalTable, DFParser, + CopyToSource, CopyToStatement, CreateExternalCatalog, CreateExternalTable, ExplainStatement, LexOrdering, ResetStatement, Statement as DFStatement, }; -use crate::planner::{ - ContextProvider, PlannerContext, SqlToRel, object_name_to_qualifier, -}; +use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use crate::utils::normalize_ident; use arrow::datatypes::{Field, FieldRef, Fields, Metadata}; use datafusion_common::error::_plan_err; use datafusion_common::format::ExplainStatementOptions; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::parsers::CompressionTypeVariant; use datafusion_common::{ Column, Constraint, Constraints, DFSchema, DFSchemaRef, DataFusionError, Result, @@ -52,9 +50,9 @@ use datafusion_expr::{ CreateExternalCatalog as PlanCreateExternalCatalog, CreateExternalTable as PlanCreateExternalTable, CreateFunction, CreateFunctionBody, CreateIndex as PlanCreateIndex, CreateMemoryTable, CreateView, Deallocate, - DescribeTable, DmlStatement, DropCatalog, DropCatalogSchema, DropFunction, DropTable, - DropView, EmptyRelation, Execute, Explain, ExplainFormat, Expr, ExprSchemable, - Filter, LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, + DmlStatement, DropCatalog, DropCatalogSchema, DropFunction, DropTable, DropView, + EmptyRelation, Execute, Explain, ExplainFormat, Expr, ExprSchemable, Filter, + LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, ResetVariable, SetVariable, SortExpr, Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart, Volatility, WriteOp, cast, @@ -1676,31 +1674,15 @@ impl SqlToRel<'_, S> { /// Generate a logical plan from a "SHOW TABLES" query fn show_tables_to_plan(&self) -> Result { - let Some(tables_table_ref) = self.resolve_info_table("tables") else { - return plan_err!( - "SHOW TABLES is not supported unless information_schema is enabled" - ); - }; - - let query = format!("SELECT * FROM {tables_table_ref};"); - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 + LogicalPlanBuilder::show_tables()?.build() } fn describe_table_to_plan(&self, table_name: ObjectName) -> Result { let table_ref = self.object_name_to_table_reference(table_name)?; + // Do a table lookup to verify the table exists + let _ = self.context_provider.get_table_source(table_ref.clone())?; - let table_source = self.context_provider.get_table_source(table_ref)?; - - let schema = table_source.schema(); - - let output_schema = DFSchema::try_from(LogicalPlan::describe_schema()).unwrap(); - - Ok(LogicalPlan::DescribeTable(DescribeTable { - schema, - output_schema: Arc::new(output_schema), - })) + LogicalPlanBuilder::show_columns(table_ref, ColumnsDetail::Describe)?.build() } fn describe_query_to_plan(&self, query: Query) -> Result { @@ -1708,12 +1690,7 @@ impl SqlToRel<'_, S> { let schema = Arc::new(plan.schema().as_arrow().clone()); - let output_schema = DFSchema::try_from(LogicalPlan::describe_schema()).unwrap(); - - Ok(LogicalPlan::DescribeTable(DescribeTable { - schema, - output_schema: Arc::new(output_schema), - })) + LogicalPlanBuilder::describe(schema)?.build() } fn copy_to_plan(&self, statement: CopyToStatement) -> Result { @@ -2213,59 +2190,25 @@ impl SqlToRel<'_, S> { } fn show_variable_to_plan(&self, variable: &[Ident]) -> Result { - let Some(df_settings_table_ref) = self.resolve_info_table("df_settings") else { - return plan_err!( - "SHOW [VARIABLE] is not supported unless information_schema is enabled" - ); - }; - let verbose = variable .last() .map(|s| ident_to_string(s) == "verbose") .unwrap_or(false); let mut variable_vec = variable.to_vec(); - let mut columns: String = "name, value".to_owned(); if verbose { - columns = format!("{columns}, description"); variable_vec = variable_vec.split_at(variable_vec.len() - 1).0.to_vec(); } let variable = object_name_to_string(&ObjectName::from(variable_vec)); - let base_query = format!("SELECT {columns} FROM {df_settings_table_ref}"); - let query = if variable == "all" { - // Add an ORDER BY so the output comes out in a consistent order - format!("{base_query} ORDER BY name") - } else if variable == "timezone" || variable == "time.zone" { - // we could introduce alias in OptionDefinition if this string matching thing grows - format!("{base_query} WHERE name = 'datafusion.execution.time_zone'") + let name = if variable == "all" { + None } else { - // These values are what are used to make the information_schema table, so we just - // check here, before actually planning or executing the query, if it would produce no - // results, and error preemptively if it would (for a better UX) - let is_valid_variable = self - .context_provider - .options() - .entries() - .iter() - .any(|opt| opt.key == variable); - - // Check if it's a runtime variable - let is_runtime_variable = variable.starts_with("datafusion.runtime."); - - if !is_valid_variable && !is_runtime_variable { - return plan_err!( - "'{variable}' is not a variable which can be viewed with 'SHOW'" - ); - } - - format!("{base_query} WHERE name = '{variable}'") + // No need to validate if the variable exists here; that is handled at execution time. + Some(variable) }; - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - - self.statement_to_plan(rewrite.pop_front().unwrap()) + LogicalPlanBuilder::show_variables(name, verbose)?.build() } fn set_statement_to_plan(&self, statement: Set) -> Result { @@ -2984,219 +2927,43 @@ impl SqlToRel<'_, S> { full: bool, sql_table_name: ObjectName, ) -> Result { - // Figure out the where clause - let where_clause = object_name_to_qualifier( - &sql_table_name, - self.options.enable_ident_normalization, - )?; - - let Some(columns_table_ref) = self.resolve_info_table("columns") else { - return plan_err!( - "SHOW COLUMNS is not supported unless information_schema is enabled" - ); - }; - - // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(sql_table_name)?; - let _ = self.context_provider.get_table_source(table_ref)?; + // Do a table lookup to verify the table exists + let _ = self.context_provider.get_table_source(table_ref.clone())?; // Treat both FULL and EXTENDED as the same - let select_list = if full || extended { - "*" + let detail = if full || extended { + ColumnsDetail::Full } else { - "table_catalog, table_schema, table_name, column_name, data_type, is_nullable" + ColumnsDetail::Basic }; - let query = - format!("SELECT {select_list} FROM {columns_table_ref} WHERE {where_clause}"); - - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 + LogicalPlanBuilder::show_columns(table_ref, detail)?.build() } - /// Rewrite `SHOW FUNCTIONS` to another SQL query - /// The query is based on the `information_schema.routines` and `information_schema.parameters` tables - /// - /// The output columns: - /// - function_name: The name of function - /// - return_type: The return type of the function - /// - parameters: The name of parameters (ordered by the ordinal position) - /// - parameter_types: The type of parameters (ordered by the ordinal position) - /// - description: The description of the function (the description defined in the document) - /// - syntax_example: The syntax_example of the function (the syntax_example defined in the document) + /// `SHOW FUNCTIONS [LIKE ]`. fn show_functions_to_plan( &self, filter: Option, ) -> Result { - let (Some(parameters_table_ref), Some(routines_table_ref)) = ( - self.resolve_info_table("parameters"), - self.resolve_info_table("routines"), - ) else { - return plan_err!( - "SHOW FUNCTIONS is not supported unless information_schema is enabled" - ); - }; - - let where_clause = if let Some(filter) = filter { - match filter { - ShowStatementFilter::Like(like) => { - format!("WHERE p.function_name like '{like}'") - } - _ => return plan_err!("Unsupported SHOW FUNCTIONS filter"), - } - } else { - "".to_string() + let filter = match filter { + Some(ShowStatementFilter::Like(like)) => Some(like), + Some(_) => return plan_err!("Unsupported SHOW FUNCTIONS filter"), + None => None, }; - // Scalar / aggregate / window functions are resolved by joining - // parameters (IN rows aggregated per OUT row) with routines. - // Table functions (UDTFs) don't have parameter rows, so they are - // sourced directly from routines via a UNION branch. Restricting - // the JOIN to non-TABLE routines prevents same-named scalar+UDTF - // pairs (e.g. `generate_series`) from cross-joining. - let where_clause = where_clause.replace("p.function_name", "sc.function_name"); - let query = format!( - r#" -SELECT DISTINCT - sc.function_name, - sc.return_type, - sc.parameters, - sc.parameter_types, - sc.function_type, - sc.description, - sc.syntax_example -FROM ( - SELECT - p.function_name, - p.return_type, - p.parameters, - p.parameter_types, - r.function_type function_type, - r.description description, - r.syntax_example syntax_example - FROM ( - SELECT - o.specific_name function_name, - o.data_type return_type, - array_agg(i.parameter_name ORDER BY i.ordinal_position ASC) parameters, - array_agg(i.data_type ORDER BY i.ordinal_position ASC) parameter_types - FROM ( - SELECT - specific_catalog, - specific_schema, - specific_name, - ordinal_position, - parameter_name, - data_type, - rid - FROM - {parameters_table_ref} - WHERE - parameter_mode = 'OUT' - ) o - LEFT JOIN - ( - SELECT - specific_catalog, - specific_schema, - specific_name, - ordinal_position, - parameter_name, - data_type, - rid - FROM - {parameters_table_ref} - WHERE - parameter_mode = 'IN' - ) i - ON i.specific_catalog = o.specific_catalog - AND i.specific_schema = o.specific_schema - AND i.specific_name = o.specific_name - AND i.rid = o.rid - GROUP BY 1, 2, o.rid - ) as p - JOIN {routines_table_ref} r - ON p.function_name = r.routine_name - AND r.function_type <> 'TABLE' - - UNION ALL - - SELECT - routine_name function_name, - data_type return_type, - array_agg(NULL) FILTER (WHERE FALSE) parameters, - array_agg(NULL) FILTER (WHERE FALSE) parameter_types, - function_type, - description, - syntax_example - FROM {routines_table_ref} - WHERE function_type = 'TABLE' - GROUP BY routine_name, data_type, function_type, description, syntax_example -) sc -{where_clause} - "# - ); - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 + LogicalPlanBuilder::show_functions(filter)?.build() } fn show_create_table_to_plan( &self, sql_table_name: ObjectName, ) -> Result { - let Some(views_table_ref) = self.resolve_info_table("views") else { - return plan_err!( - "SHOW CREATE TABLE is not supported unless information_schema is enabled" - ); - }; - - // Figure out the where clause - let where_clause = object_name_to_qualifier( - &sql_table_name, - self.options.enable_ident_normalization, - )?; - - // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(sql_table_name)?; - let _ = self.context_provider.get_table_source(table_ref)?; - - let query = format!( - "SELECT table_catalog, table_schema, table_name, definition FROM {views_table_ref} WHERE {where_clause}" - ); - - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 - } - - /// Return true if there is a table provider available for "schema.table" - fn resolve_info_table(&self, table: &str) -> Option { - let table_reference = if let Some(system_catalog) = - &self.context_provider.options().catalog.system_catalog - { - TableReference::Full { - catalog: system_catalog.deref().into(), - schema: "information_schema".into(), - table: table.into(), - } - } else { - TableReference::Partial { - schema: "information_schema".into(), - table: table.into(), - } - }; + // Do a table lookup to verify the table exists + let _ = self.context_provider.get_table_source(table_ref.clone())?; - if self - .context_provider - .get_table_source(table_reference.clone()) - .is_ok() - { - Some(table_reference) - } else { - None - } + LogicalPlanBuilder::show_create_table(table_ref)?.build() } fn validate_transaction_kind( diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index de6ba9443cd48..cc501a0971313 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -204,7 +204,7 @@ impl Unparser<'_> { | LogicalPlan::Analyze(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::RecursiveQuery(_) | LogicalPlan::Unnest(_) => not_impl_err!("Unsupported plan: {plan:?}"), LogicalPlan::MaterializedCte(_) => { diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs index a8a18ed3fd290..c744ecc1822c9 100644 --- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs +++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs @@ -173,7 +173,7 @@ async fn run_query_substrait_round_trip( | LogicalPlan::Explain(_) | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Statement(_) => logical_plan, // For any other plan, convert to Substrait logical_plan => { diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 4a1d2f13c8ef6..5b0546b9b6a54 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -23,8 +23,12 @@ SELECT * from information_schema.tables statement error DataFusion error: Error during planning: table 'datafusion.information_schema.schemata' not found SELECT * from information_schema.schemata -statement error DataFusion error: Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled -show all +# SHOW resolves its content directly (like DESCRIBE/SHOW COLUMNS), so it +# works regardless of whether information_schema is enabled. +query TT +SHOW datafusion.catalog.information_schema +---- +datafusion.catalog.information_schema false # Turn it on @@ -213,7 +217,7 @@ datafusion.catalog.has_header true datafusion.catalog.information_schema true datafusion.catalog.location NULL datafusion.catalog.newlines_in_values false -datafusion.catalog.system_catalog NULL +datafusion.catalog.partition_information_schema false datafusion.execution.batch_size 8192 datafusion.execution.coalesce_batches true datafusion.execution.collect_statistics true @@ -379,7 +383,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.catalog.system_catalog NULL The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. +datafusion.catalog.partition_information_schema false When set together with `information_schema`, the `information_schema` of each catalog only describes the objects in that catalog. When not set, the `information_schema` of each catalog describes all catalogs. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. @@ -648,17 +652,22 @@ datafusion information_schema views VIEW query TT EXPLAIN SHOW TABLES ---- -logical_plan TableScan: information_schema.tables projection=[table_catalog, table_schema, table_name, table_type] +logical_plan ShowTables physical_plan StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, table_type] # information_schema_show_tables_no_information_schema +# +# SHOW TABLES reuses the information_schema.tables machinery directly, so it +# works regardless of whether information_schema is enabled; it just won't +# list the information_schema tables themselves when disabled. statement ok set datafusion.catalog.information_schema = false; -statement error Error during planning: SHOW TABLES is not supported unless information_schema is enabled +query TTTT rowsort SHOW TABLES +---- statement ok set datafusion.catalog.information_schema = true; @@ -696,12 +705,17 @@ SHOW EXTENDED COLUMNS FROM t; datafusion public t i 0 NULL NO Int32 NULL NULL 32 2 NULL NULL NULL # information_schema_show_columns_no_information_schema +# +# SHOW COLUMNS resolves the table directly (like DESCRIBE), so it works +# regardless of whether information_schema is enabled. statement ok set datafusion.catalog.information_schema = false; -statement error Error during planning: SHOW COLUMNS is not supported unless information_schema is enabled +query TTTTTT SHOW COLUMNS FROM t +---- +datafusion public t i Int32 NO statement ok set datafusion.catalog.information_schema = true; @@ -732,17 +746,26 @@ SHOW SOMETHING_UNKNOWN; statement ok DROP TABLE t; -# show_unsupported_when_information_schema_off +# show_variable_when_information_schema_off +# +# SHOW resolves its content directly, so it still works (and still rejects +# unknown variable names) regardless of whether information_schema is +# enabled. statement ok set datafusion.catalog.information_schema = false; -statement error Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled +statement error DataFusion error: Error during planning: 'something' is not a variable which can be viewed with 'SHOW' SHOW SOMETHING -statement error Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled +statement error DataFusion error: Error during planning: 'something' is not a variable which can be viewed with 'SHOW' SHOW SOMETHING VERBOSE +query TT +SHOW datafusion.catalog.information_schema +---- +datafusion.catalog.information_schema false + statement ok set datafusion.catalog.information_schema = true; diff --git a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt index d4048a8b6b327..f189965fec0fd 100644 --- a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt +++ b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt @@ -19,8 +19,12 @@ statement error DataFusion error: Error during planning: table 'datafusion.information_schema.tables' not found SELECT * from information_schema.tables -statement error DataFusion error: Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled -show all +# SHOW resolves its content directly (like DESCRIBE/SHOW COLUMNS), so it +# works regardless of whether information_schema is enabled. +query TT +SHOW datafusion.catalog.information_schema +---- +datafusion.catalog.information_schema false # Turn it on @@ -123,14 +127,11 @@ my_other_catalog information_schema views VIEW my_other_catalog my_other_schema t3 BASE TABLE ############ -# Configure a system catalog -# -# When `datafusion.catalog.system_catalog` is set, the `information_schema` -# of each catalog only describes the objects in that catalog. Only the -# `information_schema` of the system catalog describes all catalogs. +# Enable information schema partitioning so that the `information_schema` +# of each catalog only describes the objects in that catalog. ########### statement ok -set datafusion.catalog.system_catalog = system; +set datafusion.catalog.partition_information_schema = true; query TTTT rowsort SELECT * FROM my_catalog.information_schema.tables; @@ -181,41 +182,6 @@ my_other_catalog information_schema tables VIEW my_other_catalog information_schema views VIEW my_other_catalog my_other_schema t3 BASE TABLE -query TTTT rowsort -SELECT * FROM system.information_schema.tables; ----- -datafusion information_schema columns VIEW -datafusion information_schema df_settings VIEW -datafusion information_schema parameters VIEW -datafusion information_schema routines VIEW -datafusion information_schema schemata VIEW -datafusion information_schema tables VIEW -datafusion information_schema views VIEW -my_catalog information_schema columns VIEW -my_catalog information_schema df_settings VIEW -my_catalog information_schema parameters VIEW -my_catalog information_schema routines VIEW -my_catalog information_schema schemata VIEW -my_catalog information_schema tables VIEW -my_catalog information_schema views VIEW -my_catalog my_schema t1 BASE TABLE -my_catalog my_schema t2 BASE TABLE -my_other_catalog information_schema columns VIEW -my_other_catalog information_schema df_settings VIEW -my_other_catalog information_schema parameters VIEW -my_other_catalog information_schema routines VIEW -my_other_catalog information_schema schemata VIEW -my_other_catalog information_schema tables VIEW -my_other_catalog information_schema views VIEW -my_other_catalog my_other_schema t3 BASE TABLE -system information_schema columns VIEW -system information_schema df_settings VIEW -system information_schema parameters VIEW -system information_schema routines VIEW -system information_schema schemata VIEW -system information_schema tables VIEW -system information_schema views VIEW - query TTTTTTT rowsort SELECT * FROM my_catalog.information_schema.schemata; ---- @@ -226,13 +192,6 @@ SELECT * FROM my_other_catalog.information_schema.schemata; ---- my_other_catalog my_other_schema NULL NULL NULL NULL NULL -query TTTTTTT rowsort -SELECT * FROM system.information_schema.schemata; ----- -datafusion public NULL NULL NULL NULL NULL -my_catalog my_schema NULL NULL NULL NULL NULL -my_other_catalog my_other_schema NULL NULL NULL NULL NULL - query TTTT rowsort SELECT table_catalog, table_schema, table_name, column_name FROM my_catalog.information_schema.columns; @@ -246,14 +205,6 @@ FROM my_other_catalog.information_schema.columns; ---- my_other_catalog my_other_schema t3 column1 -query TTTT rowsort -SELECT table_catalog, table_schema, table_name, column_name -FROM system.information_schema.columns; ----- -my_catalog my_schema t1 column1 -my_catalog my_schema t2 column1 -my_other_catalog my_other_schema t3 column1 - query TTT rowsort SELECT table_catalog, table_schema, table_name FROM my_catalog.information_schema.views @@ -262,16 +213,7 @@ WHERE table_schema <> 'information_schema'; my_catalog my_schema t1 my_catalog my_schema t2 -query TTT rowsort -SELECT table_catalog, table_schema, table_name -FROM system.information_schema.views -WHERE table_schema <> 'information_schema'; ----- -my_catalog my_schema t1 -my_catalog my_schema t2 -my_other_catalog my_other_schema t3 - -# SHOW statements use the system catalog's information schema +# SHOW statements are not affected by partition_information_schema and describe all catalogs query TTTT rowsort SHOW TABLES; ---- @@ -299,13 +241,6 @@ my_other_catalog information_schema schemata VIEW my_other_catalog information_schema tables VIEW my_other_catalog information_schema views VIEW my_other_catalog my_other_schema t3 BASE TABLE -system information_schema columns VIEW -system information_schema df_settings VIEW -system information_schema parameters VIEW -system information_schema routines VIEW -system information_schema schemata VIEW -system information_schema tables VIEW -system information_schema views VIEW query TTTTTT SHOW COLUMNS FROM my_catalog.my_schema.t1; @@ -316,30 +251,9 @@ my_catalog my_schema t1 column1 Int64 NO statement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not found SELECT * FROM no_such_catalog.information_schema.tables; -# A system catalog can be an existing catalog. Its information schema then -# describes all catalogs +# Resetting partition_information_schema makes every information schema global again statement ok -set datafusion.catalog.system_catalog = my_catalog; - -query TTT rowsort -SELECT table_catalog, table_schema, table_name -FROM my_catalog.information_schema.tables -WHERE table_schema <> 'information_schema'; ----- -my_catalog my_schema t1 -my_catalog my_schema t2 -my_other_catalog my_other_schema t3 - -query TTT rowsort -SELECT table_catalog, table_schema, table_name -FROM my_other_catalog.information_schema.tables -WHERE table_schema <> 'information_schema'; ----- -my_other_catalog my_other_schema t3 - -# Resetting the system catalog makes every information schema global again -statement ok -RESET datafusion.catalog.system_catalog; +RESET datafusion.catalog.partition_information_schema; query TTT rowsort SELECT table_catalog, table_schema, table_name @@ -350,11 +264,6 @@ my_catalog my_schema t1 my_catalog my_schema t2 my_other_catalog my_other_schema t3 -# Without a system catalog, the information schema of a catalog that does not -# exist cannot be queried either -statement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not found -SELECT * FROM no_such_catalog.information_schema.tables; - # Cleanup statement ok diff --git a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs index 8b02e09f4c961..dbe126327f7bb 100644 --- a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs +++ b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs @@ -70,9 +70,7 @@ pub fn to_substrait_rel( LogicalPlan::Dml(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::Ddl(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::Copy(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, - LogicalPlan::DescribeTable(plan) => { - not_impl_err!("Unsupported plan type: {plan:?}")? - } + LogicalPlan::Show(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::Unnest(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::RecursiveQuery(plan) => { not_impl_err!("Unsupported plan type: {plan:?}")? diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index ca81398508b17..5029e0ac80e02 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -69,7 +69,7 @@ The following configuration settings are available: | datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | | datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | | datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | -| datafusion.catalog.system_catalog | NULL | The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. | +| datafusion.catalog.partition_information_schema | false | When set together with `information_schema`, the `information_schema` of each catalog only describes the objects in that catalog. When not set, the `information_schema` of each catalog describes all catalogs. | | datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | | datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | | datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. |