diff --git a/Cargo.lock b/Cargo.lock index 7608aded35d..0df2415d073 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8523,6 +8523,7 @@ dependencies = [ "enum-map", "hex", "insta", + "paste", "proptest", "proptest-derive", "ron", diff --git a/crates/bindings-macro/src/sats.rs b/crates/bindings-macro/src/sats.rs index 140bb36f945..989d4d348bf 100644 --- a/crates/bindings-macro/src/sats.rs +++ b/crates/bindings-macro/src/sats.rs @@ -698,9 +698,10 @@ pub(crate) fn derive_serialize(ty: &SatsType) -> TokenStream { let (name,name_str) = (var.ident, &var.name); let tag = i as u8; if let (Some(member), Some(ty)) = (&var.member, var.ty) { + let serializer = quote!(__serializer); quote_spanned! {ty.span()=> // __ reserved name for binding to prevent name conflicts. See module-level doc comment. - Self::#name { #member: __variant } => __serializer.serialize_variant::<#ty>(#tag, Some(#name_str), __variant), + Self::#name { #member: __variant } => #serializer.serialize_variant::<#ty>(#tag, Some(#name_str), __variant), } } else { quote! { diff --git a/crates/dst/src/schema.rs b/crates/dst/src/schema.rs index 641281db3c3..80c98dabe4a 100644 --- a/crates/dst/src/schema.rs +++ b/crates/dst/src/schema.rs @@ -456,12 +456,12 @@ mod tests { }], }; - let raw = to_raw_def(&schema); + let raw = to_raw_def(&schema).into_sections(); // Should have Typespace, Types, and Tables sections. - assert!(raw.typespace().is_some()); - assert!(raw.types().is_some()); - let tables = raw.tables().unwrap(); + assert!(raw.typespace.is_some()); + assert!(raw.types.is_some()); + let tables = raw.tables.unwrap(); assert_eq!(tables.len(), 1); let t = &tables[0]; diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index c4b3088d2ec..504eb7fb2b8 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -43,6 +43,7 @@ hex.workspace = true serde = { workspace = true, optional = true } blake3.workspace = true enum-map = { workspace = true, optional = true } +paste.workspace = true # For the 'proptest' feature. proptest = { workspace = true, optional = true } diff --git a/crates/lib/src/db/raw_def.rs b/crates/lib/src/db/raw_def.rs index a29161403a5..307e893c89a 100644 --- a/crates/lib/src/db/raw_def.rs +++ b/crates/lib/src/db/raw_def.rs @@ -8,6 +8,8 @@ //! The `spacetimedb_schema` crate will in the future perform validation and normalization of these `Raw` types to a canonical form, //! which will be used everywhere. +mod sections; + pub mod v8; // for backwards-compatibility diff --git a/crates/lib/src/db/raw_def/sections.rs b/crates/lib/src/db/raw_def/sections.rs new file mode 100644 index 00000000000..44f871c3953 --- /dev/null +++ b/crates/lib/src/db/raw_def/sections.rs @@ -0,0 +1,74 @@ +pub(super) trait SectionPayload { + fn skip_serializing(&self) -> bool { + false + } +} + +impl SectionPayload for Vec { + fn skip_serializing(&self) -> bool { + self.is_empty() + } +} + +impl SectionPayload for spacetimedb_sats::Typespace { + fn skip_serializing(&self) -> bool { + self.types.is_empty() + } +} + +macro_rules! define_section_types { + (version = $VN:ident, $($(#[$attr:meta])* $name:ident($payload:ty),)*) => { + paste::paste! { + #[doc = "A section of a " $VN " module definition."] + #[derive(Debug, Clone, SpacetimeType)] + #[sats(crate = crate)] + #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] + #[non_exhaustive] + pub enum [< RawModuleDef $VN Section >] { + $( $(#[$attr])* $name($payload), )* + } + + #[derive(Debug, Default)] + pub struct [< RawModuleDef $VN Sections >] { + $($(#[$attr])* pub [< $name:snake >]: Option<$payload>,)* + } + + impl FromIterator<[< RawModuleDef $VN Section >]> for [< RawModuleDef $VN Sections >] { + fn from_iter]>>(iter: I) -> Self { + let mut sections = Self::default(); + for section in iter { + match section { + // TODO(noa): should we error when coming across duplicate sections? merge them? + $([< RawModuleDef $VN Section >]::$name(payload) => { sections.[< $name:snake >].get_or_insert(payload); })* + } + } + sections + } + } + + impl [< RawModuleDef $VN Sections >] { + #[allow(path_statements)] + const NUM_SECTIONS: usize = [$({ [< RawModuleDef $VN Section >]::$name; }),*].len(); + + $(pub fn [< $name:snake _mut >](&mut self) -> &mut $payload { + self.[< $name:snake >].get_or_insert_default() + })* + } + + impl IntoIterator for [< RawModuleDef $VN Sections >] { + type Item = [< RawModuleDef $VN Section >]; + type IntoIter = std::iter::Flatten]>, { Self::NUM_SECTIONS }>>; + fn into_iter(self) -> Self::IntoIter { + [ + $(self.[< $name:snake >] + .filter(|x| !$crate::db::raw_def::sections::SectionPayload::skip_serializing(x)) + .map([< RawModuleDef $VN Section >]::$name),)* + ] + .into_iter() + .flatten() + } + } + } + }; +} +pub(super) use define_section_types; diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 580baa1b41e..a176b40ba2e 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -8,12 +8,15 @@ use crate::db::raw_def::v9::{Lifecycle, RawIndexAlgorithm, TableAccess, TableType}; use core::fmt; use spacetimedb_primitives::{ColId, ColList}; +use spacetimedb_sats::hash::Hash; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::typespace::TypespaceBuilder; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, AlgebraicValue, ProductType, SpacetimeType, Typespace}; use std::any::TypeId; use std::collections::{btree_map, BTreeMap}; +use super::sections::{define_section_types, SectionPayload}; + /// A possibly-invalid raw module definition. /// /// ABI Version 10. @@ -41,14 +44,10 @@ pub struct RawModuleDefV10 { pub sections: Vec, } -/// A section of a V10 module definition. -/// -/// New variants MUST be added to the END of this enum, to maintain ABI compatibility. -#[derive(Debug, Clone, SpacetimeType)] -#[sats(crate = crate)] -#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] -#[non_exhaustive] -pub enum RawModuleDefV10Section { +// New variants MUST be added to the END of this enum, to maintain ABI compatibility. +define_section_types! { + version = V10, + /// The `Typespace` used by the module. /// /// `AlgebraicTypeRef`s in other sections refer to this typespace. @@ -104,6 +103,9 @@ pub enum RawModuleDefV10Section { /// Declared publish-only configuration. Even an empty section requires ENV support. Environment(Vec), + + /// Migrations from old schema versions, keyed by that schema version's hash. + Migrations(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -174,6 +176,8 @@ pub enum CaseConversionPolicy { SnakeCase, } +impl SectionPayload for CaseConversionPolicy {} + #[derive(Debug, Clone, SpacetimeType)] #[sats(crate = crate)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, Ord, PartialOrd))] @@ -226,6 +230,12 @@ pub struct ExplicitNames { entries: Vec, } +impl SectionPayload for ExplicitNames { + fn skip_serializing(&self) -> bool { + self.entries.is_empty() + } +} + impl ExplicitNames { fn insert(&mut self, entry: ExplicitNameEntry) { self.entries.push(entry); @@ -605,133 +615,16 @@ pub struct RawViewPrimaryKeyDefV10 { } impl RawModuleDefV10 { - /// Get the submodules for this module definition. - pub fn submodules(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Submodules(submodules) => Some(submodules), - _ => None, - }) - } - - /// Get the types section, if present. - pub fn types(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Types(types) => Some(types), - _ => None, - }) - } - - /// Get the tables section, if present. - pub fn tables(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Tables(tables) => Some(tables), - _ => None, - }) - } - - /// Get the typespace section, if present. - pub fn typespace(&self) -> Option<&Typespace> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Typespace(ts) => Some(ts), - _ => None, - }) - } - - /// Get the reducers section, if present. - pub fn reducers(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Reducers(reducers) => Some(reducers), - _ => None, - }) - } - - /// Get the procedures section, if present. - pub fn procedures(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Procedures(procedures) => Some(procedures), - _ => None, - }) - } - - /// Get the views section, if present. - pub fn views(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Views(views) => Some(views), - _ => None, - }) - } - - /// Get the view primary keys section, if present. - pub fn view_primary_keys(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::ViewPrimaryKeys(primary_keys) => Some(primary_keys), - _ => None, - }) - } - - /// Get the schedules section, if present. - pub fn schedules(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Schedules(schedules) => Some(schedules), - _ => None, - }) - } - - /// Get the lifecycle reducers section, if present. - pub fn lifecycle_reducers(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::LifeCycleReducers(lcrs) => Some(lcrs), - _ => None, - }) - } - - pub fn tables_mut_for_tests(&mut self) -> &mut Vec { - self.sections - .iter_mut() - .find_map(|s| match s { - RawModuleDefV10Section::Tables(tables) => Some(tables), - _ => None, - }) - .expect("Tables section must exist for tests") - } - - // Get the row-level security section, if present. - pub fn row_level_security(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::RowLevelSecurity(rls) => Some(rls), - _ => None, - }) - } - - pub fn case_conversion_policy(&self) -> CaseConversionPolicy { - self.sections - .iter() - .find_map(|s| match s { - RawModuleDefV10Section::CaseConversionPolicy(policy) => Some(*policy), - _ => None, - }) - .unwrap_or_default() - } - - pub fn explicit_names(&self) -> Option<&ExplicitNames> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::ExplicitNames(names) => Some(names), - _ => None, - }) + /// Convert the vec of sections into a struct for easier processing. + pub fn into_sections(self) -> RawModuleDefV10Sections { + self.sections.into_iter().collect() } +} - pub fn http_handlers(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::HttpHandlers(handlers) => Some(handlers), - _ => None, - }) - } - - pub fn http_routes(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::HttpRoutes(routes) => Some(routes), - _ => None, - }) +impl From for RawModuleDefV10 { + fn from(sections: RawModuleDefV10Sections) -> Self { + let sections = sections.into_iter().collect(); + Self { sections } } } @@ -739,7 +632,7 @@ impl RawModuleDefV10 { #[derive(Default)] pub struct RawModuleDefV10Builder { /// The module definition being built. - module: RawModuleDefV10, + module: RawModuleDefV10Sections, /// The type map from `T: 'static` Rust types to sats types. type_map: BTreeMap, @@ -751,158 +644,6 @@ impl RawModuleDefV10Builder { Default::default() } - /// Get mutable access to the typespace section, creating it if missing. - fn typespace_mut(&mut self) -> &mut Typespace { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Typespace(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::Typespace(Typespace::EMPTY.clone())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Typespace(ts) => ts, - _ => unreachable!("Just ensured Typespace section exists"), - } - } - - /// Get mutable access to the reducers section, creating it if missing. - fn reducers_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Reducers(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Reducers(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Reducers(reducers) => reducers, - _ => unreachable!("Just ensured Reducers section exists"), - } - } - - /// Get mutable access to the procedures section, creating it if missing. - fn procedures_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Procedures(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::Procedures(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Procedures(procedures) => procedures, - _ => unreachable!("Just ensured Procedures section exists"), - } - } - - /// Get mutable access to the views section, creating it if missing. - fn views_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Views(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Views(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Views(views) => views, - _ => unreachable!("Just ensured Views section exists"), - } - } - - /// Get mutable access to the view primary keys section, creating it if missing. - fn view_primary_keys_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::ViewPrimaryKeys(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::ViewPrimaryKeys(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::ViewPrimaryKeys(primary_keys) => primary_keys, - _ => unreachable!("Just ensured ViewPrimaryKeys section exists"), - } - } - - /// Get mutable access to the schedules section, creating it if missing. - fn schedules_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Schedules(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Schedules(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Schedules(schedules) => schedules, - _ => unreachable!("Just ensured Schedules section exists"), - } - } - - /// Get mutable access to the lifecycle reducers section, creating it if missing. - fn lifecycle_reducers_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::LifeCycleReducers(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::LifeCycleReducers(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::LifeCycleReducers(lcrs) => lcrs, - _ => unreachable!("Just ensured LifeCycleReducers section exists"), - } - } - - /// Get mutable access to the types section, creating it if missing. - fn types_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Types(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Types(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Types(types) => types, - _ => unreachable!("Just ensured Types section exists"), - } - } - /// Add a type to the in-progress module. /// /// The returned type must satisfy `AlgebraicType::is_valid_for_client_type_definition` or `AlgebraicType::is_valid_for_client_type_use`. @@ -910,106 +651,6 @@ impl RawModuleDefV10Builder { TypespaceBuilder::add_type::(self) } - /// Get mutable access to the row-level security section, creating it if missing. - fn row_level_security_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::RowLevelSecurity(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::RowLevelSecurity(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::RowLevelSecurity(rls) => rls, - _ => unreachable!("Just ensured RowLevelSecurity section exists"), - } - } - - /// Get mutable access to the case conversion policy, creating it if missing. - fn explicit_names_mut(&mut self) -> &mut ExplicitNames { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::ExplicitNames(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::ExplicitNames(ExplicitNames::default())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::ExplicitNames(names) => names, - _ => unreachable!("Just ensured ExplicitNames section exists"), - } - } - - /// Get mutable access to the HTTP handlers section, creating it if missing. - fn http_handlers_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::HttpHandlers(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::HttpHandlers(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::HttpHandlers(handlers) => handlers, - _ => unreachable!("Just ensured HttpHandlers section exists"), - } - } - - /// Get mutable access to the HTTP routes section, creating it if missing. - fn http_routes_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::HttpRoutes(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::HttpRoutes(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::HttpRoutes(routes) => routes, - _ => unreachable!("Just ensured HttpRoutes section exists"), - } - } - - /// Get mutable access to the environment section, creating it if missing. - fn environment_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Environment(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::Environment(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Environment(env) => env, - _ => unreachable!("Just ensured Environment section exists"), - } - } - /// Create a table builder. /// /// Does not validate that the product_type_ref is valid; this is left to the module validation code. @@ -1020,7 +661,7 @@ impl RawModuleDefV10Builder { ) -> RawTableDefBuilderV10<'_> { let source_name = source_name.into(); RawTableDefBuilderV10 { - module: &mut self.module, + module: self, table: RawTableDefV10 { source_name, product_type_ref, @@ -1120,10 +761,10 @@ impl RawModuleDefV10Builder { ty: AlgebraicType, custom_ordering: bool, ) -> AlgebraicTypeRef { - let ty_ref = self.typespace_mut().add(ty); + let ty_ref = self.module.typespace_mut().add(ty); let scope = scope.into_iter().collect(); let source_name = source_name.into(); - self.types_mut().push(RawTypeDefV10 { + self.module.types_mut().push(RawTypeDefV10 { source_name: RawScopedTypeNameV10 { source_name, scope }, ty: ty_ref, custom_ordering, @@ -1148,7 +789,7 @@ impl RawModuleDefV10Builder { /// This is because `SpacetimeType` is not implemented for `ReducerContext`, /// so it can never act like an ordinary argument.) pub fn add_reducer(&mut self, source_name: impl Into, params: ProductType) { - self.reducers_mut().push(RawReducerDefV10 { + self.module.reducers_mut().push(RawReducerDefV10 { source_name: source_name.into(), params, visibility: FunctionVisibility::ClientCallable, @@ -1173,7 +814,7 @@ impl RawModuleDefV10Builder { params: ProductType, return_type: AlgebraicType, ) { - self.procedures_mut().push(RawProcedureDefV10 { + self.module.procedures_mut().push(RawProcedureDefV10 { source_name: source_name.into(), params, return_type, @@ -1191,7 +832,7 @@ impl RawModuleDefV10Builder { params: ProductType, return_type: AlgebraicType, ) { - self.views_mut().push(RawViewDefV10 { + self.module.views_mut().push(RawViewDefV10 { source_name: source_name.into(), index: index as u32, is_public, @@ -1207,7 +848,7 @@ impl RawModuleDefV10Builder { C: Into, I: IntoIterator, { - self.view_primary_keys_mut().push(RawViewPrimaryKeyDefV10 { + self.module.view_primary_keys_mut().push(RawViewPrimaryKeyDefV10 { view_source_name: view_source_name.into(), columns: columns.into_iter().map(Into::into).collect(), }); @@ -1223,12 +864,12 @@ impl RawModuleDefV10Builder { params: ProductType, ) { let function_name = function_name.into(); - self.lifecycle_reducers_mut().push(RawLifeCycleReducerDefV10 { + self.module.life_cycle_reducers_mut().push(RawLifeCycleReducerDefV10 { lifecycle_spec, function_name: function_name.clone(), }); - self.reducers_mut().push(RawReducerDefV10 { + self.module.reducers_mut().push(RawReducerDefV10 { source_name: function_name, params, visibility: FunctionVisibility::Private, @@ -1249,7 +890,7 @@ impl RawModuleDefV10Builder { column: impl Into, function: impl Into, ) { - self.schedules_mut().push(RawScheduleDefV10 { + self.module.schedules_mut().push(RawScheduleDefV10 { source_name: None, table_name: table.into(), schedule_at_col: column.into(), @@ -1263,13 +904,14 @@ impl RawModuleDefV10Builder { /// /// **NOTE**: The `sql` expression must be unique within the module. pub fn add_row_level_security(&mut self, sql: &str) { - self.row_level_security_mut() + self.module + .row_level_security_mut() .push(RawRowLevelSecurityDefV10 { sql: sql.into() }); } /// Add an HTTP handler to the module. pub fn add_http_handler(&mut self, source_name: impl Into) { - self.http_handlers_mut().push(RawHttpHandlerDefV10 { + self.module.http_handlers_mut().push(RawHttpHandlerDefV10 { source_name: source_name.into(), }); } @@ -1281,7 +923,7 @@ impl RawModuleDefV10Builder { method: MethodOrAny, path: impl Into, ) { - self.http_routes_mut().push(RawHttpRouteDefV10 { + self.module.http_routes_mut().push(RawHttpRouteDefV10 { handler_function: handler_function.into(), method, path: path.into(), @@ -1289,7 +931,7 @@ impl RawModuleDefV10Builder { } pub fn add_explicit_names(&mut self, names: ExplicitNames) { - self.explicit_names_mut().merge(names); + self.module.explicit_names_mut().merge(names); } pub fn add_submodule(&mut self, namespace: impl Into, module: RawModuleDefV10) { @@ -1297,17 +939,7 @@ impl RawModuleDefV10Builder { namespace: namespace.into(), module, }; - let existing = self.module.sections.iter_mut().find_map(|s| match s { - RawModuleDefV10Section::Submodules(submodules) => Some(submodules), - _ => None, - }); - match existing { - Some(submodules) => submodules.push(submodule), - None => self - .module - .sections - .push(RawModuleDefV10Section::Submodules(vec![submodule])), - } + self.module.submodules_mut().push(submodule); } /// Set the case conversion policy for this module. @@ -1318,24 +950,19 @@ impl RawModuleDefV10Builder { /// was stored under the original naming convention). pub fn set_case_conversion_policy(&mut self, policy: CaseConversionPolicy) { // Remove any existing policy section. - self.module - .sections - .retain(|s| !matches!(s, RawModuleDefV10Section::CaseConversionPolicy(_))); - self.module - .sections - .push(RawModuleDefV10Section::CaseConversionPolicy(policy)); + self.module.case_conversion_policy = Some(policy); } /// Declare a complete environment schema. pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { - self.environment_mut().extend(declarations); + self.module.environment_mut().extend(declarations); self } /// Finish building, consuming the builder and returning the module. /// The module should be validated before use. pub fn finish(self) -> RawModuleDefV10 { - self.module + self.module.into() } } @@ -1351,7 +978,7 @@ impl TypespaceBuilder for RawModuleDefV10Builder { AlgebraicType::Ref(*o.get()) } else { let slot_ref = { - let ts = self.typespace_mut(); + let ts = self.module.typespace_mut(); // Bind a fresh alias to the unit type. let slot_ref = ts.add(AlgebraicType::unit()); // Relate `typeid -> fresh alias`. @@ -1361,7 +988,7 @@ impl TypespaceBuilder for RawModuleDefV10Builder { if let Some(sats_name) = source_name { let source_name = sats_name_to_scoped_name_v10(sats_name); - self.types_mut().push(RawTypeDefV10 { + self.module.types_mut().push(RawTypeDefV10 { source_name, ty: slot_ref, // TODO(1.0): we need to update the `TypespaceBuilder` trait to include @@ -1376,7 +1003,7 @@ impl TypespaceBuilder for RawModuleDefV10Builder { // Borrow of `v` has ended here, so we can now convince the borrow checker. let ty = make_ty(self); - self.typespace_mut()[slot_ref] = ty; + self.module.typespace_mut()[slot_ref] = ty; AlgebraicType::Ref(slot_ref) } } @@ -1410,7 +1037,7 @@ pub fn sats_name_to_scoped_name_v10(sats_name: &str) -> RawScopedTypeNameV10 { /// Builder for a `RawTableDefV10`. pub struct RawTableDefBuilderV10<'a> { - module: &'a mut RawModuleDefV10, + module: &'a mut RawModuleDefV10Builder, table: RawTableDefV10, } @@ -1522,23 +1149,8 @@ impl RawTableDefBuilderV10<'_> { pub fn finish(self) -> AlgebraicTypeRef { let product_type_ref = self.table.product_type_ref; - let tables = match self - .module - .sections - .iter_mut() - .find(|s| matches!(s, RawModuleDefV10Section::Tables(_))) - { - Some(RawModuleDefV10Section::Tables(t)) => t, - _ => { - self.module.sections.push(RawModuleDefV10Section::Tables(Vec::new())); - match self.module.sections.last_mut().expect("Just pushed Tables section") { - RawModuleDefV10Section::Tables(t) => t, - _ => unreachable!(), - } - } - }; + self.module.module.tables_mut().push(self.table); - tables.push(self.table); product_type_ref } @@ -1546,13 +1158,7 @@ impl RawTableDefBuilderV10<'_> { pub fn find_col_pos_by_name(&self, column: impl AsRef) -> Option { let column = column.as_ref(); - let typespace = self.module.sections.iter().find_map(|s| { - if let RawModuleDefV10Section::Typespace(ts) = s { - Some(ts) - } else { - None - } - })?; + let typespace = self.module.module.typespace.as_ref()?; typespace .get(self.table.product_type_ref)? @@ -1563,3 +1169,11 @@ impl RawTableDefBuilderV10<'_> { .map(|i| ColId(i as u16)) } } + +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawMigrationDefV10 { + pub schema_hash: Hash, + pub dropped: RawModuleDefV10, +} diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 87f61de936d..98903ce5689 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -33,9 +33,10 @@ use spacetimedb_data_structures::map::{Equivalent, HashMap}; use spacetimedb_lib::db::raw_def; use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, MethodOrAny, RawColumnDefaultValueV10, RawConstraintDefV10, RawHttpHandlerDefV10, - RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Section, - RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, - RawSequenceDefV10, RawSubmoduleV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, + RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawMigrationDefV10, RawModuleDefV10, + RawModuleDefV10Sections, RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, + RawScopedTypeNameV10, RawSequenceDefV10, RawSubmoduleV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, + RawViewPrimaryKeyDefV10, }; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, @@ -186,6 +187,9 @@ pub struct ModuleDef { /// `None` means undeclared; an explicitly empty declaration is `Some(empty)`. environment: Option, + + /// Migrations, keyed by the schema hash of the old module. + migrations: IndexMap, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -1043,6 +1047,7 @@ impl From for RawModuleDefV9 { submodules: _, accessor_path: _, environment: _, + migrations: _, } = val; // Extract column defaults from tables before consuming tables @@ -1105,18 +1110,11 @@ impl From for RawModuleDefV10 { submodules, accessor_path: _, environment, + migrations, } = val; - let mut sections = Vec::new(); - if let Some(environment) = environment { - sections.push(RawModuleDefV10Section::Environment( - environment.into_declarations().into_iter().map(|x| x.into()).collect(), - )); - } let mut explicit_names = ExplicitNames::default(); - sections.push(RawModuleDefV10Section::Typespace(typespace)); - // Extract lifecycle reducer names before consuming reducers. let raw_lifecycle: Vec = lifecycle_reducers .into_iter() @@ -1131,9 +1129,6 @@ impl From for RawModuleDefV10 { .collect(); let raw_types: Vec = types.into_values().map(Into::into).collect(); - if !raw_types.is_empty() { - sections.push(RawModuleDefV10Section::Types(raw_types)); - } // Collect schedules from tables (V10 stores them in a separate section). // Also collect ExplicitNames for tables: accessor_name → source_name, name → canonical_name. @@ -1157,9 +1152,6 @@ impl From for RawModuleDefV10 { td.into() }) .collect(); - if !raw_tables.is_empty() { - sections.push(RawModuleDefV10Section::Tables(raw_tables)); - } // Collect ExplicitNames for reducers: accessor_name → source_name, name → canonical_name. // local name so re-validating this raw def doesn't reject dotted identifiers. @@ -1173,9 +1165,6 @@ impl From for RawModuleDefV10 { rd.into() }) .collect(); - if !raw_reducers.is_empty() { - sections.push(RawModuleDefV10Section::Reducers(raw_reducers)); - } // Collect ExplicitNames for procedures: accessor_name → source_name, name → canonical_name. let raw_procedures: Vec = procedures @@ -1188,9 +1177,6 @@ impl From for RawModuleDefV10 { pd.into() }) .collect(); - if !raw_procedures.is_empty() { - sections.push(RawModuleDefV10Section::Procedures(raw_procedures)); - } let raw_http_handlers: Vec = http_handlers .into_values() @@ -1198,15 +1184,15 @@ impl From for RawModuleDefV10 { source_name: hd.accessor_name.into(), }) .collect(); - if !raw_http_handlers.is_empty() { - sections.push(RawModuleDefV10Section::HttpHandlers(raw_http_handlers)); - } - if !http_routes.is_empty() { - let raw_http_routes: Vec = - http_routes.into_iter().map(RawHttpRouteDefV10::from).collect(); - sections.push(RawModuleDefV10Section::HttpRoutes(raw_http_routes)); - } + let raw_http_routes: Vec = http_routes + .into_iter() + .map(|route| RawHttpRouteDefV10 { + handler_function: route.handler_name.into(), + method: route.method, + path: RawIdentifier::new(route.path.as_ref()), + }) + .collect(); // Collect ExplicitNames for views: accessor_name → source_name, name → canonical_name. let mut raw_view_primary_keys = Vec::new(); @@ -1240,26 +1226,8 @@ impl From for RawModuleDefV10 { vd.into() }) .collect(); - if !raw_views.is_empty() { - sections.push(RawModuleDefV10Section::Views(raw_views)); - } - if !raw_view_primary_keys.is_empty() { - sections.push(RawModuleDefV10Section::ViewPrimaryKeys(raw_view_primary_keys)); - } - - if !schedules.is_empty() { - sections.push(RawModuleDefV10Section::Schedules(schedules)); - } - - if !raw_lifecycle.is_empty() { - sections.push(RawModuleDefV10Section::LifeCycleReducers(raw_lifecycle)); - } let raw_rls: Vec = row_level_security_raw.into_values().collect(); - if !raw_rls.is_empty() { - sections.push(RawModuleDefV10Section::RowLevelSecurity(raw_rls)); - } - let submodules: Vec<_> = submodules .into_iter() .map(|(namespace, module)| { @@ -1271,14 +1239,38 @@ impl From for RawModuleDefV10 { } }) .collect(); - if !submodules.is_empty() { - sections.push(RawModuleDefV10Section::Submodules(submodules)); - } - // Always emit ExplicitNames so canonical names survive the round-trip. - sections.push(RawModuleDefV10Section::ExplicitNames(explicit_names)); + let environment = environment.map(|env| env.into_declarations().into_iter().map(|x| x.into()).collect()); - RawModuleDefV10 { sections } + let migrations: Vec = migrations + .into_iter() + .map(|(schema_hash, dropped)| RawMigrationDefV10 { + schema_hash, + dropped: dropped.into(), + }) + .collect(); + + RawModuleDefV10Sections { + typespace: Some(typespace), + types: Some(raw_types), + tables: Some(raw_tables), + reducers: Some(raw_reducers), + procedures: Some(raw_procedures), + views: Some(raw_views), + schedules: Some(schedules), + life_cycle_reducers: Some(raw_lifecycle), + row_level_security: Some(raw_rls), + case_conversion_policy: None, + // Always emit ExplicitNames so canonical names survive the round-trip. + explicit_names: Some(explicit_names), + http_handlers: Some(raw_http_handlers), + http_routes: Some(raw_http_routes), + view_primary_keys: Some(raw_view_primary_keys), + submodules: Some(submodules), + environment, + migrations: Some(migrations), + } + .into() } } @@ -2778,8 +2770,8 @@ mod tests { .finish(); let module_def: ModuleDef = builder.finish().try_into().expect("valid module"); - let raw = RawModuleDefV10::from(module_def); - let tables = raw.tables().expect("tables section"); + let raw = RawModuleDefV10::from(module_def).into_sections(); + let tables = raw.tables.expect("tables section"); let defaults = &tables[0].default_values; assert_eq!(defaults.len(), 1); assert_eq!(defaults[0].col_id, ColId(1)); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index e7d23b28cd9..4dccf39a502 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -84,23 +84,40 @@ impl From for ValidationCase { /// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { - let environment = validate_environment(&def); - let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); - let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); - let case_policy = def.case_conversion_policy().into(); - let explicit_names = def - .explicit_names() - .cloned() - .map(ExplicitNamesLookup::new) - .unwrap_or_default(); - let view_primary_keys = def.view_primary_keys().cloned().unwrap_or_default(); - // The parent chooses the namespaces its submodules are mounted under, so the parent's - // naming policy and explicit names decide their canonical form. - let submodules = validate_submodules( - def.submodules().into_iter().flat_map(|s| s.iter().cloned()).collect(), - case_policy, - &explicit_names, - ); + let RawModuleDefV10Sections { + typespace, + types, + tables, + reducers, + procedures, + views, + schedules, + life_cycle_reducers, + row_level_security, + case_conversion_policy, + explicit_names, + http_handlers, + http_routes, + view_primary_keys, + submodules, + environment, + migrations, + } = def.into_sections(); + + let mut typespace = typespace.unwrap_or_default(); + let known_type_definitions = types.iter().flatten().map(|def| def.ty); + let case_policy = case_conversion_policy.unwrap_or_default().into(); + let explicit_names = explicit_names.map(ExplicitNamesLookup::new).unwrap_or_default(); + let view_primary_keys = view_primary_keys.unwrap_or_default(); + let submodules = validate_submodules(submodules.into_iter().flatten().collect(), case_policy, &explicit_names); + let environment = environment + .map(|env| { + let declarations = env.into_iter().map(Into::into).collect(); + spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) + }) + .transpose() + .map_err(ValidationError::from) + .map_err(ValidationErrors::from); // Original `typespace` needs to be preserved to be assign `accesor_name`s to columns. let typespace_with_accessor_names = typespace.clone(); @@ -132,9 +149,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { // `combine_errors` or `collect_all_errors` on all the things we need to validate. // Sometimes it is unavoidable to use `?` early and this should be commented on. - let reducers = def - .reducers() - .cloned() + let reducers = reducers .into_iter() .flatten() .map(|reducer| validator.validate_reducer_def(reducer)) @@ -142,9 +157,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { // Later on, in `check_function_names_are_unique`, we'll transform this into an `IndexMap`. .collect_all_errors::>(); - let procedures = def - .procedures() - .cloned() + let procedures = procedures .into_iter() .flatten() .map(|procedure| { @@ -156,9 +169,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { // Later on, in `check_function_names_are_unique`, we'll transform this into an `IndexMap`. .collect_all_errors::>(); - let http_handlers = def - .http_handlers() - .cloned() + let http_handlers = http_handlers .into_iter() .flatten() .map(|handler| { @@ -168,9 +179,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { }) .collect_all_errors::>(); - let views = def - .views() - .cloned() + let views = views .into_iter() .flatten() .map(|view| { @@ -180,9 +189,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { }) .collect_all_errors(); - let tables = def - .tables() - .cloned() + let tables = tables .into_iter() .flatten() .map(|table| { @@ -193,9 +200,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .collect_all_errors(); let mut refmap = HashMap::default(); - let types = def - .types() - .cloned() + let types = types .into_iter() .flatten() .map(|ty| { @@ -211,8 +216,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .as_ref() .ok() .map(|tables_map| { - def.schedules() - .cloned() + schedules .into_iter() .flatten() .map(|schedule| validator.validate_schedule_def(schedule, tables_map)) @@ -225,8 +229,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .as_ref() .ok() .map(|reducers_vec| { - def.lifecycle_reducers() - .cloned() + life_cycle_reducers .into_iter() .flatten() .map(|lifecycle_def| { @@ -256,9 +259,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { let http_handlers_and_routes = http_handlers.and_then(|handlers| { let handlers = check_http_handler_names_are_unique(handlers)?; - let routes = def - .http_routes() - .cloned() + let routes = http_routes .into_iter() .flatten() .map(|route| validator.validate_http_route_def(route, &handlers)) @@ -303,8 +304,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .. } = validator.core; - let row_level_security_raw = def - .row_level_security() + let row_level_security_raw = row_level_security .into_iter() .flatten() .map(|rls| (rls.sql.clone(), rls.to_owned())) @@ -317,6 +317,28 @@ pub fn validate(def: RawModuleDefV10) -> Result { let typespace_for_generate = typespace_for_generate.finish(); + let migrations = migrations + .into_iter() + .flatten() + .map(|RawMigrationDefV10 { schema_hash, dropped }| { + let submodule_validation = dropped + .sections + .iter() + .all(|section| { + matches!( + section, + RawModuleDefV10Section::Tables(_) + | RawModuleDefV10Section::Types(_) + | RawModuleDefV10Section::Typespace(_) + ) + }) + .then_some(()) + .ok_or_else(|| ValidationError::InvalidMigrationSubmodule { schema_hash }.into()); + let ((), submodule) = (submodule_validation, validate(dropped)).combine_errors()?; + Ok((schema_hash, submodule)) + }) + .collect_all_errors()?; + let mut module_def = ModuleDef { // Set by `apply_namespace` below. path: NamespacePath::root(), @@ -337,6 +359,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { raw_module_def_version: RawModuleDefVersion::V10, submodules, environment, + migrations, }; // Submodules were validated in isolation, so their defs carry root-relative names. @@ -349,19 +372,6 @@ pub fn validate(def: RawModuleDefV10) -> Result { Ok(module_def) } -fn validate_environment(def: &RawModuleDefV10) -> Result> { - let Some(declarations) = def.sections.iter().find_map(|section| match section { - RawModuleDefV10Section::Environment(declarations) => Some(declarations), - _ => None, - }) else { - return Ok(None); - }; - let declarations = declarations.iter().map(|x| x.clone().into()).collect(); - let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) - .map_err(ValidationError::from)?; - Ok(Some(schema)) -} - /// The canonical form of a submodule namespace: the explicit name if one was given, /// otherwise the source name with the parent's case conversion policy applied. /// This mirrors how table and function names are resolved. @@ -395,7 +405,7 @@ fn validate_submodules( let mut accessors = std::collections::HashSet::with_capacity(submodules.len()); for submodule in submodules { - let source = RawIdentifier::from(submodule.namespace.clone()); + let source = RawIdentifier::new(&submodule.namespace); let accessor = match Identifier::new(source.clone()) { Ok(accessor) => accessor, Err(error) => { @@ -442,14 +452,14 @@ fn validate_submodules( Ok(mut def) => { if !def.environment().is_empty() { errors.push(ValidationError::EnvironmentInSubmodule { - namespace: submodule.namespace.clone(), + namespace: namespace.to_string(), }); } for (lifecycle, opt_id) in def.lifecycle_reducers_map() { if opt_id.is_some() { errors.push(ValidationError::LifecycleInSubmodule { lifecycle, - namespace: submodule.namespace.clone(), + namespace: namespace.to_string(), }); } } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 312dcdd7f55..4461fb89474 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -173,6 +173,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), environment: None, + migrations: Default::default(), }; // Records each def's namespace. V9 has no submodules, so this just resolves everything at diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 450d4590708..33366484010 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -2,7 +2,7 @@ use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_lib::db::raw_def::v10::MethodOrAny; use spacetimedb_lib::db::raw_def::v9::{Lifecycle, RawScopedTypeNameV9}; use spacetimedb_lib::http::ACCEPTABLE_ROUTE_PATH_CHARS_HUMAN_DESCRIPTION; -use spacetimedb_lib::{ProductType, SumType}; +use spacetimedb_lib::{Hash, ProductType, SumType}; use spacetimedb_primitives::{ColId, ColList, ColSet}; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use spacetimedb_sats::{bsatn::DecodeError, raw_identifier::RawIdentifier, AlgebraicType, AlgebraicTypeRef}; @@ -187,6 +187,8 @@ pub enum ValidationError { Environment(#[from] spacetimedb_lib::environment::EnvironmentSchemaError), #[error("submodule {namespace:?} cannot declare environment variables")] EnvironmentInSubmodule { namespace: String }, + #[error("submodule def for migration from schema with hash {schema_hash} contains disallowed sections")] + InvalidMigrationSubmodule { schema_hash: Hash }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/schema/src/identifier.rs b/crates/schema/src/identifier.rs index b5950d46c76..c059678ef5e 100644 --- a/crates/schema/src/identifier.rs +++ b/crates/schema/src/identifier.rs @@ -3,6 +3,7 @@ use lean_string::LeanString; use spacetimedb_data_structures::map::{Equivalent, HashSet}; use spacetimedb_sats::raw_identifier::{RawIdentifier, RawNamespacedIdentifier}; use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st}; +use std::borrow::Borrow; use std::fmt::{self, Debug, Display}; use std::ops::Deref; use std::sync::Arc; @@ -132,9 +133,15 @@ impl Deref for Identifier { } } -impl Equivalent for str { +impl Borrow for Identifier { + fn borrow(&self) -> &str { + self + } +} + +impl Equivalent for String { fn equivalent(&self, other: &Identifier) -> bool { - self == &other.id[..] + self.as_str().equivalent(other) } } @@ -144,6 +151,12 @@ impl PartialEq for Identifier { } } +impl PartialEq for str { + fn eq(&self, other: &Identifier) -> bool { + other == self + } +} + impl From for RawIdentifier { fn from(id: Identifier) -> Self { id.id