From e1aafe94b4c963b8ddd39645ae9569925a74a9fe Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 10 Sep 2026 14:47:27 -0500 Subject: [PATCH 1/2] Add RawModuleDefV10Sections --- Cargo.lock | 1 + crates/dst/src/schema.rs | 8 +- crates/lib/Cargo.toml | 1 + crates/lib/src/db/raw_def/v10.rs | 575 +++++++------------------- crates/schema/src/def.rs | 89 ++-- crates/schema/src/def/validate/v10.rs | 100 ++--- 6 files changed, 240 insertions(+), 534 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25166e9e27f..1c087540cdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8529,6 +8529,7 @@ dependencies = [ "enum-map", "hex", "insta", + "paste", "proptest", "proptest-derive", "ron", diff --git a/crates/dst/src/schema.rs b/crates/dst/src/schema.rs index 641281db3c3..80c98dabe4a 100644 --- a/crates/dst/src/schema.rs +++ b/crates/dst/src/schema.rs @@ -456,12 +456,12 @@ mod tests { }], }; - let raw = to_raw_def(&schema); + let raw = to_raw_def(&schema).into_sections(); // Should have Typespace, Types, and Tables sections. - assert!(raw.typespace().is_some()); - assert!(raw.types().is_some()); - let tables = raw.tables().unwrap(); + assert!(raw.typespace.is_some()); + assert!(raw.types.is_some()); + let tables = raw.tables.unwrap(); assert_eq!(tables.len(), 1); let t = &tables[0]; diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index b07774b3ee6..237b64a528b 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -43,6 +43,7 @@ hex.workspace = true serde = { workspace = true, optional = true } blake3.workspace = true enum-map = { workspace = true, optional = true } +paste.workspace = true # For the 'proptest' feature. proptest = { workspace = true, optional = true } diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 580baa1b41e..788364581bc 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -41,71 +41,147 @@ pub struct RawModuleDefV10 { pub sections: Vec, } -/// A section of a V10 module definition. -/// -/// New variants MUST be added to the END of this enum, to maintain ABI compatibility. -#[derive(Debug, Clone, SpacetimeType)] -#[sats(crate = crate)] -#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] -#[non_exhaustive] -pub enum RawModuleDefV10Section { - /// The `Typespace` used by the module. - /// - /// `AlgebraicTypeRef`s in other sections refer to this typespace. - /// See [`crate::db::raw_def::v9::RawModuleDefV9::typespace`] for validation requirements. - Typespace(Typespace), +macro_rules! with_v10_sections { + ($mac:ident) => { + // New variants MUST be added to the END of this enum, to maintain ABI compatibility. + $mac! { + /// The `Typespace` used by the module. + /// + /// `AlgebraicTypeRef`s in other sections refer to this typespace. + /// See [`crate::db::raw_def::v9::RawModuleDefV9::typespace`] for validation requirements. + Typespace(Typespace), - /// Type definitions exported by the module. - Types(Vec), + /// Type definitions exported by the module. + Types(Vec), - /// Table definitions. - Tables(Vec), + /// Table definitions. + Tables(Vec), - /// Reducer definitions. - Reducers(Vec), + /// Reducer definitions. + Reducers(Vec), - /// Procedure definitions. - Procedures(Vec), + /// Procedure definitions. + Procedures(Vec), - /// View definitions. - Views(Vec), + /// View definitions. + Views(Vec), - /// Schedule definitions. - /// - /// Unlike V9 where schedules were embedded in table definitions, - /// V10 stores them in a dedicated section. - Schedules(Vec), + /// Schedule definitions. + /// + /// Unlike V9 where schedules were embedded in table definitions, + /// V10 stores them in a dedicated section. + Schedules(Vec), - /// Lifecycle reducer assignments. - /// - /// Unlike V9 where lifecycle was a field on reducers, - /// V10 stores lifecycle-to-reducer mappings separately. - LifeCycleReducers(Vec), + /// Lifecycle reducer assignments. + /// + /// Unlike V9 where lifecycle was a field on reducers, + /// V10 stores lifecycle-to-reducer mappings separately. + LifeCycleReducers(Vec), - RowLevelSecurity(Vec), //TODO: Add section for Event tables, and Case conversion before exposing this from module + RowLevelSecurity(Vec), //TODO: Add section for Event tables, and Case conversion before exposing this from module - /// Case conversion policy for identifiers in this module. - CaseConversionPolicy(CaseConversionPolicy), + /// Case conversion policy for identifiers in this module. + CaseConversionPolicy(CaseConversionPolicy), - /// Names provided explicitly by the user that do not follow from the case conversion policy. - ExplicitNames(ExplicitNames), + /// Names provided explicitly by the user that do not follow from the case conversion policy. + ExplicitNames(ExplicitNames), + + /// HTTP handler function definitions. + HttpHandlers(Vec), + + /// HTTP route definitions. + HttpRoutes(Vec), + + /// Primary key metadata for views. + ViewPrimaryKeys(Vec), + + /// Submodules, keyed by the namespace they are registered under. + Submodules(Vec), + + /// Declared publish-only configuration. Even an empty section requires ENV support. + Environment(Vec), + } + }; +} - /// HTTP handler function definitions. - HttpHandlers(Vec), +trait SectionPayload { + fn skip_serializing(&self) -> bool { + false + } +} + +impl SectionPayload for Vec { + fn skip_serializing(&self) -> bool { + self.is_empty() + } +} + +impl SectionPayload for Typespace { + fn skip_serializing(&self) -> bool { + self.types.is_empty() + } +} + +impl SectionPayload for CaseConversionPolicy {} + +impl SectionPayload for ExplicitNames { + fn skip_serializing(&self) -> bool { + self.entries.is_empty() + } +} + +macro_rules! define_section_types { + ($($(#[$attr:meta])* $name:ident($payload:ty),)*) => { + /// A section of a V10 module definition. + #[derive(Debug, Clone, SpacetimeType)] + #[sats(crate = crate)] + #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] + #[non_exhaustive] + pub enum RawModuleDefV10Section { + $( $(#[$attr])* $name($payload), )* + } - /// HTTP route definitions. - HttpRoutes(Vec), + paste::paste! { + #[derive(Debug, Default)] + pub struct RawModuleDefV10Sections { + $($(#[$attr])* pub [< $name:snake >]: Option<$payload>,)* + } - /// Primary key metadata for views. - ViewPrimaryKeys(Vec), + impl FromIterator for RawModuleDefV10Sections { + fn from_iter>(iter: I) -> Self { + let mut sections = Self::default(); + for section in iter { + match section { + // TODO(noa): should we error when coming across duplicate sections? merge them? + $(RawModuleDefV10Section::$name(payload) => { sections.[< $name:snake >].get_or_insert(payload); })* + } + } + sections + } + } - /// Submodules, keyed by the namespace they are registered under. - Submodules(Vec), + impl RawModuleDefV10Sections { + #[allow(path_statements)] + const NUM_SECTIONS: usize = [$({ RawModuleDefV10Section::$name; }),*].len(); + } - /// Declared publish-only configuration. Even an empty section requires ENV support. - Environment(Vec), + impl IntoIterator for RawModuleDefV10Sections { + type Item = RawModuleDefV10Section; + type IntoIter = std::iter::Flatten, { Self::NUM_SECTIONS }>>; + fn into_iter(self) -> Self::IntoIter { + [ + $(self.[< $name:snake >].filter(|x| !SectionPayload::skip_serializing(x)).map(RawModuleDefV10Section::$name),)* + ] + .into_iter() + .flatten() + } + } + } + }; } +with_v10_sections!(define_section_types); + #[derive(Debug, Clone, SpacetimeType)] #[sats(crate = crate)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] @@ -605,133 +681,16 @@ pub struct RawViewPrimaryKeyDefV10 { } impl RawModuleDefV10 { - /// Get the submodules for this module definition. - pub fn submodules(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Submodules(submodules) => Some(submodules), - _ => None, - }) - } - - /// Get the types section, if present. - pub fn types(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Types(types) => Some(types), - _ => None, - }) - } - - /// Get the tables section, if present. - pub fn tables(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Tables(tables) => Some(tables), - _ => None, - }) - } - - /// Get the typespace section, if present. - pub fn typespace(&self) -> Option<&Typespace> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Typespace(ts) => Some(ts), - _ => None, - }) - } - - /// Get the reducers section, if present. - pub fn reducers(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Reducers(reducers) => Some(reducers), - _ => None, - }) - } - - /// Get the procedures section, if present. - pub fn procedures(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Procedures(procedures) => Some(procedures), - _ => None, - }) - } - - /// Get the views section, if present. - pub fn views(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Views(views) => Some(views), - _ => None, - }) - } - - /// Get the view primary keys section, if present. - pub fn view_primary_keys(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::ViewPrimaryKeys(primary_keys) => Some(primary_keys), - _ => None, - }) - } - - /// Get the schedules section, if present. - pub fn schedules(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::Schedules(schedules) => Some(schedules), - _ => None, - }) - } - - /// Get the lifecycle reducers section, if present. - pub fn lifecycle_reducers(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::LifeCycleReducers(lcrs) => Some(lcrs), - _ => None, - }) - } - - pub fn tables_mut_for_tests(&mut self) -> &mut Vec { - self.sections - .iter_mut() - .find_map(|s| match s { - RawModuleDefV10Section::Tables(tables) => Some(tables), - _ => None, - }) - .expect("Tables section must exist for tests") - } - - // Get the row-level security section, if present. - pub fn row_level_security(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::RowLevelSecurity(rls) => Some(rls), - _ => None, - }) - } - - pub fn case_conversion_policy(&self) -> CaseConversionPolicy { - self.sections - .iter() - .find_map(|s| match s { - RawModuleDefV10Section::CaseConversionPolicy(policy) => Some(*policy), - _ => None, - }) - .unwrap_or_default() - } - - pub fn explicit_names(&self) -> Option<&ExplicitNames> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::ExplicitNames(names) => Some(names), - _ => None, - }) - } - - pub fn http_handlers(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::HttpHandlers(handlers) => Some(handlers), - _ => None, - }) + /// Convert the vec of sections into a struct for easier processing. + pub fn into_sections(self) -> RawModuleDefV10Sections { + self.sections.into_iter().collect() } +} - pub fn http_routes(&self) -> Option<&Vec> { - self.sections.iter().find_map(|s| match s { - RawModuleDefV10Section::HttpRoutes(routes) => Some(routes), - _ => None, - }) +impl From for RawModuleDefV10 { + fn from(sections: RawModuleDefV10Sections) -> Self { + let sections = sections.into_iter().collect(); + Self { sections } } } @@ -739,7 +698,7 @@ impl RawModuleDefV10 { #[derive(Default)] pub struct RawModuleDefV10Builder { /// The module definition being built. - module: RawModuleDefV10, + module: RawModuleDefV10Sections, /// The type map from `T: 'static` Rust types to sats types. type_map: BTreeMap, @@ -753,154 +712,47 @@ impl RawModuleDefV10Builder { /// Get mutable access to the typespace section, creating it if missing. fn typespace_mut(&mut self) -> &mut Typespace { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Typespace(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::Typespace(Typespace::EMPTY.clone())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Typespace(ts) => ts, - _ => unreachable!("Just ensured Typespace section exists"), - } + self.module.typespace.get_or_insert_default() + } + + /// Get mutable access to the tables section, creating it if missing. + fn tables_mut(&mut self) -> &mut Vec { + self.module.tables.get_or_insert_default() } /// Get mutable access to the reducers section, creating it if missing. fn reducers_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Reducers(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Reducers(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Reducers(reducers) => reducers, - _ => unreachable!("Just ensured Reducers section exists"), - } + self.module.reducers.get_or_insert_default() } /// Get mutable access to the procedures section, creating it if missing. fn procedures_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Procedures(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::Procedures(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Procedures(procedures) => procedures, - _ => unreachable!("Just ensured Procedures section exists"), - } + self.module.procedures.get_or_insert_default() } /// Get mutable access to the views section, creating it if missing. fn views_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Views(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Views(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Views(views) => views, - _ => unreachable!("Just ensured Views section exists"), - } + self.module.views.get_or_insert_default() } /// Get mutable access to the view primary keys section, creating it if missing. fn view_primary_keys_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::ViewPrimaryKeys(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::ViewPrimaryKeys(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::ViewPrimaryKeys(primary_keys) => primary_keys, - _ => unreachable!("Just ensured ViewPrimaryKeys section exists"), - } + self.module.view_primary_keys.get_or_insert_default() } /// Get mutable access to the schedules section, creating it if missing. fn schedules_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Schedules(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Schedules(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Schedules(schedules) => schedules, - _ => unreachable!("Just ensured Schedules section exists"), - } + self.module.schedules.get_or_insert_default() } /// Get mutable access to the lifecycle reducers section, creating it if missing. fn lifecycle_reducers_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::LifeCycleReducers(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::LifeCycleReducers(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::LifeCycleReducers(lcrs) => lcrs, - _ => unreachable!("Just ensured LifeCycleReducers section exists"), - } + self.module.life_cycle_reducers.get_or_insert_default() } /// Get mutable access to the types section, creating it if missing. fn types_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Types(_))) - .unwrap_or_else(|| { - self.module.sections.push(RawModuleDefV10Section::Types(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Types(types) => types, - _ => unreachable!("Just ensured Types section exists"), - } + self.module.types.get_or_insert_default() } /// Add a type to the in-progress module. @@ -912,102 +764,27 @@ impl RawModuleDefV10Builder { /// Get mutable access to the row-level security section, creating it if missing. fn row_level_security_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::RowLevelSecurity(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::RowLevelSecurity(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::RowLevelSecurity(rls) => rls, - _ => unreachable!("Just ensured RowLevelSecurity section exists"), - } + self.module.row_level_security.get_or_insert_default() } /// Get mutable access to the case conversion policy, creating it if missing. fn explicit_names_mut(&mut self) -> &mut ExplicitNames { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::ExplicitNames(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::ExplicitNames(ExplicitNames::default())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::ExplicitNames(names) => names, - _ => unreachable!("Just ensured ExplicitNames section exists"), - } + self.module.explicit_names.get_or_insert_default() } /// Get mutable access to the HTTP handlers section, creating it if missing. fn http_handlers_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::HttpHandlers(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::HttpHandlers(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::HttpHandlers(handlers) => handlers, - _ => unreachable!("Just ensured HttpHandlers section exists"), - } + self.module.http_handlers.get_or_insert_default() } /// Get mutable access to the HTTP routes section, creating it if missing. fn http_routes_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::HttpRoutes(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::HttpRoutes(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::HttpRoutes(routes) => routes, - _ => unreachable!("Just ensured HttpRoutes section exists"), - } + self.module.http_routes.get_or_insert_default() } /// Get mutable access to the environment section, creating it if missing. fn environment_mut(&mut self) -> &mut Vec { - let idx = self - .module - .sections - .iter() - .position(|s| matches!(s, RawModuleDefV10Section::Environment(_))) - .unwrap_or_else(|| { - self.module - .sections - .push(RawModuleDefV10Section::Environment(Vec::new())); - self.module.sections.len() - 1 - }); - - match &mut self.module.sections[idx] { - RawModuleDefV10Section::Environment(env) => env, - _ => unreachable!("Just ensured Environment section exists"), - } + self.module.environment.get_or_insert_default() } /// Create a table builder. @@ -1020,7 +797,7 @@ impl RawModuleDefV10Builder { ) -> RawTableDefBuilderV10<'_> { let source_name = source_name.into(); RawTableDefBuilderV10 { - module: &mut self.module, + module: self, table: RawTableDefV10 { source_name, product_type_ref, @@ -1297,17 +1074,7 @@ impl RawModuleDefV10Builder { namespace: namespace.into(), module, }; - let existing = self.module.sections.iter_mut().find_map(|s| match s { - RawModuleDefV10Section::Submodules(submodules) => Some(submodules), - _ => None, - }); - match existing { - Some(submodules) => submodules.push(submodule), - None => self - .module - .sections - .push(RawModuleDefV10Section::Submodules(vec![submodule])), - } + self.module.submodules.get_or_insert_default().push(submodule); } /// Set the case conversion policy for this module. @@ -1318,12 +1085,7 @@ impl RawModuleDefV10Builder { /// was stored under the original naming convention). pub fn set_case_conversion_policy(&mut self, policy: CaseConversionPolicy) { // Remove any existing policy section. - self.module - .sections - .retain(|s| !matches!(s, RawModuleDefV10Section::CaseConversionPolicy(_))); - self.module - .sections - .push(RawModuleDefV10Section::CaseConversionPolicy(policy)); + self.module.case_conversion_policy = Some(policy); } /// Declare a complete environment schema. @@ -1335,7 +1097,7 @@ impl RawModuleDefV10Builder { /// Finish building, consuming the builder and returning the module. /// The module should be validated before use. pub fn finish(self) -> RawModuleDefV10 { - self.module + self.module.into() } } @@ -1410,7 +1172,7 @@ pub fn sats_name_to_scoped_name_v10(sats_name: &str) -> RawScopedTypeNameV10 { /// Builder for a `RawTableDefV10`. pub struct RawTableDefBuilderV10<'a> { - module: &'a mut RawModuleDefV10, + module: &'a mut RawModuleDefV10Builder, table: RawTableDefV10, } @@ -1522,23 +1284,8 @@ impl RawTableDefBuilderV10<'_> { pub fn finish(self) -> AlgebraicTypeRef { let product_type_ref = self.table.product_type_ref; - let tables = match self - .module - .sections - .iter_mut() - .find(|s| matches!(s, RawModuleDefV10Section::Tables(_))) - { - Some(RawModuleDefV10Section::Tables(t)) => t, - _ => { - self.module.sections.push(RawModuleDefV10Section::Tables(Vec::new())); - match self.module.sections.last_mut().expect("Just pushed Tables section") { - RawModuleDefV10Section::Tables(t) => t, - _ => unreachable!(), - } - } - }; + self.module.tables_mut().push(self.table); - tables.push(self.table); product_type_ref } @@ -1546,13 +1293,7 @@ impl RawTableDefBuilderV10<'_> { pub fn find_col_pos_by_name(&self, column: impl AsRef) -> Option { let column = column.as_ref(); - let typespace = self.module.sections.iter().find_map(|s| { - if let RawModuleDefV10Section::Typespace(ts) = s { - Some(ts) - } else { - None - } - })?; + let typespace = self.module.module.typespace.as_ref()?; typespace .get(self.table.product_type_ref)? diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 87f61de936d..69ed3558194 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -33,7 +33,7 @@ use spacetimedb_data_structures::map::{Equivalent, HashMap}; use spacetimedb_lib::db::raw_def; use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, MethodOrAny, RawColumnDefaultValueV10, RawConstraintDefV10, RawHttpHandlerDefV10, - RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Section, + RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Sections, RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawSubmoduleV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, }; @@ -1107,16 +1107,8 @@ impl From for RawModuleDefV10 { environment, } = val; - let mut sections = Vec::new(); - if let Some(environment) = environment { - sections.push(RawModuleDefV10Section::Environment( - environment.into_declarations().into_iter().map(|x| x.into()).collect(), - )); - } let mut explicit_names = ExplicitNames::default(); - sections.push(RawModuleDefV10Section::Typespace(typespace)); - // Extract lifecycle reducer names before consuming reducers. let raw_lifecycle: Vec = lifecycle_reducers .into_iter() @@ -1131,9 +1123,6 @@ impl From for RawModuleDefV10 { .collect(); let raw_types: Vec = types.into_values().map(Into::into).collect(); - if !raw_types.is_empty() { - sections.push(RawModuleDefV10Section::Types(raw_types)); - } // Collect schedules from tables (V10 stores them in a separate section). // Also collect ExplicitNames for tables: accessor_name → source_name, name → canonical_name. @@ -1157,9 +1146,6 @@ impl From for RawModuleDefV10 { td.into() }) .collect(); - if !raw_tables.is_empty() { - sections.push(RawModuleDefV10Section::Tables(raw_tables)); - } // Collect ExplicitNames for reducers: accessor_name → source_name, name → canonical_name. // local name so re-validating this raw def doesn't reject dotted identifiers. @@ -1173,9 +1159,6 @@ impl From for RawModuleDefV10 { rd.into() }) .collect(); - if !raw_reducers.is_empty() { - sections.push(RawModuleDefV10Section::Reducers(raw_reducers)); - } // Collect ExplicitNames for procedures: accessor_name → source_name, name → canonical_name. let raw_procedures: Vec = procedures @@ -1188,9 +1171,6 @@ impl From for RawModuleDefV10 { pd.into() }) .collect(); - if !raw_procedures.is_empty() { - sections.push(RawModuleDefV10Section::Procedures(raw_procedures)); - } let raw_http_handlers: Vec = http_handlers .into_values() @@ -1198,15 +1178,15 @@ impl From for RawModuleDefV10 { source_name: hd.accessor_name.into(), }) .collect(); - if !raw_http_handlers.is_empty() { - sections.push(RawModuleDefV10Section::HttpHandlers(raw_http_handlers)); - } - if !http_routes.is_empty() { - let raw_http_routes: Vec = - http_routes.into_iter().map(RawHttpRouteDefV10::from).collect(); - sections.push(RawModuleDefV10Section::HttpRoutes(raw_http_routes)); - } + let raw_http_routes: Vec = http_routes + .into_iter() + .map(|route| RawHttpRouteDefV10 { + handler_function: route.handler_name.into(), + method: route.method, + path: RawIdentifier::new(route.path.as_ref()), + }) + .collect(); // Collect ExplicitNames for views: accessor_name → source_name, name → canonical_name. let mut raw_view_primary_keys = Vec::new(); @@ -1240,26 +1220,8 @@ impl From for RawModuleDefV10 { vd.into() }) .collect(); - if !raw_views.is_empty() { - sections.push(RawModuleDefV10Section::Views(raw_views)); - } - if !raw_view_primary_keys.is_empty() { - sections.push(RawModuleDefV10Section::ViewPrimaryKeys(raw_view_primary_keys)); - } - - if !schedules.is_empty() { - sections.push(RawModuleDefV10Section::Schedules(schedules)); - } - - if !raw_lifecycle.is_empty() { - sections.push(RawModuleDefV10Section::LifeCycleReducers(raw_lifecycle)); - } let raw_rls: Vec = row_level_security_raw.into_values().collect(); - if !raw_rls.is_empty() { - sections.push(RawModuleDefV10Section::RowLevelSecurity(raw_rls)); - } - let submodules: Vec<_> = submodules .into_iter() .map(|(namespace, module)| { @@ -1271,14 +1233,29 @@ impl From for RawModuleDefV10 { } }) .collect(); - if !submodules.is_empty() { - sections.push(RawModuleDefV10Section::Submodules(submodules)); - } - - // Always emit ExplicitNames so canonical names survive the round-trip. - sections.push(RawModuleDefV10Section::ExplicitNames(explicit_names)); - RawModuleDefV10 { sections } + let environment = environment.map(|env| env.into_declarations().into_iter().map(|x| x.into()).collect()); + + RawModuleDefV10Sections { + typespace: Some(typespace), + types: Some(raw_types), + tables: Some(raw_tables), + reducers: Some(raw_reducers), + procedures: Some(raw_procedures), + views: Some(raw_views), + schedules: Some(schedules), + life_cycle_reducers: Some(raw_lifecycle), + row_level_security: Some(raw_rls), + case_conversion_policy: None, + // Always emit ExplicitNames so canonical names survive the round-trip. + explicit_names: Some(explicit_names), + http_handlers: Some(raw_http_handlers), + http_routes: Some(raw_http_routes), + view_primary_keys: Some(raw_view_primary_keys), + submodules: Some(submodules), + environment, + } + .into() } } @@ -2778,8 +2755,8 @@ mod tests { .finish(); let module_def: ModuleDef = builder.finish().try_into().expect("valid module"); - let raw = RawModuleDefV10::from(module_def); - let tables = raw.tables().expect("tables section"); + let raw = RawModuleDefV10::from(module_def).into_sections(); + let tables = raw.tables.expect("tables section"); let defaults = &tables[0].default_values; assert_eq!(defaults.len(), 1); assert_eq!(defaults[0].col_id, ColId(1)); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index e7d23b28cd9..40b9eafcbd0 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -84,23 +84,39 @@ impl From for ValidationCase { /// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { - let environment = validate_environment(&def); - let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); - let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); - let case_policy = def.case_conversion_policy().into(); - let explicit_names = def - .explicit_names() - .cloned() - .map(ExplicitNamesLookup::new) - .unwrap_or_default(); - let view_primary_keys = def.view_primary_keys().cloned().unwrap_or_default(); - // The parent chooses the namespaces its submodules are mounted under, so the parent's - // naming policy and explicit names decide their canonical form. - let submodules = validate_submodules( - def.submodules().into_iter().flat_map(|s| s.iter().cloned()).collect(), - case_policy, - &explicit_names, - ); + let RawModuleDefV10Sections { + typespace, + types, + tables, + reducers, + procedures, + views, + schedules, + life_cycle_reducers, + row_level_security, + case_conversion_policy, + explicit_names, + http_handlers, + http_routes, + view_primary_keys, + submodules, + environment, + } = def.into_sections(); + + let mut typespace = typespace.unwrap_or_default(); + let known_type_definitions = types.iter().flatten().map(|def| def.ty); + let case_policy = case_conversion_policy.unwrap_or_default().into(); + let explicit_names = explicit_names.map(ExplicitNamesLookup::new).unwrap_or_default(); + let view_primary_keys = view_primary_keys.unwrap_or_default(); + let submodules = validate_submodules(submodules.into_iter().flatten().collect(), case_policy, &explicit_names); + let environment = environment + .map(|env| { + let declarations = env.into_iter().map(Into::into).collect(); + spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) + }) + .transpose() + .map_err(ValidationError::from) + .map_err(ValidationErrors::from); // Original `typespace` needs to be preserved to be assign `accesor_name`s to columns. let typespace_with_accessor_names = typespace.clone(); @@ -132,9 +148,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { // `combine_errors` or `collect_all_errors` on all the things we need to validate. // Sometimes it is unavoidable to use `?` early and this should be commented on. - let reducers = def - .reducers() - .cloned() + let reducers = reducers .into_iter() .flatten() .map(|reducer| validator.validate_reducer_def(reducer)) @@ -142,9 +156,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { // Later on, in `check_function_names_are_unique`, we'll transform this into an `IndexMap`. .collect_all_errors::>(); - let procedures = def - .procedures() - .cloned() + let procedures = procedures .into_iter() .flatten() .map(|procedure| { @@ -156,9 +168,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { // Later on, in `check_function_names_are_unique`, we'll transform this into an `IndexMap`. .collect_all_errors::>(); - let http_handlers = def - .http_handlers() - .cloned() + let http_handlers = http_handlers .into_iter() .flatten() .map(|handler| { @@ -168,9 +178,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { }) .collect_all_errors::>(); - let views = def - .views() - .cloned() + let views = views .into_iter() .flatten() .map(|view| { @@ -180,9 +188,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { }) .collect_all_errors(); - let tables = def - .tables() - .cloned() + let tables = tables .into_iter() .flatten() .map(|table| { @@ -193,9 +199,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .collect_all_errors(); let mut refmap = HashMap::default(); - let types = def - .types() - .cloned() + let types = types .into_iter() .flatten() .map(|ty| { @@ -211,8 +215,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .as_ref() .ok() .map(|tables_map| { - def.schedules() - .cloned() + schedules .into_iter() .flatten() .map(|schedule| validator.validate_schedule_def(schedule, tables_map)) @@ -225,8 +228,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .as_ref() .ok() .map(|reducers_vec| { - def.lifecycle_reducers() - .cloned() + life_cycle_reducers .into_iter() .flatten() .map(|lifecycle_def| { @@ -256,9 +258,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { let http_handlers_and_routes = http_handlers.and_then(|handlers| { let handlers = check_http_handler_names_are_unique(handlers)?; - let routes = def - .http_routes() - .cloned() + let routes = http_routes .into_iter() .flatten() .map(|route| validator.validate_http_route_def(route, &handlers)) @@ -303,8 +303,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .. } = validator.core; - let row_level_security_raw = def - .row_level_security() + let row_level_security_raw = row_level_security .into_iter() .flatten() .map(|rls| (rls.sql.clone(), rls.to_owned())) @@ -349,19 +348,6 @@ pub fn validate(def: RawModuleDefV10) -> Result { Ok(module_def) } -fn validate_environment(def: &RawModuleDefV10) -> Result> { - let Some(declarations) = def.sections.iter().find_map(|section| match section { - RawModuleDefV10Section::Environment(declarations) => Some(declarations), - _ => None, - }) else { - return Ok(None); - }; - let declarations = declarations.iter().map(|x| x.clone().into()).collect(); - let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) - .map_err(ValidationError::from)?; - Ok(Some(schema)) -} - /// The canonical form of a submodule namespace: the explicit name if one was given, /// otherwise the source name with the parent's case conversion policy applied. /// This mirrors how table and function names are resolved. From f90ea6a60b1e365786ffebdd38424d12eefb09e9 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 10 Sep 2026 14:52:25 -0500 Subject: [PATCH 2/2] Add migrations to ModuleDef --- crates/lib/src/db/raw_def/v10.rs | 12 +++++++++++ crates/schema/src/def.rs | 21 ++++++++++++++++--- crates/schema/src/def/validate/v10.rs | 30 ++++++++++++++++++++++++--- crates/schema/src/def/validate/v9.rs | 1 + crates/schema/src/error.rs | 4 +++- crates/schema/src/identifier.rs | 17 +++++++++++++-- 6 files changed, 76 insertions(+), 9 deletions(-) diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 788364581bc..3cc94e3251c 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -8,6 +8,7 @@ use crate::db::raw_def::v9::{Lifecycle, RawIndexAlgorithm, TableAccess, TableType}; use core::fmt; use spacetimedb_primitives::{ColId, ColList}; +use spacetimedb_sats::hash::Hash; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::typespace::TypespaceBuilder; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, AlgebraicValue, ProductType, SpacetimeType, Typespace}; @@ -100,6 +101,9 @@ macro_rules! with_v10_sections { /// Declared publish-only configuration. Even an empty section requires ENV support. Environment(Vec), + + /// Migrations from old schema versions, keyed by that schema version's hash. + Migrations(Vec), } }; } @@ -1304,3 +1308,11 @@ impl RawTableDefBuilderV10<'_> { .map(|i| ColId(i as u16)) } } + +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawMigrationDefV10 { + pub schema_hash: Hash, + pub dropped: RawModuleDefV10, +} diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 69ed3558194..98903ce5689 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -33,9 +33,10 @@ use spacetimedb_data_structures::map::{Equivalent, HashMap}; use spacetimedb_lib::db::raw_def; use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, MethodOrAny, RawColumnDefaultValueV10, RawConstraintDefV10, RawHttpHandlerDefV10, - RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Sections, - RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, - RawSequenceDefV10, RawSubmoduleV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, + RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawMigrationDefV10, RawModuleDefV10, + RawModuleDefV10Sections, RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, + RawScopedTypeNameV10, RawSequenceDefV10, RawSubmoduleV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, + RawViewPrimaryKeyDefV10, }; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, @@ -186,6 +187,9 @@ pub struct ModuleDef { /// `None` means undeclared; an explicitly empty declaration is `Some(empty)`. environment: Option, + + /// Migrations, keyed by the schema hash of the old module. + migrations: IndexMap, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -1043,6 +1047,7 @@ impl From for RawModuleDefV9 { submodules: _, accessor_path: _, environment: _, + migrations: _, } = val; // Extract column defaults from tables before consuming tables @@ -1105,6 +1110,7 @@ impl From for RawModuleDefV10 { submodules, accessor_path: _, environment, + migrations, } = val; let mut explicit_names = ExplicitNames::default(); @@ -1236,6 +1242,14 @@ impl From for RawModuleDefV10 { let environment = environment.map(|env| env.into_declarations().into_iter().map(|x| x.into()).collect()); + let migrations: Vec = migrations + .into_iter() + .map(|(schema_hash, dropped)| RawMigrationDefV10 { + schema_hash, + dropped: dropped.into(), + }) + .collect(); + RawModuleDefV10Sections { typespace: Some(typespace), types: Some(raw_types), @@ -1254,6 +1268,7 @@ impl From for RawModuleDefV10 { view_primary_keys: Some(raw_view_primary_keys), submodules: Some(submodules), environment, + migrations: Some(migrations), } .into() } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 40b9eafcbd0..4dccf39a502 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -101,6 +101,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { view_primary_keys, submodules, environment, + migrations, } = def.into_sections(); let mut typespace = typespace.unwrap_or_default(); @@ -316,6 +317,28 @@ pub fn validate(def: RawModuleDefV10) -> Result { let typespace_for_generate = typespace_for_generate.finish(); + let migrations = migrations + .into_iter() + .flatten() + .map(|RawMigrationDefV10 { schema_hash, dropped }| { + let submodule_validation = dropped + .sections + .iter() + .all(|section| { + matches!( + section, + RawModuleDefV10Section::Tables(_) + | RawModuleDefV10Section::Types(_) + | RawModuleDefV10Section::Typespace(_) + ) + }) + .then_some(()) + .ok_or_else(|| ValidationError::InvalidMigrationSubmodule { schema_hash }.into()); + let ((), submodule) = (submodule_validation, validate(dropped)).combine_errors()?; + Ok((schema_hash, submodule)) + }) + .collect_all_errors()?; + let mut module_def = ModuleDef { // Set by `apply_namespace` below. path: NamespacePath::root(), @@ -336,6 +359,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { raw_module_def_version: RawModuleDefVersion::V10, submodules, environment, + migrations, }; // Submodules were validated in isolation, so their defs carry root-relative names. @@ -381,7 +405,7 @@ fn validate_submodules( let mut accessors = std::collections::HashSet::with_capacity(submodules.len()); for submodule in submodules { - let source = RawIdentifier::from(submodule.namespace.clone()); + let source = RawIdentifier::new(&submodule.namespace); let accessor = match Identifier::new(source.clone()) { Ok(accessor) => accessor, Err(error) => { @@ -428,14 +452,14 @@ fn validate_submodules( Ok(mut def) => { if !def.environment().is_empty() { errors.push(ValidationError::EnvironmentInSubmodule { - namespace: submodule.namespace.clone(), + namespace: namespace.to_string(), }); } for (lifecycle, opt_id) in def.lifecycle_reducers_map() { if opt_id.is_some() { errors.push(ValidationError::LifecycleInSubmodule { lifecycle, - namespace: submodule.namespace.clone(), + namespace: namespace.to_string(), }); } } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 312dcdd7f55..4461fb89474 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -173,6 +173,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { raw_module_def_version: RawModuleDefVersion::V9OrEarlier, submodules: IndexMap::new(), environment: None, + migrations: Default::default(), }; // Records each def's namespace. V9 has no submodules, so this just resolves everything at diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 450d4590708..33366484010 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -2,7 +2,7 @@ use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_lib::db::raw_def::v10::MethodOrAny; use spacetimedb_lib::db::raw_def::v9::{Lifecycle, RawScopedTypeNameV9}; use spacetimedb_lib::http::ACCEPTABLE_ROUTE_PATH_CHARS_HUMAN_DESCRIPTION; -use spacetimedb_lib::{ProductType, SumType}; +use spacetimedb_lib::{Hash, ProductType, SumType}; use spacetimedb_primitives::{ColId, ColList, ColSet}; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use spacetimedb_sats::{bsatn::DecodeError, raw_identifier::RawIdentifier, AlgebraicType, AlgebraicTypeRef}; @@ -187,6 +187,8 @@ pub enum ValidationError { Environment(#[from] spacetimedb_lib::environment::EnvironmentSchemaError), #[error("submodule {namespace:?} cannot declare environment variables")] EnvironmentInSubmodule { namespace: String }, + #[error("submodule def for migration from schema with hash {schema_hash} contains disallowed sections")] + InvalidMigrationSubmodule { schema_hash: Hash }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/schema/src/identifier.rs b/crates/schema/src/identifier.rs index b5950d46c76..c059678ef5e 100644 --- a/crates/schema/src/identifier.rs +++ b/crates/schema/src/identifier.rs @@ -3,6 +3,7 @@ use lean_string::LeanString; use spacetimedb_data_structures::map::{Equivalent, HashSet}; use spacetimedb_sats::raw_identifier::{RawIdentifier, RawNamespacedIdentifier}; use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st}; +use std::borrow::Borrow; use std::fmt::{self, Debug, Display}; use std::ops::Deref; use std::sync::Arc; @@ -132,9 +133,15 @@ impl Deref for Identifier { } } -impl Equivalent for str { +impl Borrow for Identifier { + fn borrow(&self) -> &str { + self + } +} + +impl Equivalent for String { fn equivalent(&self, other: &Identifier) -> bool { - self == &other.id[..] + self.as_str().equivalent(other) } } @@ -144,6 +151,12 @@ impl PartialEq for Identifier { } } +impl PartialEq for str { + fn eq(&self, other: &Identifier) -> bool { + other == self + } +} + impl From for RawIdentifier { fn from(id: Identifier) -> Self { id.id