diff --git a/Cargo.lock b/Cargo.lock index a13e6d9b3a2..fdd6a1c041f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8024,6 +8024,7 @@ dependencies = [ "futures", "git2", "glob", + "headers", "http", "ignore", "indicatif", @@ -8131,7 +8132,9 @@ dependencies = [ "bytestring", "derive_more 0.99.20", "enum-as-inner", + "headers", "hex", + "http", "itertools 0.12.1", "proptest", "serde", diff --git a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs index d3564f218ae..9406cdb51ae 100644 --- a/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs +++ b/crates/bindings-csharp/Codegen.Tests/EnvironmentTests.cs @@ -33,15 +33,15 @@ public static class Host { } } namespace SpacetimeDB.Internal { - public abstract record EnvVarType { - public sealed record String(System.ValueTuple Value) : EnvVarType; - public sealed record StringLiteral(string Value) : EnvVarType; - public sealed record Union(System.Collections.Generic.List Value) : EnvVarType; + public abstract record RawEnvVarTypeV10 { + public sealed record String(System.ValueTuple Value) : RawEnvVarTypeV10; + public sealed record StringLiteral(string Value) : RawEnvVarTypeV10; + public sealed record Union(System.Collections.Generic.List Value) : RawEnvVarTypeV10; } - public sealed record EnvironmentDeclaration(string Name, EnvVarType Ty, bool Optional); + public sealed record RawEnvironmentDeclarationV10(string Name, RawEnvVarTypeV10 Ty, bool Optional); public static class Module { - public static System.Collections.Generic.List Declarations = new(); - public static void RegisterEnvironment(EnvironmentDeclaration value) => Declarations.Add(value); + public static System.Collections.Generic.List Declarations = new(); + public static void RegisterEnvironment(RawEnvironmentDeclarationV10 value) => Declarations.Add(value); } } """; @@ -89,9 +89,9 @@ public static void Check() { catch (System.InvalidOperationException) {} var declarations = SpacetimeDB.Internal.Module.Declarations; if (declarations.Count != 5 || declarations[0].Optional || !declarations[1].Optional || - declarations[0].Ty is not SpacetimeDB.Internal.EnvVarType.String || - declarations[2].Ty is not SpacetimeDB.Internal.EnvVarType.Union { Value.Count: 2 } || - declarations[3].Ty is not SpacetimeDB.Internal.EnvVarType.StringLiteral { Value: "reserved" }) + declarations[0].Ty is not SpacetimeDB.Internal.RawEnvVarTypeV10.String || + declarations[2].Ty is not SpacetimeDB.Internal.RawEnvVarTypeV10.Union { Value.Count: 2 } || + declarations[3].Ty is not SpacetimeDB.Internal.RawEnvVarTypeV10.StringLiteral { Value: "reserved" }) throw new System.Exception("bad metadata"); } } diff --git a/crates/bindings-csharp/Codegen/Environment.cs b/crates/bindings-csharp/Codegen/Environment.cs index 5162abfe8e7..4ced2d16845 100644 --- a/crates/bindings-csharp/Codegen/Environment.cs +++ b/crates/bindings-csharp/Codegen/Environment.cs @@ -88,7 +88,7 @@ void Report(ISymbol symbol, string message) => .FirstOrDefault(a => a.AttributeClass?.ToDisplayString() == "SpacetimeDB.EnvValuesAttribute" ); - var constraint = "new global::SpacetimeDB.Internal.EnvVarType.String(default)"; + var constraint = "new global::SpacetimeDB.Internal.RawEnvVarTypeV10.String(default)"; if (attr is not null) { var values = attr.ConstructorArguments.FirstOrDefault(); @@ -115,8 +115,8 @@ void Report(ISymbol symbol, string message) => } constraint = strings.Length == 1 - ? $"new global::SpacetimeDB.Internal.EnvVarType.StringLiteral({Literal(strings[0])})" - : $"new global::SpacetimeDB.Internal.EnvVarType.Union(new global::System.Collections.Generic.List {{ {string.Join(", ", strings.Select(Literal))} }})"; + ? $"new global::SpacetimeDB.Internal.RawEnvVarTypeV10.StringLiteral({Literal(strings[0])})" + : $"new global::SpacetimeDB.Internal.RawEnvVarTypeV10.Union(new global::System.Collections.Generic.List {{ {string.Join(", ", strings.Select(Literal))} }})"; } registrations.Add( $"global::SpacetimeDB.Internal.Module.RegisterEnvironment(new({Literal(name)}, {constraint}, {(optional ? "true" : "false")}));" diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvVarType.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawEnvVarTypeV10.g.cs similarity index 83% rename from crates/bindings-csharp/Runtime/Internal/Autogen/EnvVarType.g.cs rename to crates/bindings-csharp/Runtime/Internal/Autogen/RawEnvVarTypeV10.g.cs index 252c2770d77..83af971c9b1 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvVarType.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawEnvVarTypeV10.g.cs @@ -8,7 +8,7 @@ namespace SpacetimeDB.Internal { [SpacetimeDB.Type] - public partial record EnvVarType : SpacetimeDB.TaggedEnum<( + public partial record RawEnvVarTypeV10 : SpacetimeDB.TaggedEnum<( SpacetimeDB.Unit String, string StringLiteral, System.Collections.Generic.List Union diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawEnvironmentDeclarationV10.g.cs similarity index 77% rename from crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs rename to crates/bindings-csharp/Runtime/Internal/Autogen/RawEnvironmentDeclarationV10.g.cs index fa2d506853b..5f314d1b2df 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/EnvironmentDeclaration.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawEnvironmentDeclarationV10.g.cs @@ -11,18 +11,18 @@ namespace SpacetimeDB.Internal { [SpacetimeDB.Type] [DataContract] - public sealed partial class EnvironmentDeclaration + public sealed partial class RawEnvironmentDeclarationV10 { [DataMember(Name = "name")] public string Name; [DataMember(Name = "ty")] - public EnvVarType Ty; + public RawEnvVarTypeV10 Ty; [DataMember(Name = "optional")] public bool Optional; - public EnvironmentDeclaration( + public RawEnvironmentDeclarationV10( string Name, - EnvVarType Ty, + RawEnvVarTypeV10 Ty, bool Optional ) { @@ -31,7 +31,7 @@ bool Optional this.Optional = Optional; } - public EnvironmentDeclaration() + public RawEnvironmentDeclarationV10() { this.Name = ""; this.Ty = null!; diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 52f750e3903..ee3765d8d76 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -24,6 +24,6 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( System.Collections.Generic.List HttpRoutes, System.Collections.Generic.List ViewPrimaryKeys, System.Collections.Generic.List Submodules, - System.Collections.Generic.List Environment + System.Collections.Generic.List Environment )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 438c3439146..040c3c09ff4 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -20,7 +20,7 @@ partial class RawModuleDefV10 private readonly List httpRouteDefs = []; private readonly List viewDefs = []; private readonly List viewPrimaryKeyDefs = []; - private readonly List environment = []; + private readonly List environment = []; private readonly List rowLevelSecurityDefs = []; private readonly Dictionary> defaultValuesByTable = new(StringComparer.Ordinal); @@ -87,7 +87,7 @@ internal void RegisterTable(RawTableDefV10 table, RawScheduleDefV10? schedule) internal void RegisterView(RawViewDefV10 view) => viewDefs.Add(view); - internal void RegisterEnvironment(EnvironmentDeclaration declaration) => + internal void RegisterEnvironment(RawEnvironmentDeclarationV10 declaration) => environment.Add(declaration); internal void RegisterViewPrimaryKey(string viewSourceName, IEnumerable columns) => @@ -432,7 +432,7 @@ public static void RegisterAnonymousView() moduleDef.RegisterView(def); } - public static void RegisterEnvironment(EnvironmentDeclaration declaration) => + public static void RegisterEnvironment(RawEnvironmentDeclarationV10 declaration) => moduleDef.RegisterEnvironment(declaration); public static void RegisterViewPrimaryKey(string viewSourceName, string[] columns) => diff --git a/crates/bindings-macro/src/environment.rs b/crates/bindings-macro/src/environment.rs index 424b4c85485..d8e02e7ec1d 100644 --- a/crates/bindings-macro/src/environment.rs +++ b/crates/bindings-macro/src/environment.rs @@ -63,11 +63,13 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result quote!(<#ty as ::spacetimedb::rt::EnvironmentValue>::constraint()), Some([value]) => { - quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::StringLiteral(#value.into())) + quote!(::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::StringLiteral(#value.into())) } - Some(values) => quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::Union( - ::std::vec![#(#values.into()),*] - )), + Some(values) => quote!( + ::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::Union( + ::std::vec![#(#values.into()),*] + ) + ), }; let constraint = if values.is_some() { quote!(<#ty as ::spacetimedb::rt::StringEnvironmentValue>::with_constraint(#constraint)) @@ -75,7 +77,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result::OPTIONAL, @@ -96,7 +98,7 @@ pub(crate) fn expand(args: TokenStream, mut item: ItemStruct) -> syn::Result syn::Result { values.push(value); } let constraint = match values.as_slice() { - [value] => quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::StringLiteral(#value.into())), - values => quote!(::spacetimedb::spacetimedb_lib::environment::EnvVarType::Union( - ::std::vec![#(#values.into()),*] - )), + [value] => { + quote!(::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::StringLiteral(#value.into())) + } + values => quote!( + ::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10::Union(::std::vec![#(#values.into()),*]) + ), }; let ident = &item.ident; Ok(quote! { impl ::spacetimedb::rt::EnvironmentValue for #ident { const OPTIONAL: bool = false; - fn constraint() -> ::spacetimedb::spacetimedb_lib::environment::EnvVarType { + fn constraint() -> ::spacetimedb::spacetimedb_lib::db::raw_def::v10::RawEnvVarTypeV10 { #constraint } diff --git a/crates/bindings/Cargo.toml b/crates/bindings/Cargo.toml index 53d7c5cdd51..70d444998e3 100644 --- a/crates/bindings/Cargo.toml +++ b/crates/bindings/Cargo.toml @@ -42,6 +42,7 @@ serde_json.workspace = true [dev-dependencies] insta.workspace = true trybuild.workspace = true +spacetimedb-lib = { path = "../lib", features = ["test"] } [lints] workspace = true diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 6caf1c4ee04..a6e8821429b 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -5,7 +5,8 @@ use crate::table::IndexAlgo; use crate::{sys, AnonymousViewContext, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, ViewContext}; use spacetimedb_lib::bsatn::EncodeError; use spacetimedb_lib::db::raw_def::v10::{ - CaseConversionPolicy, ExplicitNames as RawExplicitNames, RawModuleDefV10Builder, + CaseConversionPolicy, ExplicitNames as RawExplicitNames, RawEnvVarTypeV10, RawEnvironmentDeclarationV10, + RawModuleDefV10Builder, }; pub use spacetimedb_lib::db::raw_def::v9::Lifecycle as LifecycleReducer; use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, TableType, ViewResultHeader}; @@ -929,7 +930,7 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { pub trait EnvironmentValue: Sized { const OPTIONAL: bool; - fn constraint() -> spacetimedb_lib::environment::EnvVarType; + fn constraint() -> RawEnvVarTypeV10; /// Decode a checked host result. Errors must identify only the key, never its value. fn from_environment(value: Option, key: &str) -> Self; @@ -947,8 +948,8 @@ pub trait RequiredEnvironmentValue: EnvironmentValue {} impl EnvironmentValue for String { const OPTIONAL: bool = false; - fn constraint() -> spacetimedb_lib::environment::EnvVarType { - spacetimedb_lib::environment::EnvVarType::String + fn constraint() -> RawEnvVarTypeV10 { + RawEnvVarTypeV10::String } fn from_environment(value: Option, key: &str) -> Self { @@ -961,7 +962,7 @@ impl RequiredEnvironmentValue for String {} impl EnvironmentValue for Option { const OPTIONAL: bool = true; - fn constraint() -> spacetimedb_lib::environment::EnvVarType { + fn constraint() -> RawEnvVarTypeV10 { T::constraint() } @@ -983,9 +984,7 @@ mod string_environment_value_sealed { message = "`#[env(values(...))]` requires `String` or `Option`; map enum variants with `#[env(value = \"...\")]` instead" )] pub trait StringEnvironmentValue: EnvironmentValue + string_environment_value_sealed::Sealed { - fn with_constraint( - constraint: spacetimedb_lib::environment::EnvVarType, - ) -> spacetimedb_lib::environment::EnvVarType { + fn with_constraint(constraint: RawEnvVarTypeV10) -> RawEnvVarTypeV10 { constraint } } @@ -995,7 +994,7 @@ impl StringEnvironmentValue for Option {} /// Register declarative ENV metadata without reading any environment values. #[doc(hidden)] -pub fn register_environment(declarations: fn() -> Vec) { +pub fn register_environment(declarations: fn() -> Vec) { register_describer(move |module| { module.inner.add_environment(declarations()); }); @@ -1067,7 +1066,6 @@ extern "C" fn __describe_module__(description: BytesSink) { } // Serialize the module to bsatn. - module.inner.ensure_environment(); let module_def = module.inner.finish(); let module_def = RawModuleDef::V10(module_def); let bytes = bsatn::to_vec(&module_def).expect("unable to serialize typespace"); diff --git a/crates/bindings/tests/environment_enum_values.rs b/crates/bindings/tests/environment_enum_values.rs index 85ca9515c75..291ed04f1a5 100644 --- a/crates/bindings/tests/environment_enum_values.rs +++ b/crates/bindings/tests/environment_enum_values.rs @@ -1,5 +1,6 @@ use spacetimedb::rt::EnvironmentValue as _; -use spacetimedb::spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration, EnvironmentSchema}; +use spacetimedb::spacetimedb_lib::environment::EnvironmentSchema; +use spacetimedb_lib::db::raw_def::v10::{RawEnvVarTypeV10, RawEnvironmentDeclarationV10}; use std::collections::BTreeMap; #[derive(Debug, PartialEq, Eq, spacetimedb::SpacetimeType, spacetimedb::EnvironmentValue)] @@ -29,14 +30,15 @@ fn typed_mappings_match_exact_schema_strings_and_optional_absence() { ]; assert_eq!( Mode::constraint(), - EnvVarType::Union(cases.iter().map(|(s, _)| s.to_string()).collect()) + RawEnvVarTypeV10::Union(cases.iter().map(|(s, _)| s.to_string()).collect()) ); assert_eq!(Option::::constraint(), Mode::constraint()); - let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + let schema = EnvironmentSchema::new(vec![RawEnvironmentDeclarationV10 { name: "MODE".into(), ty: Mode::constraint(), optional: false, - }]) + } + .into()]) .unwrap(); for (value, variant) in cases { schema @@ -49,7 +51,7 @@ fn typed_mappings_match_exact_schema_strings_and_optional_absence() { ); } assert_eq!(Option::::from_environment(None, "MODE"), None); - assert_eq!(Literal::constraint(), EnvVarType::StringLiteral("only".into())); + assert_eq!(Literal::constraint(), RawEnvVarTypeV10::StringLiteral("only".into())); assert_eq!(Literal::from_environment(Some("only".into()), "VALUE"), Literal::Only); for rejected in ["InProgress", "ready", "in progress ", "private-unmapped-value"] { assert!(schema diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 5ea9f718833..4ca6467996d 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -42,6 +42,7 @@ dirs.workspace = true duct.workspace = true futures.workspace = true fs-err.workspace = true +headers.workspace = true http.workspace = true is-terminal.workspace = true itertools.workspace = true diff --git a/crates/cli/src/schema_extract.rs b/crates/cli/src/schema_extract.rs index 76b2ec64159..6e6d91bcc42 100644 --- a/crates/cli/src/schema_extract.rs +++ b/crates/cli/src/schema_extract.rs @@ -43,16 +43,7 @@ fn inspect_blocking(extractor: PathBuf, bytes: Vec, host_type: String) -> an } pub(crate) fn read_program(path: &std::path::Path) -> anyhow::Result> { - use std::io::Read; - let mut bytes = Vec::new(); - std::fs::File::open(path)? - .take(spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES as u64 + 1) - .read_to_end(&mut bytes)?; - ensure!( - bytes.len() <= spacetimedb_client_api_messages::publish::MAX_MODULE_BYTES, - "Module exceeds publish size limit" - ); - Ok(bytes) + Ok(std::fs::read(path)?) } const MAX_SCHEMA_BYTES: u64 = 16 * 1024 * 1024; diff --git a/crates/cli/src/schema_extract/tests.rs b/crates/cli/src/schema_extract/tests.rs index 6b77ad7cb8b..ab86403e0dd 100644 --- a/crates/cli/src/schema_extract/tests.rs +++ b/crates/cli/src/schema_extract/tests.rs @@ -89,7 +89,9 @@ fn inspector(script: &str) -> (tempfile::TempDir, PathBuf) { async fn local_inspection_passes_exact_bytes_host_and_requires_success() { use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; let raw = RawModuleDef::V10(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + sections: vec![RawModuleDefV10Section::Environment( + schema().into_declarations().into_iter().map(Into::into).collect(), + )], }); let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); let (dir, extractor) = inspector(&format!( @@ -118,7 +120,9 @@ async fn local_inspection_passes_exact_bytes_host_and_requires_success() { async fn synchronous_generate_adapter_uses_same_exact_byte_protocol_inside_a_runtime() { use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; let raw = RawModuleDef::V10(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(schema().into_declarations())], + sections: vec![RawModuleDefV10Section::Environment( + schema().into_declarations().into_iter().map(Into::into).collect(), + )], }); let json = serde_json::to_string(&SerdeWrapper(raw)).unwrap(); let (_dir, extractor) = inspector(&format!( diff --git a/crates/cli/src/subcommands/env.rs b/crates/cli/src/subcommands/env.rs index c0092bad377..614266d3923 100644 --- a/crates/cli/src/subcommands/env.rs +++ b/crates/cli/src/subcommands/env.rs @@ -97,11 +97,11 @@ async fn fetch(request: reqwest::RequestBuilder, query: Query) -> anyhow::Result .body(query.sql()?) .send() .await?; - ensure!( - response.status().is_success(), - "Environment read failed with HTTP {}", - response.status() - ); + let response = response.error_for_status().context("failed to fetch environment")?; + // Read the body incrementally so a misbehaving server cannot make us buffer an unbounded + // response, as `Response::bytes` would. The limit fits every variable at its maximum key and + // value size with worst-case JSON escaping (one byte becomes the six bytes of `\u00XX`), + // plus room for the rest of the SQL result. let mut body = Vec::new(); let limit = MAX_ENV_VARS * (MAX_ENV_KEY_BYTES + MAX_ENV_VALUE_BYTES) * 6 + 64 * 1024; let mut stream = response.bytes_stream(); diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index df51f6f62de..3902c8a5fbc 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -6,9 +6,12 @@ use anyhow::{ensure, Context}; use clap::Arg; use clap::ArgAction::{self, Set, SetTrue}; use clap::{value_parser, ArgMatches, ValueEnum}; +use headers::HeaderMapExt; use reqwest::{StatusCode, Url}; -use spacetimedb_client_api_messages::name::{is_identity, parse_database_name, PublishResult}; +use spacetimedb_client_api_messages::name::{is_identity, parse_database_name, EnvironmentPublishError, PublishResult}; use spacetimedb_client_api_messages::name::{DatabaseNameError, PrePublishResult, PrettyPrintStyle, PublishOp}; +use spacetimedb_client_api_messages::publish::{SpacetimeEnvironment, SpacetimeEnvironmentRemove}; +use spacetimedb_lib::environment::EnvironmentRemove; use std::collections::HashMap; use std::env; use std::path::PathBuf; @@ -355,68 +358,45 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), #[derive(Default)] struct EnvironmentOptions { only: bool, - remove: Vec, - replace: bool, + remove: EnvironmentRemove, } impl EnvironmentOptions { fn from_args(args: &ArgMatches) -> anyhow::Result { - let options = Self { - only: args.get_flag("env_only"), - remove: args - .get_many::("unset_env") - .map(|keys| keys.cloned().collect()) - .unwrap_or_default(), - replace: args.get_flag("replace_env"), + let remove = if args.get_flag("replace_env") { + EnvironmentRemove::All + } else if let Some(remove) = args.get_many::("unset_env") { + let remove = remove.cloned().collect::>(); + ensure!( + remove.len() <= spacetimedb_lib::environment::MAX_ENV_VARS, + "Too many environment removals" + ); + for key in &remove { + spacetimedb_lib::environment::validate_key(key)?; + } + EnvironmentRemove::Keys(remove) + } else { + EnvironmentRemove::No }; - ensure!( - options.remove.len() <= spacetimedb_lib::environment::MAX_ENV_VARS, - "Too many environment removals" - ); - for key in &options.remove { - spacetimedb_lib::environment::validate_key(key)?; - } - Ok(options) + Ok(Self { + only: args.get_flag("env_only"), + remove, + }) } fn validate_values(&self, values: &std::collections::BTreeMap) -> anyhow::Result<()> { - ensure!( - !self.replace || self.remove.is_empty(), - "--replace-env cannot be combined with --unset-env" - ); - for key in &self.remove { - ensure!( - !values.contains_key(key), - "Environment key {key:?} is both supplied and removed" - ); + if let EnvironmentRemove::Keys(remove) = &self.remove { + for key in remove { + ensure!( + !values.contains_key(key), + "Environment key {key:?} is both supplied and removed" + ); + } } Ok(()) } } -fn publication_body( - module: &spacetimedb_schema::def::ModuleDef, - bytes: Vec, - environment: std::collections::BTreeMap, - options: &EnvironmentOptions, -) -> anyhow::Result<(&'static str, Vec)> { - options.validate_values(&environment)?; - if module.environment_declared() || !environment.is_empty() || !options.remove.is_empty() || options.replace { - let body = spacetimedb_client_api_messages::publish::PublishRequest { - module: Some(bytes), - environment, - environment_remove: options.remove.clone(), - environment_replace: options.replace, - expected_module_version: None, - } - .encode()?; - Ok((spacetimedb_client_api_messages::publish::CONTENT_TYPE, body)) - } else { - // Preserve older servers for ordinary modules without environment changes. - Ok(("application/octet-stream", bytes)) - } -} - fn confirm_and_clear( name_or_identity: &str, skip_prompt: bool, @@ -692,7 +672,7 @@ async fn execute_publish_configs<'a>( command_config.get_config_value("env"), |key| std::env::var_os(key), )?; - print!("{}", environment.display()); + print!("{}", environment); let server_address = { let url = Url::parse(&database_host)?; @@ -765,18 +745,16 @@ async fn execute_publish_configs<'a>( // Set the host type. builder = builder.query(&[("host_type", host_type)]); - let (content_type, payload) = - publication_body(&module_schema, program_bytes, environment.values, environment_options)?; - let res = builder - .header(reqwest::header::CONTENT_TYPE, content_type) - .body(payload) - .send() - .await?; - anyhow::ensure!(res.status().is_success(), "Publish failed with HTTP {}", res.status()); - let response: PublishResult = res - .json() - .await - .map_err(|_| anyhow::anyhow!("Invalid publish response"))?; + let mut request = builder.body(program_bytes).build()?; + request + .headers_mut() + .typed_insert(SpacetimeEnvironment(environment.values)); + request + .headers_mut() + .typed_insert(SpacetimeEnvironmentRemove(environment_options.remove.clone())); + + let res = client.execute(request).await?; + let response: PublishResult = res.json_or_error().await?; match response { PublishResult::Success { domain, @@ -815,6 +793,10 @@ async fn execute_publish_configs<'a>( \tspacetime publish {suggested_tld}\n", )); } + PublishResult::EnvironmentError(EnvironmentPublishError::MissingRequiredEnvironment { keys }) => { + anyhow::bail!("Missing required environment variable(s) {keys:?}") + } + PublishResult::EnvironmentError(EnvironmentPublishError::VersionConflict(e)) => anyhow::bail!(e), } } diff --git a/crates/cli/src/subcommands/publish/environment.rs b/crates/cli/src/subcommands/publish/environment.rs index 626c909ddc7..e022eb24494 100644 --- a/crates/cli/src/subcommands/publish/environment.rs +++ b/crates/cli/src/subcommands/publish/environment.rs @@ -1,11 +1,16 @@ //! Resolve explicit overrides without fetching stored secrets. use std::collections::BTreeMap; use std::ffi::OsString; +use std::fmt; pub(super) use crate::schema_extract::{inspect, read_program}; +use crate::util::ResponseExt; use anyhow::Context; +use headers::HeaderMapExt; use serde_json::Value; -use spacetimedb_lib::environment::EnvironmentSchema; +use spacetimedb_client_api_messages::name::EnvironmentPublishError; +use spacetimedb_client_api_messages::publish::SpacetimeEnvironmentRemove; +use spacetimedb_lib::environment::{EnvironmentRemove, EnvironmentSchema}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Source { @@ -26,14 +31,13 @@ pub(super) struct Resolved { pub values: BTreeMap, pub sources: BTreeMap, } -impl Resolved { - pub fn display(&self) -> String { - use std::fmt::Write; - let mut output = String::new(); + +impl fmt::Display for Resolved { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for (name, source) in &self.sources { - let _ = writeln!(output, "Environment {name} ({source})"); + writeln!(f, "Environment {name} ({source})")?; } - output + Ok(()) } } @@ -88,7 +92,7 @@ pub(super) async fn publish_only( options: &super::EnvironmentOptions, ) -> anyhow::Result<()> { use crate::util::{add_auth_header_opt, get_auth_header, y_or_n}; - use spacetimedb_client_api_messages::publish::{EnvironmentMetadata, PublishRequest, CONTENT_TYPE}; + use spacetimedb_client_api_messages::publish::EnvironmentMetadata; let host = config.get_host_url(server)?; let server_url = reqwest::Url::parse(&host)?; @@ -106,52 +110,44 @@ pub(super) async fn publish_only( const { &percent_encoding::NON_ALPHANUMERIC.remove(b'_').remove(b'-') }, ) .to_string(); - let url = format!("{host}/v1/database/{encoded}"); + let url = format!("{host}/v1/database/{encoded}/environment"); // Neither credentials nor publish bodies may be forwarded to redirect destinations. let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .build()?; - let response = add_auth_header_opt(client.get(format!("{url}/environment")), &auth) - .send() - .await?; - anyhow::ensure!( - response.status().is_success(), - "Cannot read environment schema: HTTP {}", - response.status() - ); - let metadata: EnvironmentMetadata = response.json().await.context("Invalid environment metadata")?; + let response = add_auth_header_opt(client.get(&url), &auth).send().await?; + let metadata: EnvironmentMetadata = response + .json_or_error() + .await + .context("failed to fetch environment schema")?; let schema = EnvironmentSchema::new(metadata.declarations)?; let resolved = resolve(&schema, input, |key| std::env::var_os(key))?; options.validate_values(&resolved.values)?; - print!("{}", resolved.display()); - let request = PublishRequest { - module: None, - environment: resolved.values, - environment_remove: options.remove.clone(), - environment_replace: options.replace, - expected_module_version: Some(metadata.module_version), + print!("{}", resolved); + let request = if let EnvironmentRemove::All = options.remove { + client.put(url) + } else { + client.patch(url) }; - let response = add_auth_header_opt(client.put(url), &auth) - .header(reqwest::header::CONTENT_TYPE, CONTENT_TYPE) - .body(request.encode()?) - .send() - .await?; - anyhow::ensure!( - response.status().is_success(), - "Environment publish failed with HTTP {}", - response.status() - ); - match response - .json::() - .await - .map_err(|_| anyhow::anyhow!("Invalid publish response"))? - { - spacetimedb_client_api_messages::name::PublishResult::Success { database_identity, .. } => { - println!("Updated environment for database {database_identity}"); + let mut request = add_auth_header_opt(request, &auth) + .query(&[("expected_module_hash", &metadata.module_hash)]) + .json(&resolved.values) + .build()?; + if let EnvironmentRemove::Keys(_) = options.remove { + request + .headers_mut() + .typed_insert(SpacetimeEnvironmentRemove(options.remove.clone())); + } + let res = client.execute(request).await?; + let response: Result<(), EnvironmentPublishError> = res.json_or_error().await?; + match response { + Ok(()) => { + println!("Successfully updated environment"); Ok(()) } - spacetimedb_client_api_messages::name::PublishResult::PermissionDenied { .. } => { - anyhow::bail!("Permission denied publishing environment values") + Err(EnvironmentPublishError::MissingRequiredEnvironment { keys }) => { + anyhow::bail!("Missing required environment variable(s) {keys:?}") } + Err(EnvironmentPublishError::VersionConflict(e)) => anyhow::bail!(e), } } diff --git a/crates/cli/src/subcommands/publish/environment/tests.rs b/crates/cli/src/subcommands/publish/environment/tests.rs index aec691356ee..2e9642e0096 100644 --- a/crates/cli/src/subcommands/publish/environment/tests.rs +++ b/crates/cli/src/subcommands/publish/environment/tests.rs @@ -54,10 +54,10 @@ fn declared_shell_overrides_are_redacted() { ); assert_eq!(checked, vec!["A", "B", "C", "OPTIONAL"]); assert_eq!( - resolved.display(), + resolved.to_string(), "Environment A (shell)\nEnvironment B (config)\nEnvironment C (shell)\n" ); - assert!(!resolved.display().contains("sentinel")); + assert!(!resolved.to_string().contains("sentinel")); // No declaration means no ambient lookup, including PATH or credentials. let empty = resolve(&EnvironmentSchema::default(), None, |_| panic!("ambient access")).unwrap(); assert!(empty.values.is_empty()); @@ -104,7 +104,7 @@ fn invalid_inputs_fail_without_values_or_lower_priority_fallback() { .unwrap(); assert_eq!(resolved.values["UNDECLARED"], "secret"); assert!(!looked_up.iter().any(|key| key == "UNDECLARED")); - assert!(!resolved.display().contains("secret")); + assert!(!resolved.to_string().contains("secret")); } #[test] @@ -168,7 +168,6 @@ async fn actual_precompiled_declarations_are_inspected_without_server_or_values( .await .unwrap(); let schema = inspected.environment(); - assert!(inspected.environment_declared()); assert!(!schema.get("REQUIRED").unwrap().optional); assert_eq!( schema.get("MODE").unwrap().ty, @@ -177,7 +176,7 @@ async fn actual_precompiled_declarations_are_inspected_without_server_or_values( let config = serde_json::json!({"REQUIRED":"generated-local-inspection-sentinel","MODE":"ready"}); let resolved = resolve(schema, Some(&config), |_| None).unwrap(); assert_eq!(resolved.values.len(), 2); - assert!(!resolved.display().contains("generated-local-inspection-sentinel")); + assert!(!resolved.to_string().contains("generated-local-inspection-sentinel")); assert!(resolve(schema, None, |_| None).unwrap().values.is_empty()); } diff --git a/crates/cli/src/subcommands/publish/wire_tests.rs b/crates/cli/src/subcommands/publish/wire_tests.rs index daf3a265913..219f4d97277 100644 --- a/crates/cli/src/subcommands/publish/wire_tests.rs +++ b/crates/cli/src/subcommands/publish/wire_tests.rs @@ -1,54 +1,6 @@ use super::*; -use spacetimedb_lib::{ - db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}, - RawModuleDef, -}; -use spacetimedb_schema::def::ModuleDef; use std::collections::BTreeMap; -fn schema(declared: bool) -> ModuleDef { - let mut sections = vec![RawModuleDefV10Section::Typespace(Default::default())]; - if declared { - sections.push(RawModuleDefV10Section::Environment(vec![])); - } - ModuleDef::try_from(RawModuleDef::V10(RawModuleDefV10 { sections })).unwrap() -} - -#[test] -fn ordinary_and_explicit_empty_declarations_choose_distinct_wire_formats() { - let bytes = b"exact selected module bytes\0\xff".to_vec(); - let (kind, body) = publication_body( - &schema(false), - bytes.clone(), - BTreeMap::new(), - &EnvironmentOptions::default(), - ) - .unwrap(); - assert_eq!(kind, "application/octet-stream"); - assert_eq!(body, bytes); - let (kind, body) = publication_body( - &schema(true), - bytes.clone(), - BTreeMap::new(), - &EnvironmentOptions::default(), - ) - .unwrap(); - assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); - let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); - assert_eq!(envelope.module, Some(bytes.clone())); - assert!(envelope.environment.is_empty()); - let (kind, body) = publication_body( - &schema(false), - bytes, - BTreeMap::from([("KEY".into(), "secret-sentinel".into())]), - &EnvironmentOptions::default(), - ) - .unwrap(); - assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); - let envelope = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); - assert_eq!(envelope.environment["KEY"], "secret-sentinel"); -} - #[test] fn short_help_is_concise_and_long_help_explains_environment_modes() { let short = cli().render_help().to_string(); @@ -84,7 +36,7 @@ fn environment_flags_are_explicit_and_incompatible_modes_are_rejected() { .unwrap(); let options = EnvironmentOptions::from_args(&args).unwrap(); assert!(options.only); - assert_eq!(options.remove, ["A", "B"]); + assert_eq!(options.remove, EnvironmentRemove::Keys(vec!["A".into(), "B".into()])); assert!(options .validate_values(&BTreeMap::from([("A".into(), "secret-sentinel".into())])) .is_err()); @@ -92,25 +44,5 @@ fn environment_flags_are_explicit_and_incompatible_modes_are_rejected() { .try_get_matches_from(["publish", "db", "--env-only", "--replace-env"]) .unwrap(); let options = EnvironmentOptions::from_args(&args).unwrap(); - assert!(options.only && options.replace); -} - -#[test] -fn removal_and_replacement_use_json_even_for_legacy_modules() { - for options in [ - EnvironmentOptions { - remove: vec!["OLD".into()], - ..Default::default() - }, - EnvironmentOptions { - replace: true, - ..Default::default() - }, - ] { - let (kind, body) = publication_body(&schema(false), vec![1], BTreeMap::new(), &options).unwrap(); - assert_eq!(kind, spacetimedb_client_api_messages::publish::CONTENT_TYPE); - let request = spacetimedb_client_api_messages::publish::PublishRequest::decode(&body).unwrap(); - assert_eq!(request.environment_remove, options.remove); - assert_eq!(request.environment_replace, options.replace); - } + assert!(options.only && options.remove == EnvironmentRemove::All); } diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 2b199f07467..9d9957473a7 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -52,6 +52,10 @@ pub(crate) trait ResponseExt: Sized { /// Like [`reqwest::Response::json()`], but handles non-JSON error messages gracefully. async fn json_or_error(self) -> anyhow::Result; + /// Like [`reqwest::Response::error_for_status()`], but the returned error contains + /// the error message returned in the response body if present. + async fn error_msg_for_status(self) -> anyhow::Result; + /// Transforms a status of `NOT_FOUND` into `None`. fn found(self) -> Option; } @@ -70,7 +74,6 @@ fn err_status_desc(status: http::StatusCode) -> Option<&'static str> { impl ResponseExt for reqwest::Response { async fn ensure_content_type(self, content_type: &str) -> anyhow::Result { - let status = self.status(); if self .headers() .get(http::header::CONTENT_TYPE) @@ -78,9 +81,22 @@ impl ResponseExt for reqwest::Response { { return Ok(self); } + Err(match self.error_msg_for_status().await { + Ok(res) => { + anyhow::anyhow!( + "HTTP response from url ({}) was success but did not have content-type: {content_type}", + res.url() + ) + } + Err(e) => e, + }) + } + + async fn error_msg_for_status(self) -> anyhow::Result { + let status = self.status(); let url = self.url(); let Some(status_desc) = err_status_desc(status) else { - anyhow::bail!("HTTP response from url ({url}) was success but did not have content-type: {content_type}"); + return Ok(self); }; let url = url.to_string(); let status_err = match self.error_for_status_ref() { diff --git a/crates/client-api-messages/Cargo.toml b/crates/client-api-messages/Cargo.toml index f278ac59e02..2a223b1a645 100644 --- a/crates/client-api-messages/Cargo.toml +++ b/crates/client-api-messages/Cargo.toml @@ -20,6 +20,8 @@ serde_with.workspace = true smallvec.workspace = true thiserror.workspace = true derive_more.workspace = true +http.workspace = true +headers.workspace = true [dev-dependencies] hex.workspace = true diff --git a/crates/client-api-messages/src/lib.rs b/crates/client-api-messages/src/lib.rs index 67b58de659f..c174c310170 100644 --- a/crates/client-api-messages/src/lib.rs +++ b/crates/client-api-messages/src/lib.rs @@ -3,6 +3,5 @@ pub mod energy; pub mod http; pub mod name; -pub mod websocket; - pub mod publish; +pub mod websocket; diff --git a/crates/client-api-messages/src/name.rs b/crates/client-api-messages/src/name.rs index b835078ac4d..4642f882fa4 100644 --- a/crates/client-api-messages/src/name.rs +++ b/crates/client-api-messages/src/name.rs @@ -104,8 +104,33 @@ pub enum PublishResult { /// owned by an identity other than the identity that you provided, then you will receive /// this error. PermissionDenied { name: DatabaseName }, + + /// An invalid environment was supplied. + EnvironmentError(EnvironmentPublishError), +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum EnvironmentPublishError { + /// A required environment variable was missing. + MissingRequiredEnvironment { keys: Vec }, + + /// Expected module hash was different + VersionConflict(#[serde(skip)] EnvironmentVersionConflict), } +impl EnvironmentPublishError { + pub fn status_code(&self) -> http::StatusCode { + match self { + Self::MissingRequiredEnvironment { .. } => http::StatusCode::BAD_REQUEST, + Self::VersionConflict(..) => http::StatusCode::CONFLICT, + } + } +} + +#[derive(Debug, Clone, Default, thiserror::Error, serde::Serialize, serde::Deserialize)] +#[error("database program changed before publication; reload environment metadata and retry")] +pub struct EnvironmentVersionConflict; + #[derive(serde::Serialize, serde::Deserialize, Debug, Default)] pub enum MigrationPolicy { #[default] diff --git a/crates/client-api-messages/src/publish.rs b/crates/client-api-messages/src/publish.rs index c77a21d56a8..348af01ae5c 100644 --- a/crates/client-api-messages/src/publish.rs +++ b/crates/client-api-messages/src/publish.rs @@ -1,179 +1,343 @@ -//! Atomic publish input. Environment values travel only in the request body. -use serde::{Deserialize, Deserializer, Serialize}; -use serde_with::{base64::Base64, serde_as}; -use spacetimedb_lib::environment::{validate_key, validate_value, MAX_ENV_VARS}; +//! Environment wire types: the `spacetime-environment` and `spacetime-environment-remove` +//! headers, and the metadata returned by `GET /environment`. use std::collections::BTreeMap; -pub const CONTENT_TYPE: &str = "application/vnd.spacetimedb.publish+json"; -pub const MAX_MODULE_BYTES: usize = 128 * 1024 * 1024; -/// Includes base64 module expansion and worst-case JSON escaping of configuration. -pub const MAX_REQUEST_BYTES: usize = 192 * 1024 * 1024; - -/// Values deliberately have no Debug representation. Omission is an empty map, -/// including for a publish of an unchanged module. -#[serde_as] -#[derive(Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PublishRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde_as(as = "Option")] - pub module: Option>, - #[serde(default, deserialize_with = "deserialize_environment")] - pub environment: BTreeMap, - #[serde(default)] - pub environment_remove: Vec, - #[serde(default)] - pub environment_replace: bool, - #[serde(default)] - pub expected_module_version: Option, -} +use http::HeaderValue; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentMap, EnvironmentRemove}; +use spacetimedb_lib::Hash; -/// Authorized environment metadata. Values never leave the database in this response. -#[derive(Clone, Serialize, Deserialize)] -pub struct EnvironmentMetadata { - pub module_version: String, - pub declarations: Vec, - pub stored_keys: Vec, -} +/// Environment values sent with a publish, one `spacetime-environment: KEY=VALUE` header per +/// variable, written like a shell or `.env` assignment. +/// +/// A value is bare, single-quoted or double-quoted. Bare and single-quoted values behave as in +/// a shell. A bare value (`API_KEY=abc123`) is taken literally and cannot contain whitespace, +/// quotes, `,` or `\`. A single-quoted value (`GREETING='hello, world'`) is taken literally up to +/// the closing quote, with no escapes. A double-quoted value can contain anything, using JSON +/// string escapes as `.env` files do: `\"`, `\\`, `\n`, `\t` and `\uXXXX` for other control +/// characters and non-ASCII, which cannot appear raw in a header. Commas outside quotes separate +/// entries, so the list stays unambiguous if a proxy merges repeated headers into one +/// comma-separated line. +/// +/// Headers are subject to size limits that the environment limits are not: proxies often cap a +/// header line at 8 KiB and all request headers at a few tens of KiB, and hyper rejects requests +/// whose headers total more than about 400 KiB. A large environment can therefore fail to +/// publish even though `PUT /environment` accepts it. If users run into this, publish could also +/// accept a `multipart/form-data` body with the module and the environment as separate parts. +/// That would not break the API: the server would choose the format by `Content-Type`, so raw +/// module bodies with this header would keep working unchanged. +#[derive(Default)] +pub struct SpacetimeEnvironment(pub EnvironmentMap); -#[derive(Debug, Clone, Copy, thiserror::Error)] -pub enum PublishRequestError { - #[error("invalid publish request body")] - Invalid, - #[error("publish request exceeds size limit")] - TooLarge, -} +impl headers::Header for SpacetimeEnvironment { + fn name() -> &'static http::HeaderName { + static NAME: http::HeaderName = http::HeaderName::from_static("spacetime-environment"); + &NAME + } -impl PublishRequest { - pub fn decode(body: &[u8]) -> Result { - if body.len() > MAX_REQUEST_BYTES { - return Err(PublishRequestError::TooLarge); + fn decode<'i, I>(values: &mut I) -> Result + where + Self: Sized, + I: Iterator, + { + let err = headers::Error::invalid; + let mut entries = BTreeMap::new(); + for value in values { + // `to_str` rejects control characters (other than tab) and non-ASCII. + let value = value.to_str().map_err(|_| err())?; + for (key, value) in parse_assignments(value).ok_or_else(err)? { + validate_key(key).map_err(|_| err())?; + validate_value(&value).map_err(|_| err())?; + if entries.insert(key.to_owned(), value).is_some() { + return Err(err()); + } + } } - // Never expose serde's error text: it can quote a supplied secret. - let request: Self = serde_json::from_slice(body).map_err(|_| PublishRequestError::Invalid)?; - request.validate()?; - Ok(request) + Ok(Self(entries)) } - pub fn encode(&self) -> Result, PublishRequestError> { - self.validate()?; - serde_json::to_vec(self).map_err(|_| PublishRequestError::Invalid) + fn encode>(&self, values: &mut E) { + values.extend(self.0.iter().map(|(key, value)| { + let mut header = if value.bytes().all(is_bare_value_byte) { + HeaderValue::try_from(format!("{key}={value}")) + } else { + HeaderValue::try_from(format!("{key}={}", quote_value(value))) + } + // A valid key, a bare value and a quoted value are all visible ASCII. + .unwrap(); + header.set_sensitive(true); + header + })); } +} - fn validate(&self) -> Result<(), PublishRequestError> { - if self - .module - .as_ref() - .is_some_and(|module| module.len() > MAX_MODULE_BYTES) - || self.environment.len() > MAX_ENV_VARS - { - return Err(PublishRequestError::TooLarge); - } - spacetimedb_lib::environment::EnvironmentUpdate { - values: self.environment.clone(), - remove: self.environment_remove.clone(), - replace: self.environment_replace, +/// Whether `byte` may appear in an unquoted value: visible ASCII other than quotes, `,` and `\`. +fn is_bare_value_byte(byte: u8) -> bool { + byte.is_ascii_graphic() && !matches!(byte, b'"' | b'\'' | b',' | b'\\') +} + +/// Quotes `value` as an ASCII-only JSON string. +fn quote_value(value: &str) -> String { + use std::fmt::Write; + let mut quoted = String::with_capacity(value.len() + 2); + quoted.push('"'); + for c in value.chars() { + match c { + '"' => quoted.push_str("\\\""), + '\\' => quoted.push_str("\\\\"), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + ' '..='~' => quoted.push(c), + _ => { + for unit in c.encode_utf16(&mut [0; 2]) { + let _ = write!(quoted, "\\u{unit:04x}"); + } + } } - .validate() - .map_err(|_| PublishRequestError::Invalid)?; - if self - .expected_module_version - .as_ref() - .is_some_and(|hash| hash.len() != 64 || !hash.bytes().all(|byte| byte.is_ascii_hexdigit())) - { - return Err(PublishRequestError::Invalid); + } + quoted.push('"'); + quoted +} + +/// Parses `KEY=VALUE` entries separated by commas, with optional whitespace around entries. +/// Empty entries are ignored, as RFC 9110 requires for list-based headers. +/// Returns `None` if the list is malformed. +fn parse_assignments(mut rest: &str) -> Option> { + let mut entries = Vec::new(); + loop { + rest = rest.trim_start_matches([' ', '\t', ',']); + if rest.is_empty() { + return Some(entries); } - for (key, value) in &self.environment { - validate_key(key).map_err(|_| PublishRequestError::Invalid)?; - validate_value(value).map_err(|_| PublishRequestError::TooLarge)?; + let (key, after_key) = rest.split_once('=')?; + let (value, after_value) = if let Some(quoted) = after_key.strip_prefix('\'') { + let (value, after_quote) = quoted.split_once('\'')?; + (value.to_owned(), after_quote) + } else if after_key.starts_with('"') { + let end = quoted_len(after_key)?; + let value = serde_json::from_str::(&after_key[..end]).ok()?; + (value, &after_key[end..]) + } else { + let end = after_key.find(',').unwrap_or(after_key.len()); + // Trailing whitespace belongs to the list syntax, not the value. + let value = after_key[..end].trim_end_matches([' ', '\t']); + if !value.bytes().all(is_bare_value_byte) { + return None; + } + (value.to_owned(), &after_key[end..]) + }; + entries.push((key, value)); + rest = after_value.trim_start_matches([' ', '\t']); + match rest.strip_prefix(',') { + Some(after_comma) => rest = after_comma, + None if rest.is_empty() => return Some(entries), + None => return None, } - Ok(()) } } -fn deserialize_environment<'de, D: Deserializer<'de>>(de: D) -> Result, D::Error> { - struct Visitor; - impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = BTreeMap; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("a map of supplied environment strings") +/// The length of the quoted string at the start of `s`, including both quotes. +fn quoted_len(s: &str) -> Option { + let mut escaped = false; + for (i, byte) in s.bytes().enumerate().skip(1) { + match byte { + _ if escaped => escaped = false, + b'\\' => escaped = true, + b'"' => return Some(i + 1), + _ => {} } - fn visit_map>(self, mut map: A) -> Result { - use serde::de::Error; - let mut values = BTreeMap::new(); - while let Some(key) = map.next_key::()? { - if values.len() >= MAX_ENV_VARS || validate_key(&key).is_err() || values.contains_key(&key) { - return Err(A::Error::custom("invalid environment keys")); + } + None +} + +/// Environment keys to remove, as a comma-separated list of key names (`API_KEY, _PRIVATE`), +/// or `*` on its own to remove every stored value. +#[derive(Default)] +pub struct SpacetimeEnvironmentRemove(pub EnvironmentRemove); + +impl headers::Header for SpacetimeEnvironmentRemove { + fn name() -> &'static http::HeaderName { + static NAME: http::HeaderName = http::HeaderName::from_static("spacetime-environment-remove"); + &NAME + } + + fn decode<'i, I>(values: &mut I) -> Result + where + Self: Sized, + I: Iterator, + { + let err = headers::Error::invalid; + let mut entries = Vec::new(); + let mut remove_all = false; + for value in values { + let value = value.to_str().map_err(|_| err())?; + for key in value.split(',').map(str::trim).filter(|key| !key.is_empty()) { + // `*` means "remove everything" only when it stands alone. Rejecting it + // alongside keys keeps a typo like `FOO, *` from wiping the environment. + if remove_all || (key == "*" && !entries.is_empty()) { + return Err(err()); } - let value = map.next_value::()?; - if validate_value(&value).is_err() { - return Err(A::Error::custom("environment value exceeds size limit")); + if key == "*" { + remove_all = true; + } else { + validate_key(key).map_err(|_| err())?; + entries.push(key.to_owned()); } - values.insert(key, value); } - Ok(values) + } + if remove_all { + Ok(Self(EnvironmentRemove::All)) + } else if entries.is_empty() { + Ok(Self(EnvironmentRemove::No)) + } else { + Ok(Self(EnvironmentRemove::Keys(entries))) } } - de.deserialize_map(Visitor) + + fn encode>(&self, values: &mut E) { + let header = match &self.0 { + EnvironmentRemove::No => return, + EnvironmentRemove::All => HeaderValue::from_static("*"), + // Valid keys are visible ASCII without commas. + EnvironmentRemove::Keys(keys) => HeaderValue::try_from(keys.join(", ")).unwrap(), + }; + values.extend([header]); + } +} + +/// Authorized environment metadata. Values never leave the database in this response. +#[derive(Clone, Serialize, Deserialize)] +pub struct EnvironmentMetadata { + pub module_hash: Hash, + pub declarations: Vec, + pub stored_keys: Vec, } #[cfg(test)] mod tests { use super::*; + use headers::Header; + + fn encode_environment(entries: &[(&str, &str)]) -> Vec { + let env = entries.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + let mut values = Vec::new(); + SpacetimeEnvironment(env).encode(&mut values); + values.iter().map(|v| v.to_str().unwrap().to_owned()).collect() + } + + fn decode_environment(values: &[&str]) -> Result { + let values: Vec = values.iter().map(|v| HeaderValue::try_from(*v).unwrap()).collect(); + SpacetimeEnvironment::decode(&mut values.iter()).map(|header| header.0) + } + #[test] - fn roundtrip_and_omission_preserve_complete_string_input() { - let request = PublishRequest { - module: Some(vec![0, 1, 255]), - environment: BTreeMap::from([("EMPTY".into(), "".into()), ("TOKEN".into(), "雪\0false".into())]), - ..Default::default() - }; - let decoded = PublishRequest::decode(&request.encode().unwrap()).unwrap(); - assert_eq!(decoded.module, request.module); - assert_eq!(decoded.environment, request.environment); - assert!(PublishRequest::decode(br#"{"module":""}"#) - .unwrap() - .environment - .is_empty()); + fn environment_header_is_one_shell_style_line_per_variable() { + let entries = [ + ("API_KEY", "abc123"), + ("URL", "https://x.io/a?b=1&c=%20"), + ("GREETING", "h\u{e9}llo, \"world\" \\ \u{1f600}"), + ("PEM", "line1\nline2\0\x7f"), + ("_PADDED", " padded "), + ("EMPTY", ""), + ("QUOTE", "it's"), + ]; + let encoded = encode_environment(&entries); + assert_eq!( + encoded, + [ + "API_KEY=abc123", + "EMPTY=", + r#"GREETING="h\u00e9llo, \"world\" \\ \ud83d\ude00""#, + r#"PEM="line1\nline2\u0000\u007f""#, + r#"QUOTE="it's""#, + "URL=https://x.io/a?b=1&c=%20", + r#"_PADDED=" padded ""#, + ] + ); + let expected: EnvironmentMap = entries.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + let lines: Vec<&str> = encoded.iter().map(String::as_str).collect(); + assert_eq!(decode_environment(&lines).unwrap(), expected); + // A proxy may merge repeated headers into one comma-separated line. + assert_eq!(decode_environment(&[&lines.join(", ")]).unwrap(), expected); } + #[test] - fn environment_only_mutation_roundtrips_and_rejects_conflicting_operations() { - let request = PublishRequest { - environment: BTreeMap::from([("FUTURE".into(), "secret-marker".into())]), - environment_remove: vec!["OPTIONAL".into()], - expected_module_version: Some("ab".repeat(32)), - ..Default::default() - }; - let bytes = request.encode().unwrap(); - assert!(serde_json::from_slice::(&bytes) - .unwrap() - .get("module") - .is_none()); - let decoded = PublishRequest::decode(&bytes).unwrap(); - assert_eq!(decoded.environment_remove, request.environment_remove); - assert_eq!(decoded.expected_module_version, request.expected_module_version); - for body in [ - r#"{"environment_replace":true,"environment_remove":["KEY"]}"#, - r#"{"environment":{"KEY":"secret-marker"},"environment_remove":["KEY"]}"#, - r#"{"environment_remove":["KEY","KEY"]}"#, - r#"{"expected_module_version":"secret-marker"}"#, + fn environment_header_accepts_hand_written_assignments() { + let decoded = decode_environment(&[ + r#"A=1,B="two words" , C="x,y",D="""#, + "\tE=5\t", + r#"F='it, "is" \n literal', G=''"#, + ", ,H=8,,", + ]) + .unwrap(); + let expected: EnvironmentMap = [ + ("A", "1"), + ("B", "two words"), + ("C", "x,y"), + ("D", ""), + ("E", "5"), + ("F", r#"it, "is" \n literal"#), + ("G", ""), + ("H", "8"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + assert_eq!(decoded, expected); + } + + #[test] + fn environment_header_rejects_malformed_entries() { + assert_eq!(decode_environment(&[]).unwrap(), EnvironmentMap::new()); + for bad in [ + &["NO_EQUALS"][..], + &["1BAD=x"], + &["BAD-KEY=x"], + &["A =x"], + &["A=1", "A=2"], + &["A=1, A=2"], + &["A=has space"], + &["API_KEY=abc MODE=production"], + &["A=has\"quote"], + &["A=it's"], + &["A='unterminated"], + &["A='x'y"], + &["A=back\\slash"], + &[r#"A="unterminated"#], + &[r#"A="x"y"#], + &[r#"A="bad \q escape""#], + &["A=\"raw\ttab\""], ] { - let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); - assert!(!error.to_string().contains("secret-marker")); + assert!(decode_environment(bad).is_err(), "{bad:?} should be rejected"); } } + fn decode_remove(values: &[&'static str]) -> Result { + let values: Vec = values.iter().map(|v| HeaderValue::from_static(v)).collect(); + SpacetimeEnvironmentRemove::decode(&mut values.iter()).map(|header| header.0) + } + #[test] - fn malformed_inputs_and_duplicate_keys_are_rejected_without_values() { - for body in [ - r#"{"module":"","environment":{"KEY":true}}"#, - r#"{"module":"","environment":{"KEY":null}}"#, - r#"{"module":"","environment":{"KEY":"first","KEY":"secret-marker"}}"#, - r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#, - r#"{"module":"secret-marker"}"#, - r#"{"module":"","unknown":"secret-marker"}"#, + fn remove_all_must_stand_alone() { + assert_eq!(decode_remove(&[]).unwrap(), EnvironmentRemove::No); + assert_eq!(decode_remove(&["*"]).unwrap(), EnvironmentRemove::All); + let keys = EnvironmentRemove::Keys(vec!["FOO".into(), "_bar".into()]); + assert_eq!(decode_remove(&["FOO, _bar"]).unwrap(), keys); + assert_eq!(decode_remove(&["FOO", "_bar"]).unwrap(), keys); + let mut encoded = Vec::new(); + SpacetimeEnvironmentRemove(keys).encode(&mut encoded); + assert_eq!(encoded, [HeaderValue::from_static("FOO, _bar")]); + for bad in [ + &["FOO, *"][..], + &["*, FOO"], + &["*, *"], + &["*", "FOO"], + &["FOO", "*"], + &["\"FOO\""], + &["BAD-KEY"], ] { - let error = PublishRequest::decode(body.as_bytes()).err().expect("must reject"); - assert!(!format!("{error:?}: {error}").contains("secret-marker")); + assert!(decode_remove(bad).is_err(), "{bad:?} should be rejected"); } } } diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index eb383a5f8c9..0480f667a02 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -11,13 +11,15 @@ use http::StatusCode; use spacetimedb::client::ClientActorIndex; use spacetimedb::energy::{EnergyBalance, EnergyQuanta}; +use spacetimedb::host::module_host::UpdateEnvironmentResult; use spacetimedb::host::{HostController, MigratePlanResult, ModuleHost, NoSuchModule, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; use spacetimedb::sql; use spacetimedb_client_api_messages::http::{SqlStmtResult, SqlStmtStats}; use spacetimedb_client_api_messages::name::{DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld}; -use spacetimedb_lib::{ProductTypeElement, ProductValue}; +use spacetimedb_lib::environment::{EnvironmentMap, EnvironmentUpdate}; +use spacetimedb_lib::{Hash, ProductTypeElement, ProductValue}; use spacetimedb_paths::server::ModuleLogsDir; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; use thiserror::Error; @@ -208,18 +210,20 @@ impl Host { program_bytes: Box<[u8]>, policy: MigrationPolicy, environment: spacetimedb_lib::environment::EnvironmentUpdate, - expected_module_version: Option, ) -> anyhow::Result { self.host_controller - .update_module_host( - database, - host_type, - self.replica_id, - program_bytes, - policy, - environment, - expected_module_version, - ) + .update_module_host(database, host_type, self.replica_id, program_bytes, policy, environment) + .await + } + + pub async fn update_environment( + &self, + database: Database, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + self.host_controller + .update_module_environment(database, self.replica_id, environment, expected_module_hash) .await } } @@ -231,11 +235,6 @@ pub struct DatabaseDef { pub database_identity: Identity, /// The compiled program of the database module. pub program_bytes: Bytes, - /// Supplied overrides, never persisted in the public Database record. - pub environment: std::collections::BTreeMap, - pub environment_remove: Vec, - pub environment_replace: bool, - pub expected_module_version: Option, /// The desired number of replicas the database shall have. /// /// If `None`, the edition default is used. @@ -253,9 +252,6 @@ pub struct DatabaseDef { pub struct DatabaseResetDef { pub database_identity: Identity, pub program_bytes: Option, - pub environment: std::collections::BTreeMap, - pub environment_remove: Vec, - pub environment_replace: bool, pub num_replicas: Option, pub host_type: Option, } @@ -328,6 +324,7 @@ pub trait ControlStateWriteAccess: Send + Sync { publisher: &Identity, spec: DatabaseDef, policy: MigrationPolicy, + environment: EnvironmentUpdate, ) -> anyhow::Result>; async fn migrate_plan(&self, spec: DatabaseDef, style: PrettyPrintStyle) -> anyhow::Result; @@ -336,7 +333,12 @@ pub trait ControlStateWriteAccess: Send + Sync { /// Remove all data from a database, and reset it according to the /// given [DatabaseResetDef]. - async fn reset_database(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()>; + async fn reset_database( + &self, + caller_identity: &Identity, + spec: DatabaseResetDef, + environment: EnvironmentMap, + ) -> anyhow::Result<()>; // Energy async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()>; @@ -374,6 +376,14 @@ pub trait ControlStateWriteAccess: Send + Sync { database_identity: &Identity, locked: bool, ) -> anyhow::Result<()>; + + async fn update_environment( + &self, + publisher: &Identity, + database_identity: &Identity, + environment: EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result; } #[async_trait] @@ -442,8 +452,9 @@ impl ControlStateWriteAccess for Arc { identity: &Identity, spec: DatabaseDef, policy: MigrationPolicy, + environment: EnvironmentUpdate, ) -> anyhow::Result> { - (**self).publish_database(identity, spec, policy).await + (**self).publish_database(identity, spec, policy, environment).await } async fn migrate_plan(&self, spec: DatabaseDef, style: PrettyPrintStyle) -> anyhow::Result { @@ -454,8 +465,13 @@ impl ControlStateWriteAccess for Arc { (**self).delete_database(caller_identity, database_identity).await } - async fn reset_database(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { - (**self).reset_database(caller_identity, spec).await + async fn reset_database( + &self, + caller_identity: &Identity, + spec: DatabaseResetDef, + environment: EnvironmentMap, + ) -> anyhow::Result<()> { + (**self).reset_database(caller_identity, spec, environment).await } async fn add_energy(&self, identity: &Identity, amount: EnergyQuanta) -> anyhow::Result<()> { @@ -499,6 +515,18 @@ impl ControlStateWriteAccess for Arc { .set_database_lock(caller_identity, database_identity, locked) .await } + + async fn update_environment( + &self, + publisher: &Identity, + database_identity: &Identity, + environment: EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + (**self) + .update_environment(publisher, database_identity, environment, expected_module_hash) + .await + } } #[async_trait] diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 4e6bb1af152..2b6b5b28708 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1,5 +1,5 @@ -mod publish_environment; -use publish_environment::{ModuleBody, PublishBody}; +use spacetimedb_client_api_messages::publish::{SpacetimeEnvironment, SpacetimeEnvironmentRemove}; +use spacetimedb_lib::environment::{EnvironmentMap, EnvironmentRemove, EnvironmentUpdate}; use std::borrow::Cow; use std::future::Future; @@ -30,10 +30,12 @@ use futures::TryStreamExt; use http::StatusCode; use http_body_util::BodyExt; use log::{debug, info, warn}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use spacetimedb::auth::identity::ConnectionAuthCtx; use spacetimedb::database_logger::DatabaseLogger; -use spacetimedb::host::module_host::{ClientConnectedError, DurabilityExited}; +use spacetimedb::host::module_host::{ + ClientConnectedError, DurabilityExited, DurableOffset, TransactionOffset, UpdateEnvironmentResult, +}; use spacetimedb::host::{CallResult, UpdateDatabaseResult}; use spacetimedb::host::{FunctionArgs, MigratePlanResult}; use spacetimedb::host::{ModuleHost, ReducerOutcome}; @@ -42,8 +44,9 @@ use spacetimedb::identity::Identity; use spacetimedb::messages::control_db::{Database, HostType}; use spacetimedb_client_api_messages::http::SqlStmtResult; use spacetimedb_client_api_messages::name::{ - self, DatabaseName, DomainName, MigrationPolicy, PrePublishAutoMigrateResult, PrePublishManualMigrateResult, - PrePublishResult, PrettyPrintStyle, PublishOp, PublishResult, + self, DatabaseName, DomainName, EnvironmentPublishError, EnvironmentVersionConflict, MigrationPolicy, + PrePublishAutoMigrateResult, PrePublishManualMigrateResult, PrePublishResult, PrettyPrintStyle, PublishOp, + PublishResult, }; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; @@ -579,52 +582,54 @@ impl From for DatabaseResponse { } } -fn environment_validation_error(error: &anyhow::Error) -> Option { - use spacetimedb::db::environment::EnvironmentError; +fn extract_environment_error(mut error: &anyhow::Error) -> Option> { + // TODO(noa): clean up these errors my g-d this is bad use spacetimedb::host::module_host::InitDatabaseError; - use spacetimedb_lib::environment::{validate_key, EnvironmentSchemaError, EnvironmentSchemaErrorKind}; - if let Some(InitDatabaseError::Other(error)) = error.downcast_ref::() { - return environment_validation_error(error); - } - let error = - error - .downcast_ref::() - .or_else(|| match error.downcast_ref::() { - Some(EnvironmentError::Schema(error)) => Some(error), - _ => None, - })?; - // Only typed host validation can ask the caller for a secret. Never infer - // missing keys from module failures or arbitrary diagnostic text. - if error.kind == EnvironmentSchemaErrorKind::MissingRequired - && let Some(key) = error.key.as_deref().filter(|key| validate_key(key).is_ok()) - { - return Some( - ( - StatusCode::BAD_REQUEST, - axum::Json(serde_json::json!({ - "error": "missing_required_environment", - "key": key, - })), - ) - .into(), - ); + use spacetimedb_lib::environment::EnvironmentSchemaError; + while let Some(InitDatabaseError::Other(e)) = error.downcast_ref::() { + error = e; + } + let res = if let Some(error) = error.chain().find_map(|x| x.downcast_ref::()) { + // Only typed host validation can ask the caller for a secret. Never infer + // missing keys from module failures or arbitrary diagnostic text. + if let EnvironmentSchemaError::MissingRequired { key } = error { + // TODO(noa): in the future, return multiple missing keys at once + Ok(EnvironmentPublishError::MissingRequiredEnvironment { keys: vec![key.into()] }) + } else { + Err((StatusCode::BAD_REQUEST, error.to_string()).into()) + } + } else if let Some(error) = error.downcast_ref::() { + Ok(EnvironmentPublishError::VersionConflict(error.clone())) + } else { + return None; + }; + Some(res) +} + +/// Converts `error` into a response, using `wrap` to put an [`EnvironmentPublishError`] +/// into the body type that the endpoint's clients decode. +fn environment_error_response( + error: anyhow::Error, + wrap: impl FnOnce(EnvironmentPublishError) -> T, + fallback: impl FnOnce(anyhow::Error) -> axum::response::ErrorResponse, +) -> axum::response::ErrorResponse { + match extract_environment_error(&error) { + Some(Ok(err)) => (err.status_code(), axum::Json(wrap(err))).into(), + Some(Err(e)) => e, + None => fallback(error), } - Some((StatusCode::BAD_REQUEST, error.to_string()).into()) +} + +fn migration_error_fallback(error: anyhow::Error) -> axum::response::ErrorResponse { + bad_request(format!("Failed to create or update the database: {error}").into()) } fn publish_error(error: anyhow::Error) -> axum::response::ErrorResponse { - if let Some(response) = environment_validation_error(&error) { - return response; - } - if let Some(error) = error.downcast_ref::() { - return (StatusCode::CONFLICT, error.to_string()).into(); - } - log_and_500(error) + environment_error_response(error, PublishResult::EnvironmentError, log_and_500) } fn publish_migration_error(error: anyhow::Error) -> axum::response::ErrorResponse { - environment_validation_error(&error) - .unwrap_or_else(|| bad_request(format!("Failed to create or update the database: {error}").into())) + environment_error_response(error, PublishResult::EnvironmentError, migration_error_fallback) } pub async fn environment_metadata( @@ -642,6 +647,107 @@ where Ok(([(http::header::CACHE_CONTROL, "no-store")], axum::Json(metadata))) } +#[derive(Deserialize)] +pub struct EnvironmentUpdateQueryParams { + expected_module_hash: Hash, +} + +pub async fn environment_set( + State(ctx): State, + Extension(ResolvedDatabase(database)): Extension, + Extension(auth): Extension, + Query(EnvironmentUpdateQueryParams { expected_module_hash }): Query, + axum::Json(values): axum::Json, +) -> EnvironmentPublishResult +where + S: ControlStateDelegate + NodeDelegate + Authorization, +{ + ctx.authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) + .await?; + let remove = EnvironmentRemove::All; + let update = EnvironmentUpdate { values, remove }; + update + .validate() + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; + let result = ctx + .update_environment( + &auth.claims.identity, + &database.database_identity, + update, + expected_module_hash, + ) + .await; + + environment_publish_result(result).await +} + +pub async fn environment_patch( + State(ctx): State, + Extension(ResolvedDatabase(database)): Extension, + Extension(auth): Extension, + Query(EnvironmentUpdateQueryParams { expected_module_hash }): Query, + TypedHeader(SpacetimeEnvironmentRemove(remove)): TypedHeader, + axum::Json(values): axum::Json, +) -> EnvironmentPublishResult +where + S: ControlStateDelegate + NodeDelegate + Authorization, +{ + if remove == EnvironmentRemove::All { + return Err(bad_request( + format!( + "`{}: *` is not supported by PATCH. To replace the whole environment, use PUT instead.", + ::name() + ) + .into(), + )); + } + ctx.authorize_action(auth.claims.identity, database.database_identity, Action::UpdateDatabase) + .await?; + let update = EnvironmentUpdate { values, remove }; + update + .validate() + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; + let result = ctx + .update_environment( + &auth.claims.identity, + &database.database_identity, + update, + expected_module_hash, + ) + .await; + + environment_publish_result(result).await +} + +type EnvironmentPublishResult = axum::response::Result<(StatusCode, axum::Json>)>; + +async fn environment_publish_result(result: anyhow::Result) -> EnvironmentPublishResult { + let result = match result { + Ok(UpdateEnvironmentResult::ErrorExecutingMigration(e)) => { + return Err(environment_error_response( + e, + Err::<(), EnvironmentPublishError>, + migration_error_fallback, + )) + } + Ok(UpdateEnvironmentResult::NoUpdateNeeded) => Ok(()), + Ok(UpdateEnvironmentResult::UpdatePerformed { + tx_offset, + durable_offset, + }) => { + confirm_update(tx_offset, durable_offset, default_update_confirmation_timeout()).await?; + Ok(()) + } + Err(e) => match extract_environment_error(&e) { + Some(Ok(e)) => Err(e), + Some(Err(e)) => return Err(e), + None => return Err(log_and_500(e)), + }, + }; + let status = result.as_ref().err().map_or(StatusCode::OK, |e| e.status_code()); + Ok((status, axum::Json(result))) +} + pub async fn db_info( Extension(ResolvedDatabase(database)): Extension, ) -> axum::response::Result { @@ -901,21 +1007,9 @@ pub async fn reset( host_type, }): Query, Extension(auth): Extension, - PublishBody { - program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, - environment_only, - }: PublishBody, + TypedHeader(SpacetimeEnvironment(environment)): TypedHeader, + program_bytes: Bytes, ) -> axum::response::Result> { - if expected_module_version.is_some() { - return Err(bad_request( - "expected_module_version is not supported for database reset".into(), - )); - } - let _ = environment_only; let database_identity = database.database_identity; ctx.authorize_action(auth.claims.identity, database.database_identity, Action::ResetDatabase) @@ -934,13 +1028,12 @@ pub async fn reset( &auth.claims.identity, DatabaseResetDef { database_identity, - program_bytes, - environment, - environment_remove, - environment_replace, + // An empty body resets the database but keeps the installed module. + program_bytes: (!program_bytes.is_empty()).then_some(program_bytes), num_replicas, host_type: Some(host_type), }, + environment, ) .await .map_err(publish_error)?; @@ -1014,31 +1107,11 @@ pub async fn publish( organization, update_confirmation_timeout: confirmation_timeout, }): Query, + TypedHeader(SpacetimeEnvironment(environment)): TypedHeader, + TypedHeader(SpacetimeEnvironmentRemove(environment_remove)): TypedHeader, Extension(auth): Extension, - PublishBody { - program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, - environment_only, - }: PublishBody, + program_bytes: Bytes, ) -> axum::response::Result> { - if environment_only && (clear || parent.is_some() || organization.is_some() || num_replicas.is_some()) { - return Err(bad_request( - "environment-only publication cannot change database configuration or reset data".into(), - )); - } - if environment_only && expected_module_version.is_none() { - return Err(bad_request( - "environment-only publication requires expected_module_version".into(), - )); - } - if environment_only && name_or_identity.is_none() { - return Err(bad_request( - "environment-only publication requires an existing database".into(), - )); - } // If `clear`, check that the database exists and delegate to `reset`. // If it doesn't exist, ignore the `clear` parameter. // TODO: Replace with actual redirect at the next possible version bump. @@ -1067,27 +1140,21 @@ pub async fn publish( host_type, }), Extension(auth), - PublishBody { - program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, - environment_only, - }, + TypedHeader(SpacetimeEnvironment(environment)), + program_bytes, ) .await; } } } - let program_bytes = program_bytes.unwrap_or_default(); - let (database_identity, db_name) = if environment_only { - let name = name_or_identity.as_ref().expect("validated existing database name"); - (name.resolve(&ctx).await?, name.name()) - } else { - get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await? - }; + if name_or_identity.is_none() && environment_remove != EnvironmentRemove::No { + return Err(bad_request( + "cannot specify spacetime-environment-remove without an existing database".into(), + )); + } + + let (database_identity, db_name) = get_or_create_identity_and_name(&ctx, &auth, name_or_identity.as_ref()).await?; let maybe_parent_database_identity = match parent.as_ref() { None => None, Some(parent) => parent.resolve(&ctx).await.map(Some)?, @@ -1107,13 +1174,6 @@ pub async fn publish( .get_database_by_identity(&database_identity) .await .map_err(log_and_500)?; - if environment_only && existing.is_none() { - return Err(( - StatusCode::NOT_FOUND, - "environment-only publication requires an existing database", - ) - .into()); - } match existing.as_ref() { None => { allow_creation(&auth)?; @@ -1157,16 +1217,16 @@ pub async fn publish( DatabaseDef { database_identity, program_bytes, - environment, - environment_remove, - environment_replace, - expected_module_version, num_replicas, host_type, parent, organization: maybe_org_identity, }, schema_migration_policy, + EnvironmentUpdate { + values: environment, + remove: environment_remove, + }, ) .await .map_err(publish_error)?; @@ -1194,23 +1254,32 @@ pub async fn publish( durable_offset, }, ) => { - timeout(confirmation_timeout.min(MAX_UPDATE_CONFIRMATION_TIMEOUT), async { - let tx_offset = tx_offset.await?; - if let Some(mut durable_offset) = durable_offset { - durable_offset.wait_for(tx_offset).await?; - } - - Ok::<_, UpdateConfirmationError>(()) - }) - .await - .map_err(Into::into) - .flatten()?; - + confirm_update(tx_offset, durable_offset, confirmation_timeout).await?; Ok(success()) } } } +/// Waits until the update committed at `tx_offset` is durable, so that a crash +/// after we respond cannot lose an update we reported as successful. +async fn confirm_update( + tx_offset: TransactionOffset, + durable_offset: Option, + confirmation_timeout: Duration, +) -> Result<(), UpdateConfirmationError> { + timeout(confirmation_timeout.min(MAX_UPDATE_CONFIRMATION_TIMEOUT), async { + let tx_offset = tx_offset.await?; + if let Some(mut durable_offset) = durable_offset { + durable_offset.wait_for(tx_offset).await?; + } + + Ok::<_, UpdateConfirmationError>(()) + }) + .await + .map_err(Into::into) + .flatten() +} + #[derive(From)] enum UpdateConfirmationError { Cancelled(oneshot::error::RecvError), @@ -1343,7 +1412,7 @@ pub async fn pre_publish Extension(ResolvedDatabase(database)): Extension, Query(PrePublishQueryParams { style, host_type }): Query, Extension(auth): Extension, - ModuleBody(program_bytes): ModuleBody, + program_bytes: Bytes, ) -> axum::response::Result> { let database_identity = database.database_identity; @@ -1362,10 +1431,6 @@ pub async fn pre_publish DatabaseDef { database_identity, program_bytes, - environment: Default::default(), - environment_remove: Default::default(), - environment_replace: false, - expected_module_version: None, num_replicas: None, host_type, parent: None, @@ -1605,7 +1670,12 @@ pub struct DatabaseRoutes { pub call_reducer_procedure_post: MethodRouter, /// GET: /database/:name_or_identity/schema pub schema_get: MethodRouter, + /// GET: /database/:name_or_identity/environment pub environment_get: MethodRouter, + /// PUT: /database/:name_or_identity/environment + pub environment_put: MethodRouter, + /// PATCH: /database/:name_or_identity/environment + pub environment_patch: MethodRouter, /// GET: /database/:name_or_identity/logs pub logs_get: MethodRouter, /// POST: /database/:name_or_identity/sql @@ -1635,7 +1705,7 @@ where S: NodeDelegate + ControlStateDelegate + HasWebSocketOptions + Authorization + Clone + 'static, { fn default() -> Self { - use axum::routing::{any, delete, get, post, put}; + use axum::routing::{any, delete, get, patch, post, put}; Self { root_post: post(publish::), db_put: put(publish::), @@ -1649,6 +1719,8 @@ where call_reducer_procedure_post: post(call::), schema_get: get(schema::), environment_get: get(environment_metadata::), + environment_put: put(environment_set::), + environment_patch: patch(environment_patch::), logs_get: get(logs::), sql_post: post(sql::), mcp_post: post(crate::routes::mcp::mcp::), @@ -1682,6 +1754,8 @@ where .route("/call/:reducer", self.call_reducer_procedure_post) .route("/schema", self.schema_get) .route("/environment", self.environment_get) + .route("/environment", self.environment_put) + .route("/environment", self.environment_patch) .route("/logs", self.logs_get) .route("/sql", self.sql_post) .route("/mcp", self.mcp_post) @@ -1823,11 +1897,13 @@ mod tests { use spacetimedb::auth::token_validation::{TokenSigner, TokenValidationError, TokenValidator}; use spacetimedb::client::ClientActorIndex; use spacetimedb::energy::{EnergyBalance, EnergyQuanta}; + use spacetimedb::host::module_host::UpdateEnvironmentResult; use spacetimedb::identity::AuthCtx; use spacetimedb::messages::control_db::{Database, Node, Replica}; use spacetimedb_client_api_messages::name::{ DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld, }; + use spacetimedb_lib::environment::EnvironmentMap; use spacetimedb_lib::Hash; use spacetimedb_paths::server::ModuleLogsDir; use spacetimedb_paths::FromPathUnchecked; @@ -1838,12 +1914,30 @@ mod tests { use tower::util::ServiceExt; #[tokio::test] - async fn publish_environment_error_identifies_only_typed_missing_required_keys() { - use spacetimedb_lib::environment::{EnvironmentSchemaError, EnvironmentSchemaErrorKind}; - let missing = || EnvironmentSchemaError { - key: Some("API_KEY".into()), - kind: EnvironmentSchemaErrorKind::MissingRequired, + async fn environment_update_reports_success_only_after_confirmation() { + let performed = |tx_offset| { + Ok(UpdateEnvironmentResult::UpdatePerformed { + tx_offset, + durable_offset: None, + }) }; + + let (tx, rx) = oneshot::channel(); + tx.send(7).unwrap(); + let response = environment_publish_result(performed(rx)).await.into_response(); + assert_eq!(response.status(), StatusCode::OK); + + // The transaction never reports its offset, so it cannot be confirmed. + let (tx, rx) = oneshot::channel(); + drop(tx); + let response = environment_publish_result(performed(rx)).await.into_response(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } + + #[tokio::test] + async fn publish_environment_error_identifies_only_typed_missing_required_keys() { + use spacetimedb_lib::environment::EnvironmentSchemaError; + let missing = || EnvironmentSchemaError::MissingRequired { key: "API_KEY".into() }; for response in [ publish_error(anyhow::Error::new(missing()).context("publication failed")), publish_migration_error(spacetimedb::db::environment::EnvironmentError::Schema(missing()).into()), @@ -1864,25 +1958,34 @@ mod tests { assert_eq!(response.status(), StatusCode::BAD_REQUEST); assert_eq!(response.headers()[http::header::CONTENT_TYPE], "application/json"); let body = axum::body::to_bytes(response.into_body(), 1024).await.unwrap(); - assert_eq!( - serde_json::from_slice::(&body).unwrap(), - serde_json::json!({ - "error": "missing_required_environment", "key": "API_KEY", - }) - ); + // The CLI decodes publish and reset responses as `PublishResult`. + let decoded = serde_json::from_slice::(&body).unwrap(); + assert!(matches!( + decoded, + PublishResult::EnvironmentError(EnvironmentPublishError::MissingRequiredEnvironment { keys }) + if keys == ["API_KEY"] + )); + } + // The CLI decodes environment update responses as `Result<(), EnvironmentPublishError>`, + // including when the module fails while applying the new environment. + for result in [ + Ok(UpdateEnvironmentResult::ErrorExecutingMigration( + spacetimedb::db::environment::EnvironmentError::Schema(missing()).into(), + )), + Err(spacetimedb::db::environment::EnvironmentError::Schema(missing()).into()), + ] { + let response = environment_publish_result(result).await.into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = axum::body::to_bytes(response.into_body(), 1024).await.unwrap(); + let decoded = serde_json::from_slice::>(&body).unwrap(); + assert!(matches!( + decoded, + Err(EnvironmentPublishError::MissingRequiredEnvironment { keys }) if keys == ["API_KEY"] + )); } for error in [ anyhow::anyhow!("environment key API_KEY: required value is missing"), - EnvironmentSchemaError { - key: Some("API_KEY".into()), - kind: EnvironmentSchemaErrorKind::ConstraintMismatch, - } - .into(), - EnvironmentSchemaError { - key: Some("INVALID-KEY".into()), - kind: EnvironmentSchemaErrorKind::MissingRequired, - } - .into(), + EnvironmentSchemaError::ConstraintMismatch { key: "API_KEY".into() }.into(), ] { let response = Err::<(), _>(publish_migration_error(error)).into_response(); assert_eq!(response.status(), StatusCode::BAD_REQUEST); @@ -2137,6 +2240,7 @@ mod tests { _publisher: &Identity, _spec: DatabaseDef, _policy: MigrationPolicy, + _environment: EnvironmentUpdate, ) -> anyhow::Result> { Err(anyhow::anyhow!("unused")) } @@ -2166,7 +2270,12 @@ mod tests { Err(anyhow::anyhow!("unused")) } - async fn reset_database(&self, _caller_identity: &Identity, _spec: DatabaseResetDef) -> anyhow::Result<()> { + async fn reset_database( + &self, + _caller_identity: &Identity, + _spec: DatabaseResetDef, + _environment: EnvironmentMap, + ) -> anyhow::Result<()> { Err(anyhow::anyhow!("unused")) } @@ -2199,6 +2308,16 @@ mod tests { ) -> anyhow::Result { Err(anyhow::anyhow!("unused")) } + + async fn update_environment( + &self, + _publisher: &Identity, + _database_identity: &Identity, + _environment: EnvironmentUpdate, + _expected_module_hash: Hash, + ) -> anyhow::Result { + Err(anyhow::anyhow!("unused")) + } } impl Authorization for DummyState { diff --git a/crates/client-api/src/routes/database/publish_environment.rs b/crates/client-api/src/routes/database/publish_environment.rs deleted file mode 100644 index 86d8d30f9cb..00000000000 --- a/crates/client-api/src/routes/database/publish_environment.rs +++ /dev/null @@ -1,181 +0,0 @@ -//! Bounded publish extraction. Neither errors nor Debug output retain configuration values. -use axum::body::{to_bytes, Bytes}; -use axum::extract::{FromRequest, Request}; -use axum::response::{IntoResponse, Response}; -use http::{header, StatusCode}; -use spacetimedb_client_api_messages::publish::{PublishRequest, CONTENT_TYPE, MAX_MODULE_BYTES, MAX_REQUEST_BYTES}; -use std::collections::BTreeMap; - -pub struct PublishBody { - pub program_bytes: Option, - pub environment: BTreeMap, - pub environment_remove: Vec, - pub environment_replace: bool, - pub expected_module_version: Option, - pub environment_only: bool, -} - -async fn bounded_body(request: Request, limit: usize) -> Result { - if request - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .is_some_and(|len| len > limit as u64) - { - return Err((StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()); - } - to_bytes(request.into_body(), limit) - .await - .map_err(|_| (StatusCode::PAYLOAD_TOO_LARGE, "publish request exceeds size limit").into_response()) -} - -#[async_trait::async_trait] -impl FromRequest for PublishBody { - type Rejection = Response; - - async fn from_request(request: Request, _state: &S) -> Result { - let envelope = request - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| { - value - .split(';') - .next() - .unwrap_or_default() - .trim() - .eq_ignore_ascii_case(CONTENT_TYPE) - }); - let bytes = bounded_body(request, if envelope { MAX_REQUEST_BYTES } else { MAX_MODULE_BYTES }).await?; - if envelope { - let request = PublishRequest::decode(&bytes) - .map_err(|_| (StatusCode::BAD_REQUEST, "invalid publish request body").into_response())?; - if request.module.as_ref().is_some_and(|module| module.is_empty()) { - return Err((StatusCode::BAD_REQUEST, "module artifact must not be empty").into_response()); - } - let environment_only = request.module.is_none(); - Ok(Self { - program_bytes: request.module.map(Into::into), - environment: request.environment, - environment_remove: request.environment_remove, - environment_replace: request.environment_replace, - expected_module_version: request - .expected_module_version - .map(spacetimedb_lib::Hash::from_hex) - .transpose() - .map_err(|_| (StatusCode::BAD_REQUEST, "invalid expected module version").into_response())?, - environment_only, - }) - } else { - // An empty legacy reset body retains the program; reset clears all data. - Ok(Self { - program_bytes: (!bytes.is_empty()).then_some(bytes), - environment: BTreeMap::new(), - environment_remove: Vec::new(), - environment_replace: false, - expected_module_version: None, - environment_only: false, - }) - } - } -} - -pub struct ModuleBody(pub Bytes); - -#[async_trait::async_trait] -impl FromRequest for ModuleBody { - type Rejection = Response; - - async fn from_request(request: Request, _state: &S) -> Result { - bounded_body(request, MAX_MODULE_BYTES).await.map(Self) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::body::Body; - - #[tokio::test] - async fn legacy_envelope_and_empty_reset_have_complete_input_semantics() { - let legacy = PublishBody::from_request(Request::new(Body::from("module")), &()) - .await - .unwrap(); - assert_eq!(legacy.program_bytes.unwrap(), "module"); - assert!(legacy.environment.is_empty()); - let empty = PublishBody::from_request(Request::new(Body::empty()), &()) - .await - .unwrap(); - assert!(empty.program_bytes.is_none()); - assert!(empty.environment.is_empty()); - let input = PublishRequest { - module: Some(vec![1, 2, 3]), - environment: BTreeMap::from([("TOKEN".into(), "雪\0".into())]), - ..Default::default() - }; - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(input.encode().unwrap())) - .unwrap(); - let extracted = PublishBody::from_request(request, &()).await.unwrap(); - assert_eq!(extracted.environment, input.environment); - assert_eq!(extracted.program_bytes.unwrap(), input.module.unwrap()); - let reset = PublishRequest { - module: None, - environment: input.environment, - ..Default::default() - }; - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(reset.encode().unwrap())) - .unwrap(); - let extracted = PublishBody::from_request(request, &()).await.unwrap(); - assert!(extracted.program_bytes.is_none()); - assert_eq!(extracted.environment, reset.environment); - } - - #[tokio::test] - async fn empty_artifact_is_rejected_but_omission_is_environment_only() { - for (body, accepted) in [(r#"{"module":""}"#, false), (r#"{}"#, true)] { - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(body)) - .unwrap(); - match PublishBody::from_request(request, &()).await { - Ok(body) => { - assert!(accepted); - assert!(body.environment_only); - } - Err(error) => { - assert!(!accepted); - assert_eq!(error.status(), StatusCode::BAD_REQUEST); - } - } - } - } - - #[tokio::test] - async fn streamed_limits_apply_without_global_body_limit_and_errors_are_redacted() { - let stream = futures::stream::iter([ - Ok::<_, std::io::Error>(Bytes::from_static(b"12345")), - Ok(Bytes::from_static(b"67890")), - ]); - let error = bounded_body(Request::new(Body::from_stream(stream)), 8) - .await - .unwrap_err(); - assert_eq!(error.into_response().status(), StatusCode::PAYLOAD_TOO_LARGE); - let request = Request::builder() - .header(header::CONTENT_TYPE, CONTENT_TYPE) - .body(Body::from(r#"{"module":"","environment":{"KEY":["secret-marker"]}}"#)) - .unwrap(); - let error = PublishBody::from_request(request, &()) - .await - .err() - .unwrap() - .into_response(); - assert_eq!(error.status(), StatusCode::BAD_REQUEST); - let body = to_bytes(error.into_body(), 1024).await.unwrap(); - assert!(!String::from_utf8_lossy(&body).contains("secret-marker")); - } -} diff --git a/crates/client-api/src/routes/mod.rs b/crates/client-api/src/routes/mod.rs index 5c487aa5fd9..6999679d803 100644 --- a/crates/client-api/src/routes/mod.rs +++ b/crates/client-api/src/routes/mod.rs @@ -1,5 +1,7 @@ use axum::routing::MethodRouter; +use headers::Header; use http::header; +use spacetimedb_client_api_messages::publish::{SpacetimeEnvironment, SpacetimeEnvironmentRemove}; use tower_http::cors; use crate::{Authorization, ControlStateDelegate, NodeDelegate}; @@ -81,12 +83,55 @@ where .route("/ping", root_routes.ping_get) .merge(extra); - let cors = cors::CorsLayer::new() - .allow_headers([header::AUTHORIZATION, header::ACCEPT, header::CONTENT_TYPE]) - .allow_methods(cors::Any) - .allow_origin(cors::Any); - axum::Router::new() - .nest("/v1", router.layer(cors)) + .nest("/v1", router.layer(cors_layer())) .nest("/internal", internal::router()) } + +/// Browsers may call the `/v1` API from any origin. Requests authenticate with an explicit +/// bearer token rather than cookies, so every header the API reads must be allowed here. +fn cors_layer() -> cors::CorsLayer { + cors::CorsLayer::new() + .allow_headers([ + header::AUTHORIZATION, + header::ACCEPT, + header::CONTENT_TYPE, + SpacetimeEnvironment::name().clone(), + SpacetimeEnvironmentRemove::name().clone(), + ]) + .allow_methods(cors::Any) + .allow_origin(cors::Any) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use http::{Method, Request}; + use tower::ServiceExt; + + #[tokio::test] + async fn cors_preflight_allows_environment_headers() { + let app = axum::Router::new() + .route("/environment", axum::routing::patch(|| async {})) + .layer(cors_layer()); + let request = Request::builder() + .method(Method::OPTIONS) + .uri("/environment") + .header(header::ORIGIN, "https://example.com") + .header(header::ACCESS_CONTROL_REQUEST_METHOD, "PATCH") + .header( + header::ACCESS_CONTROL_REQUEST_HEADERS, + "authorization, content-type, spacetime-environment, spacetime-environment-remove", + ) + .body(Body::empty()) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + let allowed = response.headers()[header::ACCESS_CONTROL_ALLOW_HEADERS] + .to_str() + .unwrap(); + for name in ["spacetime-environment", "spacetime-environment-remove"] { + assert!(allowed.split(',').any(|h| h.trim() == name), "{name} not in {allowed}"); + } + } +} diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs index 5c156ab4ec3..51f194e3f37 100644 --- a/crates/core/src/db/environment.rs +++ b/crates/core/src/db/environment.rs @@ -18,7 +18,7 @@ use std::collections::BTreeMap; pub enum EnvironmentError { #[error(transparent)] Validation(#[from] EnvironmentValidationError), - #[error(transparent)] + #[error("couldn't validate schema")] Schema(#[from] EnvironmentSchemaError), #[error(transparent)] Datastore(#[from] DatastoreError), diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 26fedc29250..55e52350a41 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1,4 +1,6 @@ -use super::module_host::{DurableOffset, EventStatus, InitDatabaseResult, ModuleHost, ModuleInfo, NoSuchModule}; +use super::module_host::{ + DurableOffset, EventStatus, InitDatabaseResult, ModuleHost, ModuleInfo, NoSuchModule, UpdateEnvironmentResult, +}; use super::scheduler::SchedulerStarter; use super::v8::V8HeapMetrics; use super::wasmtime::{WasmMemoryBytesMetric, WasmtimeRuntime}; @@ -31,6 +33,7 @@ use durability::{Durability, EmptyHistory}; use log::{info, trace, warn}; use parking_lot::Mutex; use scopeguard::{defer, guard}; +use spacetimedb_client_api_messages::name::EnvironmentVersionConflict; use spacetimedb_commitlog::SizeOnDisk; use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_data_structures::map::{IntMap, IntSet}; @@ -92,10 +95,6 @@ where pub type ProgramStorage = Arc; -#[derive(Debug, thiserror::Error)] -#[error("database program changed before publication; reload environment metadata and retry")] -pub struct EnvironmentVersionConflict; - /// Private complete configuration for a not-yet-initialized database generation. /// Implementations must verify the exact persisted database identity, program and /// initialization generation. A source resolving the generation at load time must @@ -581,41 +580,16 @@ impl HostController { Ok(info) } - /// Update the [`ModuleHost`] identified by `replica_id` to the given - /// program. - /// - /// The host may not be running, in which case it is spawned (see - /// [`Self::get_or_launch_module_host`] for details on what this entails). - /// - /// If the host was running, and the update fails, the previous version of - /// the host keeps running. - #[tracing::instrument(level = "trace", skip_all, err)] - #[allow(clippy::too_many_arguments)] - pub async fn update_module_host( + async fn update_module_inner( &self, database: Database, - host_type: HostType, replica_id: u64, - program_bytes: Box<[u8]>, - policy: MigrationPolicy, - environment: spacetimedb_lib::environment::EnvironmentUpdate, - expected_module_version: Option, - ) -> anyhow::Result { - environment.validate()?; - let environment_only = program_bytes.is_empty(); - anyhow::ensure!( - !environment_only || expected_module_version.is_some(), - "environment-only publication requires expected_module_version" - ); - let program = Program::from_bytes(host_type.into(), program_bytes); - trace!( - "update module host {}/{}: genesis={} update-to={}", - database.database_identity, - replica_id, - database.initial_program, - program.hash - ); - + f: impl FnOnce(HostController, Host) -> Fut + Send + 'static, + ) -> anyhow::Result + where + Fut: Future)> + Send, + R: Send + 'static, + { let Ok(mut guard) = self.acquire_write_lock(replica_id).await else { bail!("unable to lock database {} for update", database.database_identity); }; @@ -623,7 +597,6 @@ impl HostController { // `HostController::clone` is fast, // as all of its fields are either `Copy` or wrapped in `Arc`. let this = self.clone(); - let database_identity = database.database_identity; // `try_init_host` is not cancel safe, as it will spawn other async tasks // which hold a filesystem lock past when `try_init_host` returns or is cancelled. @@ -640,7 +613,7 @@ impl HostController { // Note that `tokio::spawn` only cancels its tasks when the runtime shuts down, // at which point we won't be calling `try_init_host` again anyways. let update_result = tokio::spawn(async move { - let mut host = match guard.take() { + let host = match guard.take() { None => { trace!("host not running, try_init"); this.try_init_host(database, replica_id).await?.host @@ -650,29 +623,62 @@ impl HostController { host } }; + let (host, update_result) = f(this, host).await; + + // Rejected publication leaves the existing host usable. Restore it + // before propagating validation or migration failure to the caller. + *guard = Some(host); + update_result + }) + .await??; + + Ok(update_result) + } + + /// Update the [`ModuleHost`] identified by `replica_id` to the given + /// program. + /// + /// The host may not be running, in which case it is spawned (see + /// [`Self::get_or_launch_module_host`] for details on what this entails). + /// + /// If the host was running, and the update fails, the previous version of + /// the host keeps running. + #[tracing::instrument(level = "trace", skip_all, err)] + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + ) -> anyhow::Result { + environment.validate()?; + let program = Program::from_bytes(host_type.into(), program_bytes); + trace!( + "update module host {}/{}: genesis={} update-to={}", + database.database_identity, + replica_id, + database.initial_program, + program.hash + ); + + let database_identity = database.database_identity; + + self.update_module_inner(database, replica_id, async move |this, mut host| { let update_result = async { let module = host.module.borrow().clone(); - if let Some(expected) = expected_module_version - && module.info.module_hash != expected - { - return Err(EnvironmentVersionConflict.into()); - } let previous = host .replica_ctx .relational_db() .with_read_only(Workload::Internal, |tx| crate::db::environment::snapshot(tx))?; let environment = environment.resulting_values(&previous)?; - if environment_only || program.hash == module.info.module_hash { + if program.hash == module.info.module_hash { if environment == previous { return Ok(UpdateDatabaseResult::NoUpdateNeeded); } - let program = module - .relational_db() - .program()? - .context("database program is not initialized")?; - return module - .update_database_with_environment(program, module.info.clone(), policy, environment) - .await; + return module.update_environment(environment).await.map(Into::into); } host.update_module( this.runtimes.clone(), @@ -687,14 +693,43 @@ impl HostController { } .await; - // Rejected publication leaves the existing host usable. Restore it - // before propagating validation or migration failure to the caller. - *guard = Some(host); - update_result + (host, update_result) }) - .await??; + .await + } - Ok(update_result) + #[tracing::instrument(level = "trace", skip_all, err)] + #[allow(clippy::too_many_arguments)] + pub async fn update_module_environment( + &self, + database: Database, + replica_id: u64, + environment: spacetimedb_lib::environment::EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + environment.validate()?; + + self.update_module_inner(database, replica_id, async move |_, host| { + let update_result = async { + let module = host.module.borrow().clone(); + if module.info.module_hash != expected_module_hash { + return Err(EnvironmentVersionConflict.into()); + } + let previous = host + .replica_ctx + .relational_db() + .with_read_only(Workload::Internal, |tx| crate::db::environment::snapshot(tx))?; + let environment = environment.resulting_values(&previous)?; + if environment == previous { + return Ok(UpdateEnvironmentResult::NoUpdateNeeded); + } + module.update_environment(environment).await + } + .await; + + (host, update_result) + }) + .await } pub async fn migrate_plan( @@ -821,7 +856,7 @@ impl HostController { crate::db::environment::snapshot(tx).map(|values| values.into_keys().collect()) })?; Ok(spacetimedb_client_api_messages::publish::EnvironmentMetadata { - module_version: module.info.module_hash.to_string(), + module_hash: module.info.module_hash, declarations: module.info.module_def.environment().declarations().cloned().collect(), stored_keys, }) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 76f605d2244..99e6806c236 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1552,15 +1552,16 @@ mod test { } fn bind_test_environment(env: &mut InstanceEnv) -> Result { - use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; - use spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration}; + use spacetimedb_lib::db::raw_def::v10::{ + RawEnvVarTypeV10, RawEnvironmentDeclarationV10, RawModuleDefV10Builder, + }; let mut builder = RawModuleDefV10Builder::new(); builder.add_environment( [("A", false), ("MISSING", true)] .into_iter() - .map(|(name, optional)| EnvironmentDeclaration { + .map(|(name, optional)| RawEnvironmentDeclarationV10 { name: name.into(), - ty: EnvVarType::String, + ty: RawEnvVarTypeV10::String, optional, }) .collect(), diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 01e4dee6309..d0a1f23c560 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -25,10 +25,9 @@ mod wasm_common; pub use disk_storage::DiskStorage; pub use host_controller::{ - extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, EnvironmentVersionConflict, - ExternalDurability, ExternalStorage, HostController, HostRuntimeConfig, InitialEnvironmentSource, - MigratePlanResult, ModuleHostWithBootstrap, ProcedureCallResult, ProgramStorage, ReducerCallResult, - ReducerCallResultWithTxOffset, ReducerOutcome, + extract_schema, BootstrapCompletion, CallProcedureReturn, CallResult, ExternalDurability, ExternalStorage, + HostController, HostRuntimeConfig, InitialEnvironmentSource, MigratePlanResult, ModuleHostWithBootstrap, + ProcedureCallResult, ProgramStorage, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerOutcome, }; pub use module_host::{ InitDatabaseResult, ModuleHost, NoSuchModule, ProcedureCallError, ReducerCallError, UpdateDatabaseResult, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 1e9b27af2e8..167945d03f9 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1598,6 +1598,41 @@ impl UpdateDatabaseResult { } } +#[derive(Debug)] +pub enum UpdateEnvironmentResult { + NoUpdateNeeded, + UpdatePerformed { + /// The transaction offset of the successful database update. + tx_offset: TransactionOffset, + /// The durable transaction offset of the database. + /// `None` if the database is in-memory only. + durable_offset: Option, + }, + ErrorExecutingMigration(anyhow::Error), +} +impl UpdateEnvironmentResult { + /// Check if an environment update was successful. + pub fn was_successful(&self) -> bool { + matches!(self, Self::UpdatePerformed { .. }) + } +} + +impl From for UpdateDatabaseResult { + fn from(value: UpdateEnvironmentResult) -> Self { + match value { + UpdateEnvironmentResult::NoUpdateNeeded => Self::NoUpdateNeeded, + UpdateEnvironmentResult::UpdatePerformed { + tx_offset, + durable_offset, + } => Self::UpdatePerformed { + tx_offset, + durable_offset, + }, + UpdateEnvironmentResult::ErrorExecutingMigration(e) => Self::ErrorExecutingMigration(e), + } + } +} + #[derive(thiserror::Error, Debug)] #[error("no such module")] pub struct NoSuchModule; @@ -3278,6 +3313,19 @@ impl ModuleHost { )? } + pub async fn update_environment( + &self, + environment: std::collections::BTreeMap, + ) -> Result { + call_instance!( + self, + "", + environment, + |environment, inst| inst.update_environment(environment), + |environment, inst| inst.update_environment(environment).await, + )? + } + pub async fn exit(&self) { // As in `Self::marked_closed`, `Relaxed` is sufficient because we're not synchronizing any external state. self.closed.store(true, std::sync::atomic::Ordering::Relaxed); diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 90a83691fd8..2b7bd84cf5f 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -66,7 +66,7 @@ use self::syscall::{ use super::module_common::{build_common_module_from_raw, run_describer, ModuleCommon}; use super::module_host::{ CallHttpHandlerParams, CallProcedureParams, CallReducerParams, InstanceManagerMetrics, ModuleInfo, - ModuleWithInstance, + ModuleWithInstance, UpdateEnvironmentResult, }; use super::UpdateDatabaseResult; use crate::client::{ClientActorId, MeteredUnboundedReceiver, MeteredUnboundedSender}; @@ -489,6 +489,13 @@ impl JsMainInstance { .await } + pub async fn update_environment( + &self, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.request(UpdateEnvironmentRequest { environment }).await + } + pub async fn call_reducer(&self, params: CallReducerParams) -> ReducerCallResult { self.request(CallReducerRequest { params }).await } @@ -633,6 +640,12 @@ js_main_request! { } => "update_database", anyhow::Result, UpdateDatabase } +js_main_request! { + UpdateEnvironmentRequest { + environment: std::collections::BTreeMap, + } => "update_environment", anyhow::Result, UpdateEnvironment +} + js_main_request! { CallReducerRequest { params: CallReducerParams, @@ -817,6 +830,11 @@ enum JsMainWorkerRequest { policy: MigrationPolicy, environment: std::collections::BTreeMap, }, + /// See [`JsMainInstance::update_environment`]. + UpdateEnvironment { + reply_tx: JsReplyTx>, + environment: std::collections::BTreeMap, + }, /// See [`JsMainInstance::call_reducer`]. CallReducer { reply_tx: JsReplyTx, @@ -1416,6 +1434,12 @@ fn handle_main_worker_request( let res = instance_common.update_database(program, old_module_info, policy, environment, inst); (res, false) }), + JsMainWorkerRequest::UpdateEnvironment { reply_tx, environment } => { + handle_worker_request("update_environment", reply_tx, || { + let res = instance_common.update_environment(environment, inst); + (res, false) + }) + } JsMainWorkerRequest::CallReducer { reply_tx, params } => { handle_worker_request("call_reducer", reply_tx, || { let mut call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 5ec1fea3047..f0ca26e0da8 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -12,8 +12,8 @@ use crate::host::module_common::{build_common_module_from_raw, ModuleCommon}; use crate::host::module_host::{ call_identity_connected, init_database, CallHttpHandlerParams, CallProcedureParams, CallReducerParams, CallViewParams, ClientConnectedError, DatabaseUpdate, EventStatus, HttpHandlerCallError, InitDatabaseResult, - ModuleEvent, ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, ViewCallResult, - ViewCommand, ViewCommandResult, ViewOutcome, + ModuleEvent, ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, UpdateEnvironmentResult, + ViewCallResult, ViewCommand, ViewCommandResult, ViewOutcome, }; use crate::host::scheduler::{CallScheduledFunctionResult, ScheduledFunctionParams}; use crate::host::{ @@ -505,6 +505,13 @@ impl WasmModuleInstance { .update_database(program, old_module_info, policy, environment, &mut self.instance) } + pub fn update_environment( + &mut self, + environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.common.update_environment(environment, &mut self.instance) + } + pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { let (res, trapped) = self.call_reducer_with_tx(None, params); self.trapped = trapped; @@ -704,7 +711,7 @@ impl InstanceCommon { inst: &mut I, ) -> Result { if program.hash == old_module_info.module_hash { - return self.update_environment(environment, inst); + return self.update_environment(environment, inst).map(Into::into); } let replica_ctx = inst.replica_ctx().clone(); let system_logger = replica_ctx.logger.system_logger(); @@ -821,11 +828,11 @@ impl InstanceCommon { /// Apply an environment publication using the installed module instance. No /// initialization, migration, program replacement, or scheduler restart occurs. - fn update_environment( + pub(crate) fn update_environment( &mut self, environment: std::collections::BTreeMap, inst: &mut I, - ) -> anyhow::Result { + ) -> anyhow::Result { let replica_ctx = inst.replica_ctx().clone(); let db = replica_ctx.relational_db(); let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); @@ -846,7 +853,7 @@ impl InstanceCommon { if trapped || out.outcome != ViewOutcome::Success { let (_, metrics, reducer) = db.rollback_mut_tx(out.tx); db.report_mut_tx_metrics(reducer, metrics, None); - return Ok(UpdateDatabaseResult::ErrorExecutingMigration(anyhow::anyhow!( + return Ok(UpdateEnvironmentResult::ErrorExecutingMigration(anyhow::anyhow!( "view evaluation failed during environment publication" ))); } @@ -865,7 +872,7 @@ impl InstanceCommon { let durable_offset = db.durable_tx_offset(); let CommitAndBroadcastEventSuccess { tx_offset, .. } = commit_and_broadcast_event(&self.info.subscriptions, None, event, out.tx); - Ok(UpdateDatabaseResult::UpdatePerformed { + Ok(UpdateEnvironmentResult::UpdatePerformed { tx_offset, durable_offset, }) @@ -2147,10 +2154,9 @@ mod tests { use spacetimedb_datastore::system_tables::{ModuleKind, ST_ENV_ID}; use spacetimedb_datastore::traits::Program; use spacetimedb_lib::db::raw_def::{ - v10::{RawModuleDefV10Builder, RawModuleDefV10Section}, + v10::{RawEnvVarTypeV10, RawEnvironmentDeclarationV10, RawModuleDefV10Builder, RawModuleDefV10Section}, v9::TableAccess, }; - use spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration}; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_schema::auto_migrate::ponder_migrate; use std::collections::BTreeMap; @@ -2182,12 +2188,13 @@ mod tests { .finish(); } let mut raw = builder.finish(); - raw.sections - .push(RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { + raw.sections.push(RawModuleDefV10Section::Environment(vec![ + RawEnvironmentDeclarationV10 { name: "TOKEN".into(), - ty: EnvVarType::String, + ty: RawEnvVarTypeV10::String, optional: false, - }])); + }, + ])); raw.try_into().expect("valid ENV view module") } diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index b07774b3ee6..c4b3088d2ec 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -18,9 +18,9 @@ required-features = ["serde"] default = ["serde", "metrics_impls"] serde = ["dep:serde", "spacetimedb-sats/serde"] # Allows using `Arbitrary` impls defined in this crate. -proptest = ["dep:proptest", "dep:proptest-derive"] +proptest = ["dep:proptest", "dep:proptest-derive", "spacetimedb-sats/proptest"] # Allows using additional test methods. -test = ["proptest", "spacetimedb-sats/test"] +test = ["spacetimedb-sats/test"] metrics_impls = ["dep:spacetimedb-metrics", "spacetimedb-sats/metrics_impls"] enum-map = ["dep:enum-map"] # Gated to avoid including this in `spacetimedb_bindings`. diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 5f966da8276..580baa1b41e 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -103,7 +103,7 @@ pub enum RawModuleDefV10Section { Submodules(Vec), /// Declared publish-only configuration. Even an empty section requires ENV support. - Environment(Vec), + Environment(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -144,6 +144,24 @@ pub struct RawSubmoduleV10 { pub module: RawModuleDefV10, } +#[derive(Debug, Clone, crate::SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub enum RawEnvVarTypeV10 { + String, + StringLiteral(String), + Union(Vec), +} + +#[derive(Debug, Clone, crate::SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawEnvironmentDeclarationV10 { + pub name: String, + pub ty: RawEnvVarTypeV10, + pub optional: bool, +} + #[derive(Debug, Clone, Copy, Default, SpacetimeType)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] #[sats(crate = crate)] @@ -733,27 +751,6 @@ impl RawModuleDefV10Builder { Default::default() } - /// Declare a complete environment schema, including an explicit empty schema. - /// Repeated calls remain repeated sections so host validation rejects ambiguity. - pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { - self.module - .sections - .push(RawModuleDefV10Section::Environment(declarations)); - self - } - - /// New ENV-aware bindings declare an empty schema when no declaration is registered. - pub fn ensure_environment(&mut self) { - if !self - .module - .sections - .iter() - .any(|section| matches!(section, RawModuleDefV10Section::Environment(_))) - { - self.add_environment(Vec::new()); - } - } - /// Get mutable access to the typespace section, creating it if missing. fn typespace_mut(&mut self) -> &mut Typespace { let idx = self @@ -993,6 +990,26 @@ impl RawModuleDefV10Builder { } } + /// Get mutable access to the environment section, creating it if missing. + fn environment_mut(&mut self) -> &mut Vec { + let idx = self + .module + .sections + .iter() + .position(|s| matches!(s, RawModuleDefV10Section::Environment(_))) + .unwrap_or_else(|| { + self.module + .sections + .push(RawModuleDefV10Section::Environment(Vec::new())); + self.module.sections.len() - 1 + }); + + match &mut self.module.sections[idx] { + RawModuleDefV10Section::Environment(env) => env, + _ => unreachable!("Just ensured Environment section exists"), + } + } + /// Create a table builder. /// /// Does not validate that the product_type_ref is valid; this is left to the module validation code. @@ -1309,6 +1326,12 @@ impl RawModuleDefV10Builder { .push(RawModuleDefV10Section::CaseConversionPolicy(policy)); } + /// Declare a complete environment schema. + pub fn add_environment(&mut self, declarations: Vec) -> &mut Self { + self.environment_mut().extend(declarations); + self + } + /// Finish building, consuming the builder and returning the module. /// The module should be validated before use. pub fn finish(self) -> RawModuleDefV10 { diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs index ad86b19fb58..0e450b75e29 100644 --- a/crates/lib/src/environment.rs +++ b/crates/lib/src/environment.rs @@ -1,5 +1,7 @@ //! Limits shared by the database environment store and its clients. +use crate::db::raw_def::v10::{RawEnvVarTypeV10, RawEnvironmentDeclarationV10}; + pub const MAX_ENV_KEY_BYTES: usize = 256; pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; pub const MAX_ENV_VARS: usize = 256; @@ -18,7 +20,7 @@ impl std::fmt::Display for EnvironmentValidationError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { Self::InvalidKey => "invalid POSIX environment variable name (maximum 256 bytes)", - Self::ValueTooLarge => "environment value exceeds 8192 UTF-8 bytes", + Self::ValueTooLarge => "environment value too large (maximum 8192 bytes)", Self::TooManyVariables => "environment store exceeds 256 variables", }) } @@ -90,103 +92,129 @@ pub struct EnvironmentDeclaration { pub optional: bool, } +impl From for RawEnvironmentDeclarationV10 { + fn from(decl: EnvironmentDeclaration) -> Self { + Self { + name: decl.name, + ty: decl.ty.into(), + optional: decl.optional, + } + } +} + +impl From for RawEnvVarTypeV10 { + fn from(ty: EnvVarType) -> Self { + match ty { + EnvVarType::String => Self::String, + EnvVarType::StringLiteral(s) => Self::StringLiteral(s), + EnvVarType::Union(ss) => Self::Union(ss), + } + } +} + +impl From for EnvironmentDeclaration { + fn from(decl: RawEnvironmentDeclarationV10) -> Self { + Self { + name: decl.name, + ty: decl.ty.into(), + optional: decl.optional, + } + } +} + +impl From for EnvVarType { + fn from(ty: RawEnvVarTypeV10) -> Self { + match ty { + RawEnvVarTypeV10::String => Self::String, + RawEnvVarTypeV10::StringLiteral(s) => Self::StringLiteral(s), + RawEnvVarTypeV10::Union(ss) => Self::Union(ss), + } + } +} + /// An environment schema whose declarations have passed host validation. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct EnvironmentSchema { declarations: std::collections::BTreeMap, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum EnvironmentSchemaErrorKind { +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum EnvironmentSchemaError { InvalidName, TooManyDeclarations, TooManyValues, - ConflictingUpdate, - DuplicateDeclaration, - EmptyUnion, - TooManyUnionEntries, - SchemaTooLarge, - LiteralTooLarge, - Undeclared, - MissingRequired, - ValueTooLarge, - ConstraintMismatch, -} - -/// Errors identify a key and rule, and never contain a supplied or allowed value. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct EnvironmentSchemaError { - pub key: Option, - pub kind: EnvironmentSchemaErrorKind, + ConflictingUpdate { key: String }, + DuplicateDeclaration { key: String }, + EmptyUnion { key: String }, + TooManyUnionEntries { key: String }, + SchemaTooLarge { key: String }, + LiteralTooLarge { key: String }, + MissingRequired { key: String }, + ValueTooLarge { key: String }, + ConstraintMismatch { key: String }, } impl std::fmt::Display for EnvironmentSchemaError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(key) = &self.key { - write!(f, "environment key {key:?}: ")?; - } - f.write_str(match self.kind { - EnvironmentSchemaErrorKind::InvalidName => "invalid name", - EnvironmentSchemaErrorKind::TooManyDeclarations => "too many declarations", - EnvironmentSchemaErrorKind::TooManyValues => "too many stored values", - EnvironmentSchemaErrorKind::ConflictingUpdate => "conflicting environment update operations", - EnvironmentSchemaErrorKind::DuplicateDeclaration => "duplicate declaration", - EnvironmentSchemaErrorKind::EmptyUnion => "string union must not be empty", - EnvironmentSchemaErrorKind::TooManyUnionEntries => "string union has too many entries", - EnvironmentSchemaErrorKind::SchemaTooLarge => "declaration schema exceeds size limit", - EnvironmentSchemaErrorKind::LiteralTooLarge => "declared literal exceeds value size limit", - EnvironmentSchemaErrorKind::Undeclared => "key is not declared", - EnvironmentSchemaErrorKind::MissingRequired => "required value is missing", - EnvironmentSchemaErrorKind::ValueTooLarge => "value exceeds size limit", - EnvironmentSchemaErrorKind::ConstraintMismatch => "value does not satisfy its declared string constraint", - }) + let (key, msg) = match self { + Self::InvalidName => return f.write_str("invalid name"), + Self::TooManyDeclarations => return f.write_str("too many declarations"), + Self::TooManyValues => return f.write_str("too many stored values"), + Self::ConflictingUpdate { key } => (key, "conflicting environment update operations"), + Self::DuplicateDeclaration { key } => (key, "duplicate declaration"), + Self::EmptyUnion { key } => (key, "string union must not be empty"), + Self::TooManyUnionEntries { key } => (key, "string union has too many entries"), + Self::SchemaTooLarge { key } => (key, "declaration schema exceeds size limit"), + Self::LiteralTooLarge { key } => (key, "declared literal exceeds value size limit"), + Self::MissingRequired { key } => (key, "required value is missing"), + Self::ValueTooLarge { key } => (key, "value exceeds size limit"), + Self::ConstraintMismatch { key } => (key, "value does not satisfy its declared string constraint"), + }; + + write!(f, "environment key {key:?}: {msg}") } } impl std::error::Error for EnvironmentSchemaError {} impl EnvironmentSchema { + pub const fn empty() -> Self { + Self { + declarations: std::collections::BTreeMap::new(), + } + } + fn validate_metadata(declarations: &[EnvironmentDeclaration]) -> Result<(), EnvironmentSchemaError> { - use EnvironmentSchemaErrorKind as Kind; if declarations.len() > MAX_ENV_VARS { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::TooManyDeclarations, - }); + return Err(EnvironmentSchemaError::TooManyDeclarations); } let mut bytes = 0usize; for declaration in declarations { // Never retain or format unvalidated key bytes in diagnostics. - validate_key(&declaration.name).map_err(|_| EnvironmentSchemaError { - key: None, - kind: Kind::InvalidName, - })?; - let error = |kind| EnvironmentSchemaError { - key: Some(declaration.name.clone()), - kind, - }; + validate_key(&declaration.name).map_err(|_| EnvironmentSchemaError::InvalidName)?; + let key = &declaration.name; bytes += declaration.name.len(); if bytes > MAX_ENV_SCHEMA_BYTES { - return Err(error(Kind::SchemaTooLarge)); + return Err(EnvironmentSchemaError::SchemaTooLarge { key: key.clone() }); } let literals = match &declaration.ty { EnvVarType::String => &[][..], EnvVarType::StringLiteral(value) => std::slice::from_ref(value), EnvVarType::Union(values) => { if values.is_empty() { - return Err(error(Kind::EmptyUnion)); + return Err(EnvironmentSchemaError::EmptyUnion { key: key.clone() }); } if values.len() > MAX_ENV_UNION_ENTRIES { - return Err(error(Kind::TooManyUnionEntries)); + return Err(EnvironmentSchemaError::TooManyUnionEntries { key: key.clone() }); } values.as_slice() } }; for value in literals { - validate_value(value).map_err(|_| error(Kind::LiteralTooLarge))?; + validate_value(value).map_err(|_| EnvironmentSchemaError::LiteralTooLarge { key: key.clone() })?; bytes += value.len(); if bytes > MAX_ENV_SCHEMA_BYTES { - return Err(error(Kind::SchemaTooLarge)); + return Err(EnvironmentSchemaError::SchemaTooLarge { key: key.clone() }); } } } @@ -205,10 +233,7 @@ impl EnvironmentSchema { values.dedup(); } if schema.declarations.contains_key(&declaration.name) { - return Err(EnvironmentSchemaError { - key: Some(declaration.name), - kind: EnvironmentSchemaErrorKind::DuplicateDeclaration, - }); + return Err(EnvironmentSchemaError::DuplicateDeclaration { key: declaration.name }); } schema.declarations.insert(declaration.name.clone(), declaration); } @@ -216,8 +241,8 @@ impl EnvironmentSchema { } /// Check bounds before cloning raw untrusted metadata into the validated schema. - pub fn from_declarations(declarations: &[EnvironmentDeclaration]) -> Result { - Self::validate_metadata(declarations)?; + pub fn from_declarations(declarations: Vec) -> Result { + Self::validate_metadata(&declarations)?; Self::new(declarations.to_vec()) } @@ -238,37 +263,20 @@ impl EnvironmentSchema { } /// Validate the complete resulting store, including required declarations. - pub fn validate_values( - &self, - values: &std::collections::BTreeMap, - ) -> Result<(), EnvironmentSchemaError> { + pub fn validate_values(&self, values: &EnvironmentMap) -> Result<(), EnvironmentSchemaError> { self.validate_supplied_values(values)?; self.validate_required(values) } /// Validate supplied values without requiring every required key in this input. /// Undeclared values are stored strings but are not readable by module code. - pub fn validate_supplied_values( - &self, - values: &std::collections::BTreeMap, - ) -> Result<(), EnvironmentSchemaError> { - use EnvironmentSchemaErrorKind as Kind; + pub fn validate_supplied_values(&self, values: &EnvironmentMap) -> Result<(), EnvironmentSchemaError> { if values.len() > MAX_ENV_VARS { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::TooManyValues, - }); + return Err(EnvironmentSchemaError::TooManyValues); } for (name, value) in values { - validate_key(name).map_err(|_| EnvironmentSchemaError { - key: None, - kind: Kind::InvalidName, - })?; - let error = |kind| EnvironmentSchemaError { - key: Some(name.clone()), - kind, - }; - validate_value(value).map_err(|_| error(Kind::ValueTooLarge))?; + validate_key(name).map_err(|_| EnvironmentSchemaError::InvalidName)?; + validate_value(value).map_err(|_| EnvironmentSchemaError::ValueTooLarge { key: name.clone() })?; let Some(declaration) = self.get(name) else { continue }; let matches = match &declaration.ty { EnvVarType::String => true, @@ -276,21 +284,17 @@ impl EnvironmentSchema { EnvVarType::Union(allowed) => allowed.binary_search(value).is_ok(), }; if !matches { - return Err(error(Kind::ConstraintMismatch)); + return Err(EnvironmentSchemaError::ConstraintMismatch { key: name.clone() }); } } Ok(()) } - fn validate_required( - &self, - values: &std::collections::BTreeMap, - ) -> Result<(), EnvironmentSchemaError> { + fn validate_required(&self, values: &EnvironmentMap) -> Result<(), EnvironmentSchemaError> { for declaration in self.declarations() { if !declaration.optional && !values.contains_key(&declaration.name) { - return Err(EnvironmentSchemaError { - key: Some(declaration.name.clone()), - kind: EnvironmentSchemaErrorKind::MissingRequired, + return Err(EnvironmentSchemaError::MissingRequired { + key: declaration.name.clone(), }); } } @@ -301,13 +305,23 @@ impl EnvironmentSchema { /// An environment mutation. Deliberately does not implement Debug because values are secrets. #[derive(Clone, Default)] pub struct EnvironmentUpdate { - pub values: std::collections::BTreeMap, - pub remove: Vec, - pub replace: bool, + pub values: EnvironmentMap, + pub remove: EnvironmentRemove, +} + +#[derive(Clone, Default, PartialEq, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum EnvironmentRemove { + #[default] + No, + All, + Keys(Vec), } -impl From> for EnvironmentUpdate { - fn from(values: std::collections::BTreeMap) -> Self { +pub type EnvironmentMap = std::collections::BTreeMap; + +impl From for EnvironmentUpdate { + fn from(values: EnvironmentMap) -> Self { Self { values, ..Self::default() @@ -318,31 +332,17 @@ impl From> for EnvironmentUpdate { impl EnvironmentUpdate { /// Validate operations before any mutation. Diagnostics never include values. pub fn validate(&self) -> Result<(), EnvironmentSchemaError> { - use EnvironmentSchemaErrorKind as Kind; EnvironmentSchema::default().validate_supplied_values(&self.values)?; - if self.remove.len() > MAX_ENV_VARS { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::TooManyValues, - }); - } - if self.replace && !self.remove.is_empty() { - return Err(EnvironmentSchemaError { - key: None, - kind: Kind::ConflictingUpdate, - }); - } - let mut seen = std::collections::BTreeSet::new(); - for key in &self.remove { - validate_key(key).map_err(|_| EnvironmentSchemaError { - key: None, - kind: Kind::InvalidName, - })?; - if self.values.contains_key(key) || !seen.insert(key) { - return Err(EnvironmentSchemaError { - key: Some(key.clone()), - kind: Kind::ConflictingUpdate, - }); + if let EnvironmentRemove::Keys(remove) = &self.remove { + if remove.len() > MAX_ENV_VARS { + return Err(EnvironmentSchemaError::TooManyValues); + } + let mut seen = std::collections::BTreeSet::new(); + for key in remove { + validate_key(key).map_err(|_| EnvironmentSchemaError::InvalidName)?; + if self.values.contains_key(key) || !seen.insert(key) { + return Err(EnvironmentSchemaError::ConflictingUpdate { key: key.clone() }); + } } } Ok(()) @@ -350,19 +350,19 @@ impl EnvironmentUpdate { /// Resolve the new store without mutating the old store. Schema validation follows /// against the deployed schema or the schema from the proposed new module. - pub fn resulting_values( - &self, - previous: &std::collections::BTreeMap, - ) -> Result, EnvironmentSchemaError> { + pub fn resulting_values(&self, previous: &EnvironmentMap) -> Result { self.validate()?; - let mut values = if self.replace { - Default::default() - } else { - previous.clone() + let mut values = match &self.remove { + EnvironmentRemove::No => previous.clone(), + EnvironmentRemove::All => EnvironmentMap::default(), + EnvironmentRemove::Keys(remove) => { + let mut values = previous.clone(); + for key in remove { + values.remove(key); + } + values + } }; - for key in &self.remove { - values.remove(key); - } values.extend(self.values.clone()); EnvironmentSchema::default().validate_supplied_values(&values)?; Ok(values) @@ -372,6 +372,7 @@ impl EnvironmentUpdate { #[cfg(test)] mod schema_tests { use super::*; + use std::assert_matches; use std::collections::BTreeMap; fn declaration(name: &str, ty: EnvVarType, optional: bool) -> EnvironmentDeclaration { @@ -399,18 +400,18 @@ mod schema_tests { values.insert("OPTIONAL".into(), "".into()); schema.validate_values(&values).unwrap(); values.insert("MODE".into(), "False".into()); - assert_eq!( - schema.validate_values(&values).unwrap_err().kind, - EnvironmentSchemaErrorKind::ConstraintMismatch + assert_matches!( + schema.validate_values(&values).unwrap_err(), + EnvironmentSchemaError::ConstraintMismatch { .. } ); values.remove("MODE"); - assert_eq!( - schema.validate_values(&values).unwrap_err().kind, - EnvironmentSchemaErrorKind::MissingRequired + assert_matches!( + schema.validate_values(&values).unwrap_err(), + EnvironmentSchemaError::MissingRequired { .. } ); values.insert("UNDECLARED".into(), "secret-marker".into()); let error = schema.validate_values(&values).unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::MissingRequired); + assert_matches!(error, EnvironmentSchemaError::MissingRequired { .. }); assert!(!format!("{error:?}: {error}").contains("secret-marker")); } @@ -427,19 +428,18 @@ mod schema_tests { assert_eq!(unchanged, old); schema.validate_values(&unchanged).unwrap(); let removed = EnvironmentUpdate { - remove: vec!["REQUIRED".into()], + remove: EnvironmentRemove::Keys(vec!["REQUIRED".into()]), ..Default::default() } .resulting_values(&old) .unwrap(); - assert_eq!( - schema.validate_values(&removed).unwrap_err().kind, - EnvironmentSchemaErrorKind::MissingRequired + assert_matches!( + schema.validate_values(&removed).unwrap_err(), + EnvironmentSchemaError::MissingRequired { .. } ); let replace = EnvironmentUpdate { - replace: true, + remove: EnvironmentRemove::All, values: BTreeMap::from([("REQUIRED".into(), "ready".into())]), - ..Default::default() } .resulting_values(&old) .unwrap(); @@ -451,9 +451,9 @@ mod schema_tests { false, )]) .unwrap(); - assert_eq!( - newly_declared.validate_values(&old).unwrap_err().kind, - EnvironmentSchemaErrorKind::ConstraintMismatch + assert_matches!( + newly_declared.validate_values(&old).unwrap_err(), + EnvironmentSchemaError::ConstraintMismatch { .. } ); let corrected = EnvironmentUpdate::from(BTreeMap::from([("UNUSED".into(), "other".into())])) .resulting_values(&old) @@ -465,31 +465,24 @@ mod schema_tests { #[test] fn mutation_conflicts_and_resulting_store_limit_are_rejected() { let stored = (0..MAX_ENV_VARS).map(|i| (format!("KEY{i}"), String::new())).collect(); - assert_eq!( + assert_matches!( EnvironmentUpdate::from(BTreeMap::from([("EXTRA".into(), String::new())])) .resulting_values(&stored) - .unwrap_err() - .kind, - EnvironmentSchemaErrorKind::TooManyValues + .unwrap_err(), + EnvironmentSchemaError::TooManyValues ); for update in [ - EnvironmentUpdate { - replace: true, - remove: vec!["KEY".into()], - ..Default::default() - }, EnvironmentUpdate { values: BTreeMap::from([("KEY".into(), "secret-marker".into())]), - remove: vec!["KEY".into()], - ..Default::default() + remove: EnvironmentRemove::Keys(vec!["KEY".into()]), }, EnvironmentUpdate { - remove: vec!["KEY".into(), "KEY".into()], + remove: EnvironmentRemove::Keys(vec!["KEY".into(), "KEY".into()]), ..Default::default() }, ] { let error = update.validate().unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::ConflictingUpdate); + assert_matches!(error, EnvironmentSchemaError::ConflictingUpdate { .. }); assert!(!error.to_string().contains("secret-marker")); } } @@ -497,35 +490,34 @@ mod schema_tests { #[test] fn declaration_limits_count_absent_optionals_and_reject_invalid_metadata() { let optional = declaration("A", EnvVarType::String, true); - assert_eq!( - EnvironmentSchema::new(vec![optional.clone(), optional]) - .unwrap_err() - .kind, - EnvironmentSchemaErrorKind::DuplicateDeclaration + assert_matches!( + EnvironmentSchema::new(vec![optional.clone(), optional]).unwrap_err(), + EnvironmentSchemaError::DuplicateDeclaration { .. } ); - assert_eq!( + assert_matches!( EnvironmentSchema::new( (0..=MAX_ENV_VARS) .map(|i| declaration(&format!("K{i}"), EnvVarType::String, true)) .collect() ) - .unwrap_err() - .kind, - EnvironmentSchemaErrorKind::TooManyDeclarations + .unwrap_err(), + EnvironmentSchemaError::TooManyDeclarations ); for (name, constraint, expected) in [ - ("A-B", EnvVarType::String, EnvironmentSchemaErrorKind::InvalidName), - ("A", EnvVarType::Union(vec![]), EnvironmentSchemaErrorKind::EmptyUnion), + ("A-B", EnvVarType::String, EnvironmentSchemaError::InvalidName), + ( + "A", + EnvVarType::Union(vec![]), + EnvironmentSchemaError::EmptyUnion { key: "A".into() }, + ), ( "A", EnvVarType::StringLiteral("x".repeat(MAX_ENV_VALUE_BYTES + 1)), - EnvironmentSchemaErrorKind::LiteralTooLarge, + EnvironmentSchemaError::LiteralTooLarge { key: "A".into() }, ), ] { assert_eq!( - EnvironmentSchema::new(vec![declaration(name, constraint, true)]) - .unwrap_err() - .kind, + EnvironmentSchema::new(vec![declaration(name, constraint, true)]).unwrap_err(), expected ); } @@ -539,7 +531,6 @@ mod schema_tests { fn raw_metadata_is_bounded_before_copying_or_formatting_untrusted_keys() { let invalid = format!("private-marker\n{}", "x".repeat(100_000)); let error = EnvironmentSchema::new(vec![declaration(&invalid, EnvVarType::String, true)]).unwrap_err(); - assert_eq!(error.key, None); assert!(!format!("{error:?}: {error}").contains("private-marker")); let error = EnvironmentSchema::new(vec![declaration( "A", @@ -547,13 +538,13 @@ mod schema_tests { true, )]) .unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::TooManyUnionEntries); + assert_matches!(error, EnvironmentSchemaError::TooManyUnionEntries { .. }); let error = EnvironmentSchema::new(vec![declaration( "A", EnvVarType::Union(vec!["x".repeat(MAX_ENV_VALUE_BYTES); MAX_ENV_UNION_ENTRIES]), true, )]) .unwrap_err(); - assert_eq!(error.kind, EnvironmentSchemaErrorKind::SchemaTooLarge); + assert_matches!(error, EnvironmentSchemaError::SchemaTooLarge { .. }); } } diff --git a/crates/sats/Cargo.toml b/crates/sats/Cargo.toml index ff1ac21eb16..521addc23c6 100644 --- a/crates/sats/Cargo.toml +++ b/crates/sats/Cargo.toml @@ -13,7 +13,7 @@ serde = ["dep:serde"] # Allows using `Arbitrary` impls defined in this crate. proptest = ["dep:proptest", "dep:proptest-derive"] # Allows using additional test-only methods defined in this crate. -test = ["proptest"] +test = [] # Impls `Serialize` and `Deserialize` for `blake3::Hash`. # Used by `spacetimedb_table`, # which serializes and deserializes Blake3 hashes during snapshotting. diff --git a/crates/schema/Cargo.toml b/crates/schema/Cargo.toml index b79cc0a0974..f2b1e993da2 100644 --- a/crates/schema/Cargo.toml +++ b/crates/schema/Cargo.toml @@ -33,7 +33,7 @@ termcolor.workspace = true convert_case.workspace = true [dev-dependencies] -spacetimedb-lib = { path = "../lib", features = ["test"] } +spacetimedb-lib = { path = "../lib", features = ["test", "proptest"] } # these are circular dependencies, but only in tests, so it's fine spacetimedb-testing = { path = "../testing" } diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 674f7d7afb3..87f61de936d 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -18,7 +18,6 @@ use std::collections::BTreeMap; use std::fmt::{self, Debug, Write}; use std::hash::Hash; -use std::sync::LazyLock; use crate::error::{IdentifierError, ValidationErrors}; use crate::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; @@ -200,18 +199,12 @@ pub enum RawModuleDefVersion { impl ModuleDef { /// The validated root environment schema. Legacy modules have an empty schema. pub fn environment(&self) -> &EnvironmentSchema { - static EMPTY: LazyLock = LazyLock::new(EnvironmentSchema::default); match &self.environment { Some(schema) => schema, - None => &EMPTY, + None => const { &EnvironmentSchema::empty() }, } } - /// Whether the raw module explicitly required environment support. - pub fn environment_declared(&self) -> bool { - self.environment.is_some() - } - /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -1116,7 +1109,9 @@ impl From for RawModuleDefV10 { let mut sections = Vec::new(); if let Some(environment) = environment { - sections.push(RawModuleDefV10Section::Environment(environment.into_declarations())); + sections.push(RawModuleDefV10Section::Environment( + environment.into_declarations().into_iter().map(|x| x.into()).collect(), + )); } let mut explicit_names = ExplicitNames::default(); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 6d6f7426668..e7d23b28cd9 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -350,18 +350,15 @@ pub fn validate(def: RawModuleDefV10) -> Result { } fn validate_environment(def: &RawModuleDefV10) -> Result> { - let mut sections = def.sections.iter().filter_map(|section| match section { + let Some(declarations) = def.sections.iter().find_map(|section| match section { RawModuleDefV10Section::Environment(declarations) => Some(declarations), _ => None, - }); - let Some(declarations) = sections.next() else { + }) else { return Ok(None); }; - if sections.next().is_some() { - return Err(ValidationError::RepeatedEnvironmentDeclaration.into()); - } + let declarations = declarations.iter().map(|x| x.clone().into()).collect(); let schema = spacetimedb_lib::environment::EnvironmentSchema::from_declarations(declarations) - .map_err(|error| ValidationError::Environment { error })?; + .map_err(ValidationError::from)?; Ok(Some(schema)) } @@ -3064,33 +3061,21 @@ mod tests { #[cfg(test)] mod environment_tests { use super::*; - use spacetimedb_lib::environment::{EnvVarType, EnvironmentDeclaration}; fn declared(name: &str) -> RawModuleDefV10 { RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(vec![EnvironmentDeclaration { - name: name.into(), - ty: EnvVarType::String, - optional: true, - }])], + sections: vec![RawModuleDefV10Section::Environment(vec![ + RawEnvironmentDeclarationV10 { + name: name.into(), + ty: RawEnvVarTypeV10::String, + optional: true, + }, + ])], } } #[test] - fn environment_schema_round_trip_preserves_explicit_empty_and_exact_keys() { - let legacy = validate(RawModuleDefV10::default()).unwrap(); - assert!(legacy.environment().is_empty()); - assert!(!legacy.environment_declared()); - let raw: RawModuleDefV10 = legacy.into(); - assert!(!validate(raw).unwrap().environment_declared()); - let explicit = validate(RawModuleDefV10 { - sections: vec![RawModuleDefV10Section::Environment(vec![])], - }) - .unwrap(); - assert!(explicit.environment().is_empty()); - assert!(explicit.environment_declared()); - let raw: RawModuleDefV10 = explicit.into(); - assert!(validate(raw).unwrap().environment_declared()); + fn environment_schema_round_trip_preserves_exact_keys() { let module = validate(declared("Mixed_CASE")).unwrap(); assert!(module.environment().get("Mixed_CASE").is_some()); assert!(module.environment().get("mixed_case").is_none()); @@ -3104,12 +3089,6 @@ mod environment_tests { #[test] fn environment_rejects_ambiguous_sections_and_nested_declarations() { - let mut duplicate = declared("A"); - duplicate.sections.push(RawModuleDefV10Section::Environment(vec![])); - assert!(validate(duplicate) - .unwrap_err() - .into_iter() - .any(|error| matches!(error, ValidationError::RepeatedEnvironmentDeclaration))); let nested = RawModuleDefV10 { sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { namespace: "outer".into(), diff --git a/crates/schema/src/describe.rs b/crates/schema/src/describe.rs index ead8deb4ed2..c4adeeb005b 100644 --- a/crates/schema/src/describe.rs +++ b/crates/schema/src/describe.rs @@ -992,8 +992,8 @@ mod tests { use super::*; use crate::identifier::Identifier; use spacetimedb_lib::db::raw_def::v10::{ - CaseConversionPolicy, FunctionVisibility as RawFunctionVisibility, RawModuleDefV10Builder, - RawModuleDefV10Section, RawSubmoduleV10, + CaseConversionPolicy, FunctionVisibility as RawFunctionVisibility, RawEnvVarTypeV10, + RawEnvironmentDeclarationV10, RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10, }; use spacetimedb_lib::db::raw_def::v9::{btree, direct, hash}; use spacetimedb_lib::{ProductType, ScheduleAt}; @@ -1364,16 +1364,16 @@ mod tests { // Declared after `/webhook`, so the output shows declaration order is kept. builder.add_http_route("health", MethodOrAny::Any, "/health"); builder.add_environment(vec![ - env_declaration("API_KEY", EnvVarType::String, false), + env_declaration("API_KEY", RawEnvVarTypeV10::String, false), env_declaration( "MODE", - EnvVarType::Union(vec!["production".into(), "development".into()]), + RawEnvVarTypeV10::Union(vec!["production".into(), "development".into()]), false, ), - env_declaration("REGION", EnvVarType::StringLiteral("eu west".into()), true), + env_declaration("REGION", RawEnvVarTypeV10::StringLiteral("eu west".into()), true), env_declaration( "LOG_LEVEL", - EnvVarType::Union(vec!["info".into(), "debug".into()]), + RawEnvVarTypeV10::Union(vec!["info".into(), "debug".into()]), true, ), ]); @@ -1414,8 +1414,8 @@ mod tests { .expect("the describe fixture should be a valid module definition") } - fn env_declaration(name: &str, ty: EnvVarType, optional: bool) -> EnvironmentDeclaration { - EnvironmentDeclaration { + fn env_declaration(name: &str, ty: RawEnvVarTypeV10, optional: bool) -> RawEnvironmentDeclarationV10 { + RawEnvironmentDeclarationV10 { name: name.into(), ty, optional, diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index eb16c0d2a99..450d4590708 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,14 +22,6 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { - #[error("module has repeated environment declarations")] - RepeatedEnvironmentDeclaration, - #[error("invalid environment declaration: {error}")] - Environment { - error: spacetimedb_lib::environment::EnvironmentSchemaError, - }, - #[error("submodule {namespace:?} cannot declare environment variables")] - EnvironmentInSubmodule { namespace: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] @@ -191,6 +183,10 @@ pub enum ValidationError { LifecycleInSubmodule { lifecycle: Lifecycle, namespace: String }, #[error("submodule namespace `{namespace}` is {len} bytes, which exceeds the 63-byte limit")] NamespaceTooLong { namespace: RawIdentifier, len: usize }, + #[error("invalid environment declaration: {0}")] + Environment(#[from] spacetimedb_lib::environment::EnvironmentSchemaError), + #[error("submodule {namespace:?} cannot declare environment variables")] + EnvironmentInSubmodule { namespace: String }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/smoketests/tests/standalone/cli/environment.rs b/crates/smoketests/tests/standalone/cli/environment.rs index 8b848ede0a5..ec9ffd9c9d3 100644 --- a/crates/smoketests/tests/standalone/cli/environment.rs +++ b/crates/smoketests/tests/standalone/cli/environment.rs @@ -1,9 +1,10 @@ //! Publish-only environment configuration through the real CLI and local server. use serde_json::{json, Value}; use spacetimedb_guard::ensure_binaries_built; +use spacetimedb_lib::Hash; use spacetimedb_smoketests::{modules, random_string, require_local_server, Smoketest}; use std::{ - fs, + assert_matches, fs, io::{Read as _, Seek as _}, path::PathBuf, process::{Child, Command, Output, Stdio}, @@ -474,17 +475,13 @@ fn cli_environment_preservation_and_environment_only_updates() { .timeout(Duration::from_secs(20)) .build() .unwrap(); - let url = format!("{}/v1/database/{}", f.test.server_url, f.database); - let denied = client.get(format!("{url}/environment")).send().unwrap(); - assert!(matches!(denied.status().as_u16(), 401 | 403)); + let url = format!("{}/v1/database/{}/environment", f.test.server_url, f.database); + let denied = client.get(&url).send().unwrap(); + assert_matches!(denied.status().as_u16(), 401 | 403); // This config and token were created by the fixture's isolated local login. let config: toml::Value = toml::from_str(&fs::read_to_string(&f.test.config_path).unwrap()).unwrap(); let token = config["spacetimedb_token"].as_str().unwrap(); - let metadata = client - .get(format!("{url}/environment")) - .bearer_auth(token) - .send() - .unwrap(); + let metadata = client.get(&url).bearer_auth(token).send().unwrap(); assert!(metadata.status().is_success()); assert_eq!(metadata.headers()["cache-control"], "no-store"); let metadata = metadata.text().unwrap(); @@ -494,19 +491,19 @@ fn cli_environment_preservation_and_environment_only_updates() { .as_array() .unwrap() .contains(&json!("SMOKE_REQUIRED"))); - for (body, status) in [ + for (body, query, status) in [ ( - json!({"environment":{"SMOKE_REQUIRED":"stale-replacement"},"expected_module_version":"00".repeat(32)}), + json!({"SMOKE_REQUIRED":"stale-replacement"}), + &[("expected_module_hash", Hash::from_hex("00".repeat(32)).unwrap())][..], 409, ), - (json!({"environment":{"SMOKE_REQUIRED":"missing-version"}}), 400), - (json!({"module":""}), 400), + (json!({"SMOKE_REQUIRED":"missing-version"}), &[], 400), ] { let response = client .put(&url) + .query(query) .bearer_auth(token) - .header("Content-Type", "application/vnd.spacetimedb.publish+json") - .body(body.to_string()) + .json(&body) .send() .unwrap(); assert_eq!(response.status().as_u16(), status); diff --git a/crates/standalone/src/control_db/environment.rs b/crates/standalone/src/control_db/environment.rs index ff15abf2a0a..bad48553a3b 100644 --- a/crates/standalone/src/control_db/environment.rs +++ b/crates/standalone/src/control_db/environment.rs @@ -17,9 +17,8 @@ //! initialization is durable, but is currently retained until reset replaces it //! or database deletion removes it atomically with both indexes and the binding. use super::*; -use spacetimedb_client_api_messages::publish::PublishRequest; +use spacetimedb_lib::environment::EnvironmentMap; use spacetimedb_lib::Hash; -use std::collections::BTreeMap; // Keep the existing tree name for on-disk compatibility. const METADATA_TREE: &str = "database_bootstrap"; @@ -85,7 +84,7 @@ impl ControlDb { } /// Recheck the exact persisted generation and program before releasing any values. - pub(crate) fn initial_environment(&self, database: &Database, replica_id: u64) -> Result> { + pub(crate) fn initial_environment(&self, database: &Database, replica_id: u64) -> Result { let databases = self.db.open_tree("database_by_identity")?; let metadata = self.db.open_tree(METADATA_TREE)?; let values = self.db.open_tree(VALUES_TREE)?; @@ -149,14 +148,8 @@ impl ControlDb { }); let bytes = result.map_err(transaction_error)?; match bytes { - None => Ok(BTreeMap::new()), - Some(bytes) => { - let request = PublishRequest::decode(&bytes).map_err(|_| invalid())?; - if request.module.as_ref().is_some_and(|module| !module.is_empty()) { - return Err(invalid()); - } - Ok(request.environment) - } + None => Ok(Default::default()), + Some(bytes) => serde_json::from_slice(&bytes).map_err(|_| invalid()), } } @@ -171,7 +164,7 @@ impl ControlDb { &self, mut database: Database, expected: Option<&Database>, - environment: BTreeMap, + environment: EnvironmentMap, previous_replicas: &[Replica], ) -> Result<(Database, Replica)> { if expected.is_none() { @@ -186,13 +179,7 @@ impl ControlDb { node_id: 0, leader: true, }; - let input = PublishRequest { - module: None, - environment, - ..Default::default() - } - .encode() - .map_err(|_| invalid())?; + let input = serde_json::to_vec(&environment).map_err(|_| invalid())?; let binding = InitializationMetadata { version: 1, database_id: database.id, @@ -300,7 +287,7 @@ fn transaction_error(error: TransactionError) -> Error { #[async_trait::async_trait] impl spacetimedb::host::InitialEnvironmentSource for ControlDb { - async fn load(&self, database: &Database, replica_id: u64) -> anyhow::Result> { + async fn load(&self, database: &Database, replica_id: u64) -> anyhow::Result { let source = self.clone(); let database = database.clone(); spacetimedb::util::asyncify(move || { @@ -314,6 +301,8 @@ impl spacetimedb::host::InitialEnvironmentSource for ControlDb { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::*; use spacetimedb::messages::control_db::HostType; diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 4a7479cd6a8..8047eed7c87 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -14,6 +14,7 @@ use spacetimedb::config::{CertificateAuthority, MetadataFile, ModuleHttpConfig, use spacetimedb::db; use spacetimedb::db::persistence::{DurabilityConfig, LocalPersistenceProvider}; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; +use spacetimedb::host::module_host::UpdateEnvironmentResult; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; use spacetimedb::messages::control_db::{Database, HostType, Node, Replica}; @@ -30,6 +31,8 @@ use spacetimedb_client_api_messages::name::{ use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::traits::Program; +use spacetimedb_lib::environment::{EnvironmentMap, EnvironmentUpdate}; +use spacetimedb_lib::Hash; use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::StandaloneDataDirExt; use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; @@ -277,26 +280,18 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { publisher: &Identity, spec: spacetimedb_client_api::DatabaseDef, policy: MigrationPolicy, + environment: EnvironmentUpdate, ) -> anyhow::Result> { let existing_db = self.control_db.get_database_by_identity(&spec.database_identity)?; - let update = spacetimedb_lib::environment::EnvironmentUpdate { - values: spec.environment, - remove: spec.environment_remove, - replace: spec.environment_replace, - }; - update.validate()?; // standalone does not support replication. let num_replicas = 1; match existing_db { // The database does not already exist, so we'll create it. None => { - anyhow::ensure!( - !spec.program_bytes.is_empty() && spec.expected_module_version.is_none(), - "initial publication requires a module and cannot require an existing version" - ); - let environment = update.resulting_values(&Default::default())?; + anyhow::ensure!(!spec.program_bytes.is_empty(), "initial publication requires a module"); + let environment = environment.values; let program = Program::from_bytes(spec.host_type.into(), &spec.program_bytes[..]); let database = Database { @@ -347,51 +342,12 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { spec.host_type, spec.program_bytes.to_vec().into(), policy, - update, - spec.expected_module_version, + environment, ) .await?; if update_result.was_successful() { - let replicas = self.control_db.get_replicas_by_database(database_id)?; - let desired_replicas = num_replicas as usize; - if desired_replicas == 0 { - log::info!("Decommissioning all replicas of database {database_identity}"); - for instance in replicas { - self.delete_replica(instance.id).await?; - } - } else if desired_replicas > replicas.len() { - let n = desired_replicas - replicas.len(); - log::info!( - "Scaling up database {} from {} to {} replicas", - database_identity, - replicas.len(), - n - ); - for _ in 0..n { - self.insert_replica(Replica { - id: 0, - database_id, - node_id: 0, - leader: false, - }) - .await?; - } - } else if desired_replicas < replicas.len() { - let n = replicas.len() - desired_replicas; - log::info!( - "Scaling down database {} from {} to {} replicas", - database_identity, - replicas.len(), - n - ); - for instance in replicas.into_iter().filter(|instance| !instance.leader).take(n) { - self.delete_replica(instance.id).await?; - } - } else { - log::debug!( - "Desired replica count {desired_replicas} for database {database_identity} already satisfied" - ); - } + self.scale_replicas(database_id, &database_identity, num_replicas) + .await?; } anyhow::Ok(Some(update_result)) @@ -443,7 +399,12 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { Ok(()) } - async fn reset_database(&self, caller_identity: &Identity, spec: DatabaseResetDef) -> anyhow::Result<()> { + async fn reset_database( + &self, + caller_identity: &Identity, + spec: DatabaseResetDef, + environment: EnvironmentMap, + ) -> anyhow::Result<()> { let previous = self .control_db .get_database_by_identity(&spec.database_identity)? @@ -453,12 +414,6 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { "database ownership changed before reset" ); let previous = self.control_db.with_initialization_generation(previous)?; - let environment = spacetimedb_lib::environment::EnvironmentUpdate { - values: spec.environment, - remove: spec.environment_remove, - replace: spec.environment_replace, - } - .resulting_values(&Default::default())?; let mut database = previous.clone(); let program = match spec.program_bytes { Some(bytes) => { @@ -550,6 +505,86 @@ impl spacetimedb_client_api::ControlStateWriteAccess for StandaloneEnv { self.control_db.set_database_lock(database_identity, locked)?; Ok(()) } + + async fn update_environment( + &self, + publisher: &Identity, + database_identity: &Identity, + environment: EnvironmentUpdate, + expected_module_hash: Hash, + ) -> anyhow::Result { + let Some(database) = self.control_db.get_database_by_identity(database_identity)? else { + anyhow::bail!("Database not found: {}", database_identity.to_abbreviated_hex()); + }; + + // standalone does not support replication. + let num_replicas = 1; + + anyhow::ensure!( + database.owner_identity == *publisher, + "database ownership changed before publication" + ); + let database_id = database.id; + + let leader = self.leader(database_id).await?; + let update_result = leader + .update_environment(database, environment, expected_module_hash) + .await?; + if update_result.was_successful() { + self.scale_replicas(database_id, database_identity, num_replicas) + .await?; + } + + anyhow::Ok(update_result) + } +} + +impl StandaloneEnv { + async fn scale_replicas( + &self, + database_id: u64, + database_identity: &Identity, + desired_replicas: usize, + ) -> anyhow::Result<()> { + let replicas = self.control_db.get_replicas_by_database(database_id)?; + if desired_replicas == 0 { + log::info!("Decommissioning all replicas of database {database_identity}"); + for instance in replicas { + self.delete_replica(instance.id).await?; + } + } else if desired_replicas > replicas.len() { + let n = desired_replicas - replicas.len(); + log::info!( + "Scaling up database {} from {} to {} replicas", + database_identity, + replicas.len(), + n + ); + for _ in 0..n { + self.insert_replica(Replica { + id: 0, + database_id, + node_id: 0, + leader: false, + }) + .await?; + } + } else if desired_replicas < replicas.len() { + let n = replicas.len() - desired_replicas; + log::info!( + "Scaling down database {} from {} to {} replicas", + database_identity, + replicas.len(), + n + ); + for instance in replicas.into_iter().filter(|instance| !instance.leader).take(n) { + self.delete_replica(instance.id).await?; + } + } else { + log::debug!("Desired replica count {desired_replicas} for database {database_identity} already satisfied"); + } + Ok(()) + } } impl spacetimedb_client_api::Authorization for StandaloneEnv { diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index c3a274048ac..2101df47ee8 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -207,6 +207,7 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { db_routes.root_post = db_routes.root_post.layer(DefaultBodyLimit::disable()); db_routes.db_put = db_routes.db_put.layer(DefaultBodyLimit::disable()); db_routes.pre_publish = db_routes.pre_publish.layer(DefaultBodyLimit::disable()); + db_routes.db_reset = db_routes.db_reset.layer(DefaultBodyLimit::disable()); let extra = axum::Router::new().nest("/health", spacetimedb_client_api::routes::health::router()); let task_dumps = TaskDumpRegistry::new([("main", main_rt)]); let service = router(&ctx, db_routes, IdentityRoutes::default(), extra) diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 5f7c2b23db9..6838775d722 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -16,6 +16,7 @@ use spacetimedb::util::jobs::JobCores; use spacetimedb::Identity; use spacetimedb_client_api::auth::SpacetimeAuth; use spacetimedb_client_api::routes::subscribe::{generate_random_connection_id, WebSocketOptions}; +use spacetimedb_lib::environment::{EnvironmentRemove, EnvironmentUpdate}; use spacetimedb_lib::http as st_http; use spacetimedb_lib::AlgebraicValue; use spacetimedb_paths::{RootDir, SpacetimePaths}; @@ -96,16 +97,16 @@ impl ModuleHandle { DatabaseDef { database_identity: self.db_identity, program_bytes, - environment, - environment_remove: Vec::new(), - environment_replace: true, - expected_module_version: None, num_replicas: None, host_type, parent: None, organization: None, }, MigrationPolicy::Compatible, + EnvironmentUpdate { + values: environment, + remove: EnvironmentRemove::All, + }, ) .await? .ok_or_else(|| anyhow::anyhow!("expected an update to the existing database")) @@ -423,16 +424,13 @@ impl CompiledModule { DatabaseDef { database_identity: db_identity, program_bytes: self.program_bytes(), - environment, - environment_remove: Vec::new(), - environment_replace: false, - expected_module_version: None, num_replicas: None, host_type: self.host_type, parent: None, organization: None, }, MigrationPolicy::Compatible, + environment.into(), ) .await .unwrap(); diff --git a/crates/testing/tests/environment_lifecycle.rs b/crates/testing/tests/environment_lifecycle.rs index ab945e4455c..4449946f27e 100644 --- a/crates/testing/tests/environment_lifecycle.rs +++ b/crates/testing/tests/environment_lifecycle.rs @@ -10,6 +10,7 @@ use spacetimedb_client_api::routes::subscribe::WebSocketOptions; use spacetimedb_client_api::{ ControlStateReadAccess as _, ControlStateWriteAccess as _, DatabaseDef, DatabaseResetDef, NodeDelegate as _, }; +use spacetimedb_lib::environment::{EnvironmentRemove, EnvironmentUpdate}; use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use spacetimedb_paths::{server::ServerDataDir, FromPathUnchecked}; @@ -67,13 +68,9 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh ("REQUIRED".into(), "initial-required".into()), ("MODE".into(), "ready".into()), ]); - let spec = |environment| DatabaseDef { + let spec = || DatabaseDef { database_identity: Identity::ZERO, program_bytes: bytes.clone(), - environment, - environment_remove: Vec::new(), - environment_replace: false, - expected_module_version: None, num_replicas: None, host_type: HostType::Wasm, parent: None, @@ -83,7 +80,12 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, spec(initial.clone()), MigrationPolicy::Compatible) + .publish_database( + &Identity::ZERO, + spec(), + MigrationPolicy::Compatible, + initial.clone().into() + ) .await? .is_none()); let database = env @@ -100,8 +102,7 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh .unwrap(); log::info!("ENV standalone fixture: rejected publication preserves live host"); for changed_program in [false, true] { - let mut invalid = spec(Values::new()); - invalid.environment_replace = true; + let mut invalid = spec(); if changed_program { // An empty custom section changes the Wasm hash without changing // its declarations, exercising rejection of a candidate module. @@ -111,9 +112,15 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh } let rejected = tokio::time::timeout( std::time::Duration::from_secs(30), - env.as_ref() - .unwrap() - .publish_database(&Identity::ZERO, invalid, MigrationPolicy::Compatible), + env.as_ref().unwrap().publish_database( + &Identity::ZERO, + invalid, + MigrationPolicy::Compatible, + EnvironmentUpdate { + values: Values::new(), + remove: EnvironmentRemove::All, + }, + ), ) .await .expect("rejected publication must not hang"); @@ -128,21 +135,27 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh assert!(matches!( env.as_ref() .unwrap() - .publish_database(&Identity::ZERO, spec(Values::new()), MigrationPolicy::Compatible) + .publish_database( + &Identity::ZERO, + spec(), + MigrationPolicy::Compatible, + Values::new().into() + ) .await?, Some(UpdateDatabaseResult::NoUpdateNeeded) )); let previous_module = env.as_ref().unwrap().leader(database.id).await?.module().await?; let version = previous_module.info.module_hash; - let mut env_only = spec(Values::from([("FUTURE".into(), "undeclared".into())])); - env_only.program_bytes = Default::default(); - env_only.expected_module_version = Some(version); assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, env_only, MigrationPolicy::Compatible) + .update_environment( + &Identity::ZERO, + &Identity::ZERO, + Values::from([("FUTURE".into(), "undeclared".into())]).into(), + version + ) .await? - .unwrap() .was_successful()); let current = env.as_ref().unwrap().leader(database.id).await?.module().await?; assert!(Arc::ptr_eq(&previous_module.info, ¤t.info)); @@ -159,13 +172,16 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh AlgebraicValue::from(Some("initial-required".to_owned())) ); assert!(read(env.as_ref().unwrap(), database.id, "FUTURE").await.is_err()); - let mut stale = spec(Values::new()); - stale.program_bytes = Default::default(); - stale.expected_module_version = Some(spacetimedb_lib::Hash::from_hex("00".repeat(32))?); + let expected_module_hash = spacetimedb_lib::Hash::from_hex("00".repeat(32))?; assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, stale, MigrationPolicy::Compatible) + .update_environment( + &Identity::ZERO, + &Identity::ZERO, + Values::new().into(), + expected_module_hash + ) .await .is_err()); log::info!("ENV standalone fixture: same-program update"); @@ -174,7 +190,7 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, spec(updated), MigrationPolicy::Compatible) + .publish_database(&Identity::ZERO, spec(), MigrationPolicy::Compatible, updated.into()) .await? .unwrap() .was_successful()); @@ -193,15 +209,16 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh drop(env.take()); env = Some(StandaloneEnv::init(config, &ca, data_dir.clone(), JobCores::without_pinned_cores()).await?); // The first request after restart is env-only, before leader lookup has started a host. - let mut cold = spec(Values::from([("EMPTY".into(), "after-restart".into())])); - cold.program_bytes = Default::default(); - cold.expected_module_version = Some(version); assert!(env .as_ref() .unwrap() - .publish_database(&Identity::ZERO, cold, MigrationPolicy::Compatible) + .update_environment( + &Identity::ZERO, + &Identity::ZERO, + Values::from([("EMPTY".into(), "after-restart".into())]).into(), + version + ) .await? - .unwrap() .was_successful()); assert_eq!( read(env.as_ref().unwrap(), database.id, "EMPTY").await?, @@ -221,12 +238,10 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh DatabaseResetDef { database_identity: Identity::ZERO, program_bytes: None, - environment_remove: Default::default(), - environment_replace: false, - environment: Values::new(), num_replicas: None, host_type: None, - } + }, + Default::default() ) .await .is_err()); @@ -244,12 +259,10 @@ fn real_module_reopen_and_environment_only_publication_preserve_values() -> anyh DatabaseResetDef { database_identity: Identity::ZERO, program_bytes: None, - environment_remove: Default::default(), - environment_replace: false, - environment: reset, num_replicas: None, host_type: None, }, + reset, ) .await?; assert_ne!( diff --git a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md index 4286906ee01..473c953fe86 100644 --- a/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md +++ b/docs/docs/00300-resources/00200-reference/00200-http-api/00300-database.md @@ -9,7 +9,7 @@ The HTTP endpoints in `/v1/database` allow clients to interact with Spacetime da ## At a glance | Route | Description | -|----------------------------------------------------------------------------------------------------|---------------------------------------------------| +| -------------------------------------------------------------------------------------------------- | ------------------------------------------------- | | [`POST /v1/database`](#post-v1database) | Publish a new database given its module code. | | [`PUT /v1/database/:name_or_identity`](#put-v1databasename_or_identity) | Publish to a database given its module code. | | [`GET /v1/database/:name_or_identity`](#get-v1databasename_or_identity) | Get a JSON description of a database. | @@ -21,6 +21,9 @@ The HTTP endpoints in `/v1/database` allow clients to interact with Spacetime da | [`GET /v1/database/:name_or_identity/subscribe`](#get-v1databasename_or_identitysubscribe) | Begin a WebSocket connection. | | [`POST /v1/database/:name_or_identity/call/:reducer`](#post-v1databasename_or_identitycallreducer) | Invoke a reducer OR procedure in a database. | | [`GET /v1/database/:name_or_identity/schema`](#get-v1databasename_or_identityschema) | Get the schema for a database. | +| [`GET /v1/database/:name_or_identity/environment`](#get-v1databasename_or_identityenvironment) | Get environment declarations and stored keys. | +| [`PUT /v1/database/:name_or_identity/environment`](#put-v1databasename_or_identityenvironment) | Replace a database's environment. | +| [`PATCH /v1/database/:name_or_identity/environment`](#patch-v1databasename_or_identityenvironment) | Set or delete environment values. | | [`GET /v1/database/:name_or_identity/logs`](#get-v1databasename_or_identitylogs) | Retrieve logs from a database. | | [`POST /v1/database/:name_or_identity/sql`](#post-v1databasename_or_identitysql) | Run a SQL query against a database. | | [`ANY /v1/database/:name_or_identity/route/{*path}`](#any-v1databasename_or_identityroutepath) | Access database-defined HTTP APIs. | @@ -33,15 +36,17 @@ Accessible through the CLI as `spacetime publish`. #### Optional Headers -| Name | Value | -| --------------- | ----------------------------------------------------------------------------------- | -| `Authorization` | A Spacetime token [as Bearer auth](./00100-authorization.md#authorization-headers). | +| Name | Value | +| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Authorization` | A Spacetime token [as Bearer auth](./00100-authorization.md#authorization-headers). | +| `spacetime-environment` | One `KEY=VALUE` environment value. Repeat the header for each value. See [Publishing with environment values](#publishing-with-environment-values). | +| `spacetime-environment-remove` | Stored environment keys to delete, or `*` to replace the whole environment. | If no `Authorization` header is provided, a new anonymous identity will be created and will own the new database. This is generally not what you want. #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). To publish with environment values, see [Publishing with environment values](#publishing-with-environment-values). #### Returns @@ -68,15 +73,17 @@ Accessible through the CLI as `spacetime publish`. #### Optional Headers -| Name | Value | -| --------------- | ----------------------------------------------------------------------------------- | -| `Authorization` | A Spacetime token [as Bearer auth](./00100-authorization.md#authorization-headers). | +| Name | Value | +| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Authorization` | A Spacetime token [as Bearer auth](./00100-authorization.md#authorization-headers). | +| `spacetime-environment` | One `KEY=VALUE` environment value. Repeat the header for each value. See [Publishing with environment values](#publishing-with-environment-values). | +| `spacetime-environment-remove` | Stored environment keys to delete, or `*` to replace the whole environment. | If no `Authorization` header is provided, a new anonymous identity will be created. When updating an existing database, the token must correspond to the database's owner, or the request will be rejected. #### Data -A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html), or a [publish request with environment values](#publishing-with-environment-values). +A WebAssembly module in the [binary format](https://webassembly.github.io/spec/core/binary/index.html). To publish with environment values, see [Publishing with environment values](#publishing-with-environment-values). #### Returns @@ -100,57 +107,133 @@ If a database with the given name exists, but the identity provided in the `Auth ### Publishing with environment values -Both publish endpoints accept `Content-Type: application/vnd.spacetimedb.publish+json` with this JSON body: +Both publish endpoints accept [environment values](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) in headers, alongside the module in the body. Send one `spacetime-environment` header per value, in the form `KEY=VALUE`: -```json -{ - "module": "", - "environment": { - "API_KEY": "development-only-key", - "MODE": "development" - } -} +``` +spacetime-environment: API_KEY=development-only-key +spacetime-environment: MODE=development +``` + +Values are written like shell or `.env` assignments: + +- A bare value, such as `API_KEY=abc123`, is taken literally. It cannot contain spaces, tabs, quotes, `,` or `\`. +- A single-quoted value, such as `GREETING='hello, world'`, is taken literally up to the closing quote. It cannot contain `'`. +- A double-quoted value, such as `PEM="line1\nline2"`, supports JSON string escapes, as in `.env` files: `\"`, `\\`, `\n`, `\t`, and `\uXXXX` for other control characters and non-ASCII characters, which cannot appear raw in a header. + +Unlike a shell, the server does not expand variables. Values that break these rules, such as `GREETING=hello world`, are rejected with `400 Bad Request`. The server also accepts several values in one header, separated by commas outside quotes. + +Formally, in [ABNF](https://www.rfc-editor.org/rfc/rfc5234), using the list syntax (`#`) from [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-5.6.1): + +```text +spacetime-environment = #assignment +assignment = key "=" value +key = ( ALPHA / "_" ) *( ALPHA / DIGIT / "_" ) +value = bare / single-quoted / double-quoted +bare = *( %x21 / %x23-26 / %x28-2B / %x2D-5B / %x5D-7E ) + ; visible ASCII except " ' , \ +single-quoted = "'" *( HTAB / %x20-26 / %x28-7E ) "'" + ; taken literally +double-quoted = DQUOTE *( %x20-21 / %x23-5B / %x5D-7E / escape ) DQUOTE +escape = "\" ( DQUOTE / "\" / "/" / %x62 / %x66 / %x6E / %x72 / %x74 / %x75 4HEXDIG ) + ; \" \\ \/ \b \f \n \r \t \uXXXX, lowercase only, as in JSON + ; strings (RFC 8259). A \u escape of a UTF-16 high surrogate must be + ; followed by one of a low surrogate, and a low surrogate cannot + ; appear on its own. + +spacetime-environment-remove = #( "*" / key ) + ; "*" must be the only element ``` -`module` uses standard padded Base64. `environment` supplies string overrides for declared or undeclared names. Unspecified stored values survive by default. The server validates the resulting environment against the module's declarations and installs both in one transaction: all required values must exist and all present declared values must satisfy their constraints. An empty or omitted map preserves stored values, including when publishing unchanged module bytes. +Keys are at most 256 bytes and values at most 8 KiB. A key cannot appear twice across all `spacetime-environment` headers. -A missing required value returns HTTP 400 with `Content-Type: application/json` and `{"error":"missing_required_environment","key":"API_KEY"}` identifying the missing key. Clients can prompt for that value and retry the publish. Other validation and module failures do not use this error code. +Supplied values override stored values with the same key, and stored values that are not supplied are kept. To delete stored values, list their keys in a `spacetime-environment-remove` header, separated by commas (`OLD_KEY, _UNUSED`). To replace the whole environment with only the supplied values, send `spacetime-environment-remove: *`. A key cannot be both supplied and removed. -Optional fields `environment_remove` (an array of keys) and `environment_replace` (a boolean, default `false`) request explicit deletion or complete replacement. A key cannot be both supplied and removed. Replacement uses only the supplied map, deleting every unspecified declared and undeclared key, and rejects any nonempty removal list. Invalid updates leave the database unchanged. +The server validates the resulting environment against the module's declarations and installs the module and the environment in one transaction. All required values must be present, and every declared value must satisfy its declaration. Invalid updates leave the database unchanged. -To update an existing database without a module, omit `module` and provide `expected_module_version` from `GET /v1/database/{name_or_identity}/environment`. That authorized endpoint returns `module_version`, `declarations`, and `stored_keys`, without secret values. Each declaration contains `name`, `optional`, and `ty`: `"String"`, `{"StringLiteral":"value"}`, or `{"Union":["a","b"]}`. Metadata comes from one database version. For example: +If a required value is missing, the server returns `400 Bad Request` with JSON that names the missing key, so clients can prompt for it and retry: ```json { - "expected_module_version": "", - "environment": { "FUTURE_KEY": "development-only-value" }, - "environment_remove": ["OLD_OPTIONAL_KEY"], - "environment_replace": false + "EnvironmentError": { "MissingRequiredEnvironment": { "keys": ["API_KEY"] } } } ``` -Send this body to the existing database's PUT endpoint with the same content type and publish authorization. Environment-only updates validate against the deployed schema, never invoke initialization or migration, and reject stale module versions. A new database still requires a module and all initially required values in its initial publish. +A reset (`clear=true`) deletes all stored values, so the request must supply every required value again. -For example, use `curl`, `jq`, and `base64` to publish a Wasm module to a local server. Export `SPACETIME_TOKEN` with a token authorized to publish and `API_KEY` with the required value. Change `module.wasm` to the artifact you built. +For example, to publish a module to a local server with `curl`, export `SPACETIME_TOKEN` with a token authorized to publish, then run: ```bash -base64 < module.wasm | - jq --raw-input --slurp '{ - module: gsub("[\\r\\n]"; ""), - environment: { API_KEY: env.API_KEY, MODE: "development" } - }' | - curl --fail-with-body --request PUT \ - 'http://127.0.0.1:3000/v1/database/env-example?host_type=wasm' \ - --header "Authorization: Bearer $SPACETIME_TOKEN" \ - --header 'Content-Type: application/vnd.spacetimedb.publish+json' \ - --data-binary @- +curl --fail-with-body --request PUT \ + 'http://127.0.0.1:3000/v1/database/env-example?host_type=wasm' \ + --header "Authorization: Bearer $SPACETIME_TOKEN" \ + --header 'spacetime-environment: API_KEY=development-only-key' \ + --header 'spacetime-environment: MODE=development' \ + --data-binary @module.wasm ``` -`jq` handles JSON escaping for the supplied value, including quotes and newlines. For a procedure or another HTTP client, construct the same JSON object with that language's JSON serializer and Base64 encoder. +To change environment values without publishing a module, use [`PUT`](#put-v1databasename_or_identityenvironment) or [`PATCH /v1/database/:name_or_identity/environment`](#patch-v1databasename_or_identityenvironment). + +## `GET /v1/database/:name_or_identity/environment` + +Get the environment declarations of the database's current module and the keys of its stored values. Values are never returned. Requires a token authorized to update the database. + +#### Returns + +```typescript +{ + "module_hash": string, + "declarations": [{ + "name": string, + "optional": boolean, + "ty": "String" | { "StringLiteral": string } | { "Union": string[] } + }], + "stored_keys": string[] +} +``` + +`module_hash` identifies the module the declarations came from. Treat it as an opaque string and pass it unchanged as `expected_module_hash` when updating the environment. + +## `PUT /v1/database/:name_or_identity/environment` -Direct HTTP callers, including module procedures, use this same format; the server does not load project configuration or shell values for them. See [Environment Variables](../../../00200-core-concepts/00100-databases/00700-environment-variables.md) for declaration syntax and value limits. The decoded module is limited to 128 MiB and the complete encoded request to 192 MiB. +Replace the database's whole environment without publishing a module. Stored values that are not supplied are deleted. + +#### Query Parameters + +| Name | Value | +| ---------------------- | ---------------------------------------------------------------------------------------- | +| `expected_module_hash` | The `module_hash` from [`GET /environment`](#get-v1databasename_or_identityenvironment). | + +#### Data + +A JSON object mapping each key to its string value, for example `{"API_KEY": "new-key", "MODE": "production"}`. + +#### Returns + +On success, returns `200 OK` with `{"Ok": null}`. If the resulting environment does not satisfy the module's declarations, returns `400 Bad Request`, with `{"Err": {"MissingRequiredEnvironment": {"keys": [...]}}}` when a required value is missing. If the database's module no longer has `expected_module_hash`, returns `409 Conflict`; fetch the metadata again before retrying. + +## `PATCH /v1/database/:name_or_identity/environment` + +Set and delete individual environment values without publishing a module. Stored values that are neither supplied nor removed are kept. + +#### Query Parameters + +| Name | Value | +| ---------------------- | ---------------------------------------------------------------------------------------- | +| `expected_module_hash` | The `module_hash` from [`GET /environment`](#get-v1databasename_or_identityenvironment). | + +#### Optional Headers + +| Name | Value | +| ------------------------------ | ----------------------------------------------------------------------------------- | +| `spacetime-environment-remove` | Stored keys to delete, separated by commas. `*` is not accepted; use `PUT` instead. | + +#### Data + +A JSON object mapping each key to set to its string value. Send `{}` to only delete keys. + +#### Returns -Raw module bodies continue to work and supply no environment overrides, retaining stored values on ordinary publishes. A destructive clear/reset still removes stored values and requires all initially required values again. Use `application/octet-stream` for that format. This preserves compatibility with older servers for modules that do not require ENV support; older servers do not support the JSON publish format. +The same as [`PUT /environment`](#put-v1databasename_or_identityenvironment). ## `GET /v1/database/:name_or_identity`