diff --git a/CHANGELOG.md b/CHANGELOG.md index 532761e87..9e5355a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ air-gapped signing # Unreleased +* feat: a canister that uses a `recipe` can now declare its own `sync` steps, which previously was rejected outright. They run after the sync steps the recipe renders, so a recipe's post-deployment work stays intact and yours is appended to it. `recipe` and `build` remain mutually exclusive. * feat: `script` build steps now receive `ICP_CLI_ENVIRONMENT`, the name of the environment the canisters are being built for, so a build can vary by environment the way a sync step already could. * feat: `icp completions ` prints a shell completion script for `bash`, `zsh`, `fish`, `powershell`, or `elvish` to stdout. See the [installation guide](docs/guides/installation.md#shell-completions) for where to put it. * feat(sync-plugin): `dirs:` and `files:` on a `plugin` sync step may now name anything inside the project, not just paths below the canister's own directory. Entries are still written relative to the canister directory, but may rise out of it — `dirs: ["../shared/assets"]` — so several canisters can be handed the same tree without duplicating it. The project directory is the boundary: an entry that resolves above it is rejected before the plugin runs, as is an absolute one, and an entry that is (or traverses) a symlink is still rejected outright. The plugin sees each directory at the path the manifest wrote, `..` and all. diff --git a/Cargo.lock b/Cargo.lock index bd140f275..5a93a621b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3625,7 +3625,6 @@ dependencies = [ "flate2", "futures", "glob", - "handlebars", "hex", "hmac 0.13.0", "httptest", @@ -3637,6 +3636,7 @@ dependencies = [ "ic-management-canister-types 0.8.0", "ic-utils", "icp-canister-interfaces", + "icp-deploy-canister", "icp-sync-plugin", "icrc-ledger-types", "indexmap", @@ -3728,6 +3728,7 @@ dependencies = [ "ic-utils", "icp", "icp-canister-interfaces", + "icp-deploy-canister", "icp-sync-plugin", "icrc-ledger-types", "indicatif", @@ -3773,6 +3774,42 @@ dependencies = [ "wslpath2", ] +[[package]] +name = "icp-deploy-canister" +version = "1.3.0" +dependencies = [ + "async-trait", + "bigdecimal", + "camino", + "camino-tempfile", + "candid", + "candid_parser", + "clap", + "futures", + "glob", + "handlebars", + "hex", + "ic-management-canister-types 0.8.0", + "indexmap", + "indoc", + "itertools 0.14.0", + "jsonschema", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "pathdiff", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "sha2 0.11.0", + "snafu", + "strum 0.28.0", + "tokio", + "tracing", + "url", +] + [[package]] name = "icp-sync-plugin" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 60dc41e1b..12840d4f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ ic-management-canister-types = { version = "0.8.0" } ic-utils = { version = "0.49.1" } icp = { path = "crates/icp" } icp-canister-interfaces = { path = "crates/icp-canister-interfaces" } +icp-deploy-canister = { path = "crates/icp-deploy-canister" } icp-sync-plugin = { path = "crates/icp-sync-plugin" } ic-identity-hsm = "0.49.1" icrc-ledger-types = "0.1.10" diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index 7e326084c..8ff4393da 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -44,6 +44,7 @@ ic-management-canister-types.workspace = true ic-utils.workspace = true icp-canister-interfaces.workspace = true icp = { workspace = true, features = ["clap"] } +icp-deploy-canister.workspace = true icp-sync-plugin.workspace = true icrc-ledger-types.workspace = true indicatif.workspace = true diff --git a/crates/icp-cli/src/commands/canister/install.rs b/crates/icp-cli/src/commands/canister/install.rs index a55fe5451..f44ae0246 100644 --- a/crates/icp-cli/src/commands/canister/install.rs +++ b/crates/icp-cli/src/commands/canister/install.rs @@ -10,14 +10,14 @@ use icp::fs; use icp::prelude::*; use tracing::{info, warn}; +use icp_deploy_canister::install_canister_wasm; + use crate::{ commands::args::{self, ArgsOpt}, operations::{ + access::AgentIcpAccess, candid_compat::{CandidCompatibility, check_candid_compatibility}, - install::{ - WasmMemoryPersistenceOpt, install_canister, is_eop_canister, - resolve_install_mode_and_status, - }, + install::{WasmMemoryPersistenceOpt, is_eop_canister, resolve_install_mode_and_status}, }, }; @@ -184,16 +184,21 @@ pub(crate) async fn exec(ctx: &Context, args: &InstallArgs) -> Result<(), anyhow } } - install_canister( - &agent, - args.proxy, - &canister_id, + let icp = AgentIcpAccess::new(agent.clone(), args.proxy); + let wmp = args + .wasm_memory_persistence + .map(WasmMemoryPersistenceOpt::to_ic); + // Install the bytes read above, not a fresh read of the same source, so the + // module installed is the one the Candid check ran against. + install_canister_wasm( &canister_display, + canister_id, &wasm, install_mode, status, init_args_bytes.as_deref(), - args.wasm_memory_persistence, + wmp, + &icp, ) .await?; diff --git a/crates/icp-cli/src/commands/deploy.rs b/crates/icp-cli/src/commands/deploy.rs index 50acdad2d..749823e8a 100644 --- a/crates/icp-cli/src/commands/deploy.rs +++ b/crates/icp-cli/src/commands/deploy.rs @@ -489,9 +489,10 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: .into_iter() .collect(); - let pkg_cache = ctx.dirs.package_cache()?; + let resolver = ctx.resource_resolver()?; sync_many( ctx.syncer.clone(), + resolver, agent.clone(), sync_canisters, ctx.project.load().await?.dir, @@ -500,7 +501,6 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: canister_ids, args.proxy, ctx.debug, - &pkg_cache, ) .await?; } diff --git a/crates/icp-cli/src/commands/sync.rs b/crates/icp-cli/src/commands/sync.rs index c3f293c0e..4c72afefb 100644 --- a/crates/icp-cli/src/commands/sync.rs +++ b/crates/icp-cli/src/commands/sync.rs @@ -9,8 +9,10 @@ use icp::identity::IdentitySelection; use std::collections::BTreeMap; use tracing::info; +use icp::Canister; + use crate::{ - operations::{proxy_management, sync::sync_many}, + operations::{binding_env_vars::set_binding_env_vars_many, proxy_management, sync::sync_many}, options::{EnvironmentOpt, IdentityOpt}, }; @@ -124,9 +126,27 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E .into_iter() .collect(); - let pkg_cache = ctx.dirs.package_cache()?; + // Apply the generated `PUBLIC_CANISTER_ID:*` environment variables before + // syncing. `deploy` does this, but standalone `icp sync` previously did not, + // so a synced canister could run against stale/absent binding ids. + let target_canisters: Vec<(Principal, Canister)> = sync_canisters + .iter() + .map(|(cid, _, info)| (*cid, info.clone())) + .collect(); + set_binding_env_vars_many( + agent.clone(), + args.proxy, + environment_selection.name(), + target_canisters, + canister_ids.clone(), + ctx.debug, + ) + .await?; + + let resolver = ctx.resource_resolver()?; sync_many( ctx.syncer.clone(), + resolver, agent, sync_canisters, ctx.project.load().await?.dir, @@ -135,7 +155,6 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E canister_ids, args.proxy, ctx.debug, - &pkg_cache, ) .await?; diff --git a/crates/icp-cli/src/operations/access.rs b/crates/icp-cli/src/operations/access.rs new file mode 100644 index 000000000..993074caf --- /dev/null +++ b/crates/icp-cli/src/operations/access.rs @@ -0,0 +1,131 @@ +//! Host implementations of the `icp-deploy-canister` IO traits, backing the +//! library's install/sync/deploy core with the CLI's `ic-agent` transport and +//! on-disk stores. + +use std::sync::Arc; + +use async_trait::async_trait; +use candid::Principal; +use icp::prelude::*; +use icp::store_artifact; +use icp_deploy_canister::files::{FileAccess, FileAccessError}; +use icp_deploy_canister::icp_access::{IcpAccess, IcpAccessError}; + +use super::proxy::update_or_proxy_raw; + +/// [`IcpAccess`] over an `ic-agent` `Agent`. Proxy routing is baked in (the +/// impl is constructed with the proxy principal); the library never threads a +/// proxy per call. The caller's principal is captured once at construction. +pub struct AgentIcpAccess { + agent: ic_agent::Agent, + proxy: Option, + caller: Principal, +} + +impl AgentIcpAccess { + pub fn new(agent: ic_agent::Agent, proxy: Option) -> Self { + let caller = agent + .get_principal() + .unwrap_or_else(|_| Principal::anonymous()); + Self { + agent, + proxy, + caller, + } + } +} + +#[async_trait] +impl IcpAccess for AgentIcpAccess { + async fn canister_update( + &self, + canister: Principal, + method: &str, + arg: Vec, + effective_canister_id: Principal, + cycles: u128, + ) -> Result, IcpAccessError> { + update_or_proxy_raw( + &self.agent, + canister, + method, + arg, + self.proxy, + Some(effective_canister_id), + cycles, + ) + .await + .map_err(|e| IcpAccessError::Update { + canister, + method: method.to_owned(), + message: e.to_string(), + }) + } + + async fn read_canister_metadata( + &self, + canister: Principal, + path: &str, + ) -> Result>, IcpAccessError> { + // A read failure is treated as "metadata absent" (matching the previous + // EOP-detection behavior), so a missing custom section never aborts an + // install. + Ok(self + .agent + .read_state_canister_metadata(canister, path) + .await + .ok()) + } + + fn caller_principal(&self) -> Principal { + self.caller + } +} + +/// [`FileAccess`] backed by the canister build-artifact store. The library reads +/// a canister's built wasm via `read_file(artifact_path)`; here the "path" is the +/// canister's store key, resolved through the (locked) artifact store. Only +/// `read_file` is used by the install path; the other methods have benign +/// defaults. +pub struct ArtifactFileAccess(pub Arc); + +#[async_trait] +impl FileAccess for ArtifactFileAccess { + async fn read_file(&self, path: &Path) -> Result, FileAccessError> { + self.0 + .lookup(path.as_str()) + .await + .map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn read_to_string(&self, path: &Path) -> Result { + let bytes = self.read_file(path).await?; + String::from_utf8(bytes).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn exists(&self, path: &Path) -> bool { + self.0.lookup(path.as_str()).await.is_ok() + } + + async fn is_file(&self, path: &Path) -> bool { + self.exists(path).await + } + + async fn is_dir(&self, _path: &Path) -> bool { + false + } + + async fn read_dir(&self, _path: &Path) -> Result, FileAccessError> { + Ok(Vec::new()) + } + + async fn canonicalize(&self, path: &Path) -> Option { + Some(path.to_owned()) + } +} diff --git a/crates/icp-cli/src/operations/binding_env_vars.rs b/crates/icp-cli/src/operations/binding_env_vars.rs index a2118f193..41c21e07a 100644 --- a/crates/icp-cli/src/operations/binding_env_vars.rs +++ b/crates/icp-cli/src/operations/binding_env_vars.rs @@ -1,29 +1,16 @@ use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::{Agent, export::Principal}; -use ic_management_canister_types::{CanisterSettings, EnvironmentVariable, UpdateSettingsArgs}; use icp::Canister; +use icp_deploy_canister::{SyncCanisterError, apply_binding_env_vars}; use snafu::Snafu; use tracing::error; +use crate::operations::access::AgentIcpAccess; use crate::progress::{ProgressManager, ProgressManagerSettings}; -use super::proxy::UpdateOrProxyError; -use super::proxy_management; - -#[derive(Debug, Snafu)] -pub enum BindingEnvVarsOperationError { - #[snafu(display("Could not find canister id(s) for {} in environment '{environment}'. Make sure they are created first", canister_names.join(", ")))] - CanisterNotCreated { - environment: String, - canister_names: Vec, - }, - - #[snafu(transparent)] - UpdateOrProxy { source: UpdateOrProxyError }, -} - #[derive(Debug, Snafu)] #[snafu(display("Canister(s) {names:?} failed to update environment variables."))] pub struct SetBindingEnvVarsManyError { @@ -34,50 +21,15 @@ pub struct SetBindingEnvVarsManyError { struct BindingEnvVarsFailure { canister_name: String, canister_id: Principal, - error: BindingEnvVarsOperationError, -} - -pub(crate) async fn set_env_vars_for_canister( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_info: &Canister, - binding_vars: &[(String, String)], -) -> Result<(), BindingEnvVarsOperationError> { - let mut environment_variables = canister_info - .settings - .environment_variables - .to_owned() - .unwrap_or_default(); - - // inject the ids of the other canisters - for (k, v) in binding_vars.iter() { - environment_variables.insert(k.to_string(), v.to_string()); - } - - let environment_variables = environment_variables - .into_iter() - .map(|(name, value)| EnvironmentVariable { name, value }) - .collect::>(); - - proxy_management::update_settings( - agent, - proxy, - UpdateSettingsArgs { - canister_id: *canister_id, - settings: CanisterSettings { - environment_variables: Some(environment_variables), - ..Default::default() - }, - sender_canister_version: None, - }, - ) - .await?; - - Ok(()) + error: SyncCanisterError, } -/// Orchestrates setting environment variables for multiple canisters with progress tracking +/// Orchestrates setting environment variables for multiple canisters with progress tracking. +/// +/// The per-canister work (computing the generated `PUBLIC_CANISTER_ID:*` +/// bindings, merging with manifest env vars, and applying them) lives in +/// `icp_deploy_canister::apply_binding_env_vars`; this wrapper only adds the +/// missing-id precheck and progress display. pub(crate) async fn set_binding_env_vars_many( agent: Agent, proxy: Option, @@ -86,20 +38,14 @@ pub(crate) async fn set_binding_env_vars_many( canister_list: BTreeMap, debug: bool, ) -> Result<(), SetBindingEnvVarsManyError> { - // Check that all the canisters in this environment have an id - // We need to have all the ids to generate environment variables - // for the bindings + // Check that all the canisters in this environment have an id: we need all + // ids to generate the binding environment variables. let canisters_with_ids: HashSet<&String> = canister_list.keys().collect(); - let all_canister_names: Vec = target_canisters + let missing_canisters: Vec = target_canisters .iter() .map(|(_, info)| info.name.clone()) - .collect(); - - let missing_canisters: Vec = all_canister_names - .iter() - .filter(|c| !canisters_with_ids.contains(*c)) - .map(|c| c.to_string()) + .filter(|c| !canisters_with_ids.contains(c)) .collect(); if !missing_canisters.is_empty() { @@ -116,38 +62,23 @@ pub(crate) async fn set_binding_env_vars_many( .fail(); } + let icp = Arc::new(AgentIcpAccess::new(agent, proxy)); + let canister_list = Arc::new(canister_list); + let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); for (cid, info) in target_canisters { let pb = progress_manager.create_progress_bar(&info.name); let canister_name = info.name.clone(); - - // Each canister receives only the ids it is wired to (its own project's - // canisters by their local names, plus any declared dependencies under - // their aliases), resolved to the ids that exist in this environment. - // A project without dependencies wires every canister to every sibling, - // reproducing the previous flat behavior. - let binding_vars: Vec<(String, String)> = info - .bindings - .iter() - .filter_map(|(env_name, referenced_key)| { - canister_list.get(referenced_key).map(|principal| { - ( - format!("PUBLIC_CANISTER_ID:{env_name}"), - principal.to_text(), - ) - }) - }) - .collect(); + let icp = icp.clone(); + let canister_list = canister_list.clone(); let settings_fn = { - let agent = agent.clone(); let pb = pb.clone(); - async move { pb.set_message("Updating environment variables..."); - set_env_vars_for_canister(&agent, proxy, &cid, &info, &binding_vars).await + apply_binding_env_vars(&info, cid, &canister_list, icp.as_ref()).await } }; @@ -160,7 +91,6 @@ pub(crate) async fn set_binding_env_vars_many( ) .await; - // Map error to include canister context for deferred printing result.map_err(|error| BindingEnvVarsFailure { canister_name, canister_id: cid, diff --git a/crates/icp-cli/src/operations/install.rs b/crates/icp-cli/src/operations/install.rs index 24f90fd4e..f7b5190e7 100644 --- a/crates/icp-cli/src/operations/install.rs +++ b/crates/icp-cli/src/operations/install.rs @@ -1,16 +1,15 @@ -use candid::Encode; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::{Agent, export::Principal}; use ic_management_canister_types::{ - CanisterId, CanisterIdRecord, CanisterInstallMode, CanisterStatusType, ChunkHash, - ClearChunkStoreArgs, InstallChunkedCodeArgs, InstallCodeArgs, UpgradeFlags, UploadChunkArgs, - WasmMemoryPersistence, + CanisterId, CanisterIdRecord, CanisterInstallMode, CanisterStatusType, WasmMemoryPersistence, }; -use sha2::{Digest, Sha256}; +use icp::prelude::*; +use icp_deploy_canister::{InstallCanisterError, install_canister_resolved}; use snafu::{ResultExt, Snafu}; use std::sync::Arc; -use tracing::{debug, error, warn}; +use tracing::error; +use crate::operations::access::{AgentIcpAccess, ArtifactFileAccess}; use crate::progress::{ProgressManager, ProgressManagerSettings}; use super::misc::fetch_canister_metadata; @@ -28,7 +27,7 @@ pub enum WasmMemoryPersistenceOpt { } impl WasmMemoryPersistenceOpt { - fn to_ic(self) -> WasmMemoryPersistence { + pub(crate) fn to_ic(self) -> WasmMemoryPersistence { match self { WasmMemoryPersistenceOpt::Keep => WasmMemoryPersistence::Keep, WasmMemoryPersistenceOpt::Replace => WasmMemoryPersistence::Replace, @@ -44,43 +43,13 @@ pub(crate) async fn is_eop_canister(agent: &Agent, canister_id: &Principal) -> b .is_some() } -#[derive(Debug, Snafu)] -pub enum InstallOperationError { - #[snafu(display("Could not find build artifact for canister '{canister_name}'"))] - ArtifactNotFound { canister_name: String }, - - #[snafu(display("Failed to stop canister '{canister_name}' before upgrade"))] - StopCanister { - canister_name: String, - source: UpdateOrProxyError, - }, - - #[snafu(display("Failed to start canister '{canister_name}' after upgrade"))] - StartCanister { - canister_name: String, - source: UpdateOrProxyError, - }, - - #[snafu(transparent)] - UpdateOrProxy { source: UpdateOrProxyError }, -} - -#[derive(Debug, Snafu)] -#[snafu(display("Canister(s) {names:?} failed to install."))] -pub struct InstallManyError { - names: Vec, -} - -/// Holds error information from a failed canister install operation -struct InstallFailure { - canister_name: String, - canister_id: Principal, - error: InstallOperationError, -} - /// Resolve a mode string ("auto", "install", "reinstall", "upgrade") into /// a [`CanisterInstallMode`]. For "auto", queries `canister_status` to /// determine whether the canister already has code installed. +/// +/// Returns the resolved mode plus the current status; callers (deploy, the +/// candid-compat gate) need the resolved mode before installing, so resolution +/// happens here once and the result is handed to [`install_canister_resolved`]. pub(crate) async fn resolve_install_mode_and_status( agent: &Agent, proxy: Option, @@ -118,235 +87,48 @@ pub(crate) struct ResolveInstallModeError { source: UpdateOrProxyError, } -pub(crate) async fn install_canister( - agent: &Agent, - proxy: Option, +/// Install one canister whose build artifact lives in the store, addressed by +/// its store key `canister_name`. The install-code/chunking/EOP logic lives in +/// `icp_deploy_canister::install_canister_resolved`; this is a thin wrapper over +/// the artifact-backed `FileAccess` and agent-backed `IcpAccess`. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn install_stored_canister( + icp: &AgentIcpAccess, + files: &ArtifactFileAccess, canister_id: &Principal, canister_name: &str, - wasm: &[u8], mode: CanisterInstallMode, status: CanisterStatusType, init_args: Option<&[u8]>, wasm_memory_persistence: Option, -) -> Result<(), InstallOperationError> { - let mode = match mode { - CanisterInstallMode::Upgrade(_) => { - // if this is a motoko canister using EOP we need to set additional options. - // If the caller supplied an explicit override, trust it (the CLI layer has - // already validated that it's an EOP canister); otherwise auto-detect and - // default to Keep. - let persistence = match wasm_memory_persistence { - Some(opt) => Some(opt.to_ic()), - None => is_eop_canister(agent, canister_id) - .await - .then_some(WasmMemoryPersistence::Keep), - }; - if let Some(persistence) = persistence { - CanisterInstallMode::Upgrade(Some(UpgradeFlags { - skip_pre_upgrade: None, - wasm_memory_persistence: Some(persistence), - })) - } else { - mode - } - } - _ => mode, - }; - - debug!( - "Install new canister code for {} with mode `{:?}`", - canister_name, mode - ); - - do_install_operation( - agent, - proxy, - canister_id, +) -> Result<(), InstallCanisterError> { + install_canister_resolved( canister_name, - wasm, + *canister_id, + // The artifact `FileAccess` resolves the store key, so the "path" is the + // canister name. + Path::new(canister_name), mode, status, init_args, + wasm_memory_persistence.map(WasmMemoryPersistenceOpt::to_ic), + files, + icp, ) .await } -async fn do_install_operation( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_name: &str, - wasm: &[u8], - mode: CanisterInstallMode, - status: CanisterStatusType, - init_args: Option<&[u8]>, -) -> Result<(), InstallOperationError> { - // Threshold for chunked installation: 2 MB - // Raw install_code messages are limited to 2 MiB - const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024; - - // Chunk size: 1 MB (spec limit is 1 MiB per chunk) - const CHUNK_SIZE: usize = 1024 * 1024; - - // Generous overhead for encoding, target canister ID, install mode, etc. - const ENCODING_OVERHEAD: usize = 500; - - let cid = CanisterId::from(*canister_id); - let arg = init_args - .map(|a| a.to_vec()) - .unwrap_or_else(|| Encode!().unwrap()); - - // Calculate total install message size - let total_install_size = wasm.len() + arg.len() + ENCODING_OVERHEAD; - - if total_install_size <= CHUNK_THRESHOLD { - // Small wasm: use regular install_code - debug!("Installing wasm for {canister_name} using install_code"); - - let install_args = InstallCodeArgs { - mode, - canister_id: cid, - wasm_module: wasm.to_vec(), - arg, - sender_canister_version: None, - }; - - stop_and_start_if_upgrade( - agent, - proxy, - canister_id, - canister_name, - mode, - status, - async { - proxy_management::install_code(agent, proxy, install_args).await?; - Ok(()) - }, - ) - .await?; - } else { - // Large wasm: use chunked installation - debug!("Installing wasm for {canister_name} using chunked installation"); - - // Clear any existing chunks to ensure a clean state - proxy_management::clear_chunk_store(agent, proxy, ClearChunkStoreArgs { canister_id: cid }) - .await?; - - // Split wasm into chunks and upload them - let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect(); - let mut chunk_hashes: Vec = Vec::new(); - - for (i, chunk) in chunks.iter().enumerate() { - debug!( - "Uploading chunk {}/{} ({} bytes)", - i + 1, - chunks.len(), - chunk.len() - ); - - let upload_args = UploadChunkArgs { - canister_id: cid, - chunk: chunk.to_vec(), - }; - - let chunk_hash = proxy_management::upload_chunk(agent, proxy, upload_args).await?; - - chunk_hashes.push(chunk_hash); - } - - // Compute SHA-256 hash of the entire wasm module - let mut hasher = Sha256::new(); - hasher.update(wasm); - let wasm_module_hash = hasher.finalize().to_vec(); - - debug!("Installing chunked code with {} chunks", chunk_hashes.len()); - - let chunked_args = InstallChunkedCodeArgs { - mode, - target_canister: cid, - store_canister: None, - chunk_hashes_list: chunk_hashes, - wasm_module_hash, - arg, - sender_canister_version: None, - }; - - let install_res = stop_and_start_if_upgrade( - agent, - proxy, - canister_id, - canister_name, - mode, - status, - async { - proxy_management::install_chunked_code(agent, proxy, chunked_args).await?; - Ok(()) - }, - ) - .await; - - // Clear chunk store after successful installation to free up storage - let clear_res = proxy_management::clear_chunk_store( - agent, - proxy, - ClearChunkStoreArgs { canister_id: cid }, - ) - .await - .map_err(InstallOperationError::from); - - if let Err(clear_error) = clear_res { - if let Err(install_error) = install_res { - warn!("Failed to clear chunk store after failed install: {clear_error}"); - return Err(install_error); - } else { - return Err(clear_error); - } - } - install_res?; - } - - Ok(()) +#[derive(Debug, Snafu)] +#[snafu(display("Canister(s) {names:?} failed to install."))] +pub struct InstallManyError { + names: Vec, } -async fn stop_and_start_if_upgrade( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_name: &str, - mode: CanisterInstallMode, - status: CanisterStatusType, - f: impl Future>, -) -> Result<(), InstallOperationError> { - let should_guard = matches!( - mode, - CanisterInstallMode::Upgrade(_) | CanisterInstallMode::Reinstall - ) && matches!(status, CanisterStatusType::Running); - let cid_record = CanisterIdRecord { - canister_id: CanisterId::from(*canister_id), - }; - // Stop the canister before proceeding - if should_guard { - proxy_management::stop_canister(agent, proxy, cid_record.clone()) - .await - .context(StopCanisterSnafu { canister_name })?; - } - // Install the canister - let install_result = f.await; - // Restart the canister whether or not the installation succeeded - if should_guard { - let start_result = proxy_management::start_canister(agent, proxy, cid_record).await; - if let Err(start_error) = start_result { - // If both install and start failed, report the install error since it's more likely to be the root cause - if let Err(install_error) = install_result { - warn!("Failed to start canister after failed upgrade: {start_error}"); - return Err(install_error); - } else { - return Err(start_error).context(StartCanisterSnafu { canister_name }); - } - } - } - - install_result +/// Holds error information from a failed canister install operation +struct InstallFailure { + canister_name: String, + canister_id: Principal, + error: InstallCanisterError, } /// Installs code to multiple canisters and displays progress bars. @@ -365,32 +147,27 @@ pub(crate) async fn install_many( artifacts: Arc, debug: bool, ) -> Result<(), InstallManyError> { + let icp = Arc::new(AgentIcpAccess::new(agent, proxy)); + let files = Arc::new(ArtifactFileAccess(artifacts)); + let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); for (name, cid, mode, status, init_args) in canisters { let pb = progress_manager.create_progress_bar(&name); - let agent = agent.clone(); + let icp = icp.clone(); + let files = files.clone(); let install_fn = { let pb = pb.clone(); - let artifacts = artifacts.clone(); let name = name.clone(); async move { pb.set_message("Installing..."); - - let wasm = artifacts.lookup(&name).await.map_err(|_| { - InstallOperationError::ArtifactNotFound { - canister_name: name.clone(), - } - })?; - - install_canister( - &agent, - proxy, + install_stored_canister( + &icp, + &files, &cid, &name, - &wasm, mode, status, init_args.as_deref(), diff --git a/crates/icp-cli/src/operations/mod.rs b/crates/icp-cli/src/operations/mod.rs index 918b80613..c621fbd23 100644 --- a/crates/icp-cli/src/operations/mod.rs +++ b/crates/icp-cli/src/operations/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod access; pub(crate) mod binding_env_vars; pub(crate) mod build; pub(crate) mod bundle; diff --git a/crates/icp-cli/src/operations/proxy_management.rs b/crates/icp-cli/src/operations/proxy_management.rs index ffc7cac70..43ec4aa52 100644 --- a/crates/icp-cli/src/operations/proxy_management.rs +++ b/crates/icp-cli/src/operations/proxy_management.rs @@ -1,15 +1,14 @@ use candid::Principal; use ic_agent::Agent; use ic_management_canister_types::{ - CanisterIdRecord, CanisterStatusResult, ClearChunkStoreArgs, CreateCanisterArgs, - DeleteCanisterArgs, DeleteCanisterSnapshotArgs, FetchCanisterLogsArgs, FetchCanisterLogsResult, - InstallChunkedCodeArgs, InstallCodeArgs, ListCanisterSnapshotsArgs, - ListCanisterSnapshotsResult, LoadCanisterSnapshotArgs, ReadCanisterSnapshotDataArgs, - ReadCanisterSnapshotDataResult, ReadCanisterSnapshotMetadataArgs, + CanisterIdRecord, CanisterStatusResult, CreateCanisterArgs, DeleteCanisterArgs, + DeleteCanisterSnapshotArgs, FetchCanisterLogsArgs, FetchCanisterLogsResult, InstallCodeArgs, + ListCanisterSnapshotsArgs, ListCanisterSnapshotsResult, LoadCanisterSnapshotArgs, + ReadCanisterSnapshotDataArgs, ReadCanisterSnapshotDataResult, ReadCanisterSnapshotMetadataArgs, ReadCanisterSnapshotMetadataResult, StartCanisterArgs, StopCanisterArgs, TakeCanisterSnapshotArgs, TakeCanisterSnapshotResult, UpdateSettingsArgs, UploadCanisterSnapshotDataArgs, UploadCanisterSnapshotMetadataArgs, - UploadCanisterSnapshotMetadataResult, UploadChunkArgs, UploadChunkResult, + UploadCanisterSnapshotMetadataResult, }; use snafu::{ResultExt, Snafu}; @@ -144,61 +143,6 @@ pub async fn install_code( .await } -pub async fn install_chunked_code( - agent: &Agent, - proxy: Option, - args: InstallChunkedCodeArgs, -) -> Result<(), UpdateOrProxyError> { - let effective = args.target_canister; - update_or_proxy::<_, ()>( - agent, - Principal::management_canister(), - "install_chunked_code", - (args,), - proxy, - Some(effective), - 0, - ) - .await -} - -pub async fn upload_chunk( - agent: &Agent, - proxy: Option, - args: UploadChunkArgs, -) -> Result { - let effective = args.canister_id; - let (result,): (UploadChunkResult,) = update_or_proxy( - agent, - Principal::management_canister(), - "upload_chunk", - (args,), - proxy, - Some(effective), - 0, - ) - .await?; - Ok(result) -} - -pub async fn clear_chunk_store( - agent: &Agent, - proxy: Option, - args: ClearChunkStoreArgs, -) -> Result<(), UpdateOrProxyError> { - let effective = args.canister_id; - update_or_proxy::<_, ()>( - agent, - Principal::management_canister(), - "clear_chunk_store", - (args,), - proxy, - Some(effective), - 0, - ) - .await -} - #[derive(Debug, Snafu)] pub enum FetchCanisterLogsError { #[snafu(display("failed to encode call arguments: {source}"))] diff --git a/crates/icp-cli/src/operations/sync.rs b/crates/icp-cli/src/operations/sync.rs index b24775f96..ba7a2cfd7 100644 --- a/crates/icp-cli/src/operations/sync.rs +++ b/crates/icp-cli/src/operations/sync.rs @@ -1,15 +1,23 @@ +use async_trait::async_trait; use candid::Principal; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::Agent; use icp::{ Canister, - canister::sync::{Params, Synchronize, SynchronizeError}, - package::PackageCache, - prelude::{Path, PathBuf}, + canister::recipe::RemoteResourceResolve, + canister::sync::{Synchronize, SynchronizeError}, + prelude::PathBuf, }; +use icp_deploy_canister::manifest::adapter::prebuilt::SourceField; +use icp_deploy_canister::sync_exec::{ + PluginExecutor, PluginExecutorError, PluginInvocation, ScriptInvocation, ScriptRunError, + ScriptRunner, StepProgress, +}; +use icp_deploy_canister::{SyncCanisterError, SyncStepContext, run_sync_steps}; use snafu::prelude::*; use std::collections::BTreeMap; use std::sync::Arc; +use tokio::sync::Mutex; use tracing::error; use crate::progress::{MultiStepProgressBar, ProgressManager, ProgressManagerSettings}; @@ -24,16 +32,108 @@ pub struct SyncOperationError { struct SyncFailure { canister_name: String, canister_id: Principal, - error: SynchronizeError, + error: SyncCanisterError, progress_output: Vec, } -/// Synchronizes a single canister using its configured sync steps +/// Per-canister mutable state guarded so the `&self` [`PluginExecutor`] can drive +/// the (mutable, sequential) progress bar. +struct SyncStepState<'a> { + pb: &'a mut MultiStepProgressBar, + /// 1-based index of the step about to run, for the progress header. + next: usize, +} + +/// Sync-step executor that runs a resolved step via the host [`Synchronize`] +/// implementation (WASI plugin / subprocess script) and frames it on the +/// canister's multi-step progress bar. The library owns the step loop and all +/// input derivation ([`run_sync_steps`]); this only performs the host action and +/// streams its output. +struct AgentSyncExecutor<'a> { + syncer: Arc, + agent: Agent, + resolver: Arc, + total: usize, + state: Mutex>, +} + +impl AgentSyncExecutor<'_> { + /// Frame a step on the shared progress bar: advance the counter, print the + /// header, run `f` against a fresh line sender, and close the step. Holding + /// the guard across `f` keeps steps framed sequentially on the shared bar. + async fn framed( + &self, + header: impl FnOnce(usize, usize) -> String, + f: F, + ) -> Result, SynchronizeError> + where + F: FnOnce(tokio::sync::mpsc::Sender) -> Fut, + Fut: Future, SynchronizeError>>, + { + let mut st = self.state.lock().await; + st.next += 1; + let header = header(st.next, self.total); + let tx = st.pb.begin_step(header); + let result = f(tx).await; + st.pb.end_step().await; + result + } +} + +#[async_trait] +impl PluginExecutor for AgentSyncExecutor<'_> { + async fn run_plugin( + &self, + invocation: PluginInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, PluginExecutorError> { + let src = match &invocation.source { + SourceField::Local(l) => format!("path: {}", l.path), + SourceField::Remote(r) => format!("url: {}", r.url), + }; + self.framed( + |n, total| format!("\nSyncing: plugin {src} {n} of {total}"), + |tx| async move { + self.syncer + .run_plugin(&invocation, &self.agent, Some(tx), self.resolver.as_ref()) + .await + }, + ) + .await + .map_err(|source| PluginExecutorError { + source: Box::new(source), + }) + } +} + +#[async_trait] +impl ScriptRunner for AgentSyncExecutor<'_> { + async fn run_script( + &self, + invocation: ScriptInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, ScriptRunError> { + let desc = invocation.commands.join("\n"); + self.framed( + |n, total| format!("\nSyncing: script {desc} {n} of {total}"), + |tx| async move { self.syncer.run_script(&invocation, Some(tx)).await }, + ) + .await + .map_err(|source| ScriptRunError { + source: Box::new(source), + }) + } +} + +/// Synchronize a single canister's steps through the library, framing progress +/// on `pb`. Environment variables are applied separately by the caller. +#[allow(clippy::too_many_arguments)] async fn sync_canister( - syncer: &Arc, - agent: &Agent, + syncer: Arc, + resolver: Arc, + agent: Agent, canister_path: PathBuf, - project_dir: &Path, + project_dir: PathBuf, canister_id: Principal, canister_info: &Canister, environment: &str, @@ -41,50 +141,32 @@ async fn sync_canister( canister_ids: &BTreeMap, proxy: Option, pb: &mut MultiStepProgressBar, - pkg_cache: &PackageCache, -) -> Result, SynchronizeError> { - let step_count = canister_info.sync.steps.len(); - let mut stderr_lines = Vec::new(); - - for (i, step) in canister_info.sync.steps.iter().enumerate() { - // Indicate to user the current step being executed - let current_step = i + 1; - let pb_hdr = format!("\nSyncing: {step} {current_step} of {step_count}"); - - let tx = pb.begin_step(pb_hdr); - - // Execute step - let sync_result = syncer - .sync( - step, - &Params { - path: canister_path.clone(), - project_dir: project_dir.to_path_buf(), - cid: canister_id, - name: canister_info.name.clone(), - environment: environment.to_owned(), - network: network.to_owned(), - canister_ids: canister_ids.clone(), - proxy, - }, - agent, - Some(tx), - pkg_cache, - ) - .await; - - // Ensure background receiver drains all messages - pb.end_step().await; - - stderr_lines.extend(sync_result?); - } - - Ok(stderr_lines) +) -> Result, SyncCanisterError> { + let ctx = SyncStepContext { + canister_path, + project_dir, + canister_id, + canister_name: canister_info.name.clone(), + environment: environment.to_owned(), + network: network.to_owned(), + canister_ids: canister_ids.clone(), + proxy, + }; + let executor = AgentSyncExecutor { + syncer, + agent, + resolver, + total: canister_info.sync.steps.len(), + state: Mutex::new(SyncStepState { pb, next: 0 }), + }; + run_sync_steps(canister_info, &ctx, &executor, &executor, None).await } /// Orchestrates syncing multiple canisters with progress tracking +#[allow(clippy::too_many_arguments)] pub(crate) async fn sync_many( syncer: Arc, + resolver: Arc, agent: Agent, canisters: Vec<(Principal, PathBuf, Canister)>, project_dir: PathBuf, @@ -93,7 +175,6 @@ pub(crate) async fn sync_many( canister_ids: BTreeMap, proxy: Option, debug: bool, - pkg_cache: &PackageCache, ) -> Result<(), SyncOperationError> { let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); @@ -104,18 +185,19 @@ pub(crate) async fn sync_many( let fut = { let agent = agent.clone(); let syncer = syncer.clone(); + let resolver = resolver.clone(); let environment = environment.clone(); let network = network.clone(); let canister_ids = canister_ids.clone(); let project_dir = project_dir.clone(); async move { - // Define the sync logic let sync_result = sync_canister( - &syncer, - &agent, + syncer, + resolver, + agent, canister_path, - &project_dir, + project_dir, cid, &canister_info, &environment, @@ -123,7 +205,6 @@ pub(crate) async fn sync_many( &canister_ids, proxy, &mut pb, - pkg_cache, ) .await; diff --git a/crates/icp-cli/tests/recipe_tests.rs b/crates/icp-cli/tests/recipe_tests.rs index 85a889b2c..e10554f00 100644 --- a/crates/icp-cli/tests/recipe_tests.rs +++ b/crates/icp-cli/tests/recipe_tests.rs @@ -332,3 +332,67 @@ fn recipe_local_file_valid_checksum() { .assert() .success(); } + +/// A canister may declare sync steps alongside a recipe; they land after the +/// steps the recipe renders. +#[test] +fn recipe_with_manifest_sync_steps() { + let ctx = TestContext::new(); + + // Setup project + let project_dir = ctx.create_project_dir("icp"); + + // Recipe rendering a sync step of its own + write_string( + &project_dir.join("recipe.hbs"), // path + indoc! {r#" + build: + steps: + - type: script + command: echo "test" > "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: script + command: echo from-recipe + "#}, // contents + ) + .expect("failed to write recipe template"); + + let pm = indoc! {" + canisters: + - name: my-canister + recipe: + type: file://./recipe.hbs + sync: + steps: + - type: script + command: echo from-manifest + "}; + + write_string( + &project_dir.join("icp.yaml"), // path + pm, // contents + ) + .expect("failed to write project manifest"); + + // The effective configuration holds both steps, the recipe's first + let assert = ctx + .icp() + .current_dir(project_dir) + .args(["project", "show"]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()) + .expect("`icp project show` output is not UTF-8"); + + let recipe_at = stdout + .find("echo from-recipe") + .unwrap_or_else(|| panic!("recipe's sync step missing from:\n{stdout}")); + let manifest_at = stdout + .find("echo from-manifest") + .unwrap_or_else(|| panic!("manifest's sync step missing from:\n{stdout}")); + assert!( + recipe_at < manifest_at, + "the manifest's sync step should follow the recipe's, got:\n{stdout}" + ); +} diff --git a/crates/icp-cli/tests/sync_tests.rs b/crates/icp-cli/tests/sync_tests.rs index 14afac03d..13545871f 100644 --- a/crates/icp-cli/tests/sync_tests.rs +++ b/crates/icp-cli/tests/sync_tests.rs @@ -448,7 +448,8 @@ async fn sync_plugin_registers_seed_data() { .args(["deploy", "--environment", "random-environment"]) .assert() .success() - .stderr(contains("candid:service: absent")); + .stderr(contains("candid:service: absent")) + .stderr(contains("SEEDED_BY=random-environment")); // Query the canister to verify all three fruits were registered ctx.icp() @@ -470,6 +471,27 @@ async fn sync_plugin_registers_seed_data() { .and(contains("banana")) .and(contains("cherry")), ); + + // The plugin's environment variable really landed in the canister's + // settings, alongside the PUBLIC_CANISTER_ID binding deploy writes itself — + // proving the host read the current list and wrote it back with the new + // variable added, rather than replacing it. + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "settings", + "show", + "my-canister", + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout( + contains("SEEDED_BY: random-environment") + .and(contains("PUBLIC_CANISTER_ID:my-canister")), + ); } /// `dirs:` may be written as a map (name → path, or name → list of paths) @@ -953,6 +975,10 @@ async fn sync_plugin_routes_through_proxy() { // This manifest skips the example's ic-wasm step, so the section really is // missing — and the host must report the resulting rejection as an absent // section, the same answer a direct read proves from the certificate. + // + // Its environment-variable write is proxied as well: both the settings read + // and the settings write are made by the proxy, which is a controller, so + // the pair is checked against the proxy rather than the user identity. ctx.icp() .current_dir(&project_dir) .args([ @@ -964,7 +990,8 @@ async fn sync_plugin_routes_through_proxy() { ]) .assert() .success() - .stderr(contains("candid:service: absent")); + .stderr(contains("candid:service: absent")) + .stderr(contains("SEEDED_BY=random-environment")); // Query the canister to verify all three fruits were registered ctx.icp() diff --git a/crates/icp-deploy-canister/Cargo.toml b/crates/icp-deploy-canister/Cargo.toml new file mode 100644 index 000000000..bfe5f5792 --- /dev/null +++ b/crates/icp-deploy-canister/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "icp-deploy-canister" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +# This crate is intentionally dependency-light so that its install/sync core and +# project model can be compiled into a canister. All host-only IO (filesystem, +# HTTP, the ICP API, sync-step execution, the canister-id store) is abstracted +# behind trait objects; do NOT add ic-agent, reqwest, tokio, wasmtime, keyring, +# bollard, sysinfo, or std::fs-driven crates here. + +[features] +# Enables `clap::ValueEnum` derives on manifest enums used as CLI value types +# (e.g. `ArgsFormat`). Enabled transitively by `icp/clap`. +clap = ["dep:clap"] + +[dependencies] +async-trait.workspace = true +bigdecimal.workspace = true +camino.workspace = true +candid.workspace = true +clap = { workspace = true, optional = true } +candid_parser.workspace = true +futures.workspace = true +glob.workspace = true +handlebars.workspace = true +hex.workspace = true +ic-management-canister-types.workspace = true +indexmap.workspace = true +itertools.workspace = true +num-bigint.workspace = true +num-integer.workspace = true +num-traits.workspace = true +pathdiff.workspace = true +schemars.workspace = true +serde.workspace = true +sha2.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true +snafu.workspace = true +strum.workspace = true +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +camino-tempfile.workspace = true +indoc.workspace = true +jsonschema.workspace = true +tokio.workspace = true diff --git a/crates/icp-deploy-canister/src/canister/mod.rs b/crates/icp-deploy-canister/src/canister/mod.rs new file mode 100644 index 000000000..e8631dbca --- /dev/null +++ b/crates/icp-deploy-canister/src/canister/mod.rs @@ -0,0 +1,684 @@ +use std::collections::HashMap; + +use candid::{Nat, Principal}; +use ic_management_canister_types::{CanisterSettings, LogVisibility}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + parsers::{CyclesAmount, DurationAmount, MemoryAmount}, + prelude::*, +}; + +pub mod recipe; + +/// Controls who can read canister logs. +/// Supports both string format ("controllers", "public") and object format ({ allowed_viewers: [...] }). +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(untagged)] +pub enum LogVisibilityDef { + /// Simple string variants for controllers or public + Simple(LogVisibilitySimple), + /// Object format with allowed_viewers list + AllowedViewers { allowed_viewers: Vec }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum LogVisibilitySimple { + Controllers, + Public, +} + +impl<'de> Deserialize<'de> for LogVisibilityDef { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::{Error, MapAccess, Visitor}; + use std::fmt; + + struct LogVisibilityVisitor; + + impl<'de> Visitor<'de> for LogVisibilityVisitor { + type Value = LogVisibilityDef; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("'controllers', 'public', or object with 'allowed_viewers'") + } + + fn visit_str(self, value: &str) -> Result { + LogVisibilitySimple::deserialize( + serde::de::value::StrDeserializer::::new(value), + ) + .map(LogVisibilityDef::Simple) + .map_err(|_| { + E::custom(format!( + "unknown log_visibility value: '{}', expected 'controllers' or 'public'", + value + )) + }) + } + + fn visit_map(self, mut map: M) -> Result + where + M: MapAccess<'de>, + { + let mut allowed_viewers: Option> = None; + + while let Some(key) = map.next_key::()? { + match key.as_str() { + "allowed_viewers" => { + if allowed_viewers.is_some() { + return Err(Error::duplicate_field("allowed_viewers")); + } + allowed_viewers = Some(map.next_value()?); + } + _ => { + return Err(Error::unknown_field(&key, &["allowed_viewers"])); + } + } + } + + allowed_viewers + .map(|v| LogVisibilityDef::AllowedViewers { allowed_viewers: v }) + .ok_or_else(|| Error::missing_field("allowed_viewers")) + } + } + + deserializer.deserialize_any(LogVisibilityVisitor) + } +} + +impl JsonSchema for LogVisibilityDef { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::borrow::Cow::Borrowed("LogVisibility") + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "Controls who can read canister logs.", + "oneOf": [ + { + "type": "string", + "enum": ["controllers", "public"], + "description": "Simple log visibility: 'controllers' (only controllers can view) or 'public' (anyone can view)" + }, + { + "type": "object", + "properties": { + "allowed_viewers": { + "type": "array", + "items": { + "type": "string", + "description": "A principal ID that can view logs" + }, + "description": "List of principal IDs that can view canister logs" + } + }, + "required": ["allowed_viewers"], + "additionalProperties": false, + "description": "Specific principals that can view logs" + } + ] + }) + } +} + +impl From for LogVisibility { + fn from(value: LogVisibilityDef) -> Self { + match value { + LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) => { + LogVisibility::Controllers + } + LogVisibilityDef::Simple(LogVisibilitySimple::Public) => LogVisibility::Public, + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + LogVisibility::AllowedViewers(allowed_viewers) + } + } + } +} + +/// A reference to a controller: either an explicit principal or a canister name in this project. +/// +/// During deserialization, principal text format is tried first; strings that don't parse as a +/// principal are treated as canister names. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ControllerRef { + /// An explicitly specified principal (e.g. "2vxsx-fae") + Principal(candid::Principal), + /// A canister name from the same project (e.g. "my_canister") + CanisterName(String), +} + +impl ControllerRef { + /// Resolve to a `Principal` using the provided ID mapping. + /// Returns `None` if this is a `CanisterName` not present in `ids`. + pub fn resolve(&self, ids: &crate::ids::IdMapping) -> Option { + match self { + ControllerRef::Principal(p) => Some(*p), + ControllerRef::CanisterName(name) => ids.get(name).copied(), + } + } + + /// If this is a `CanisterName`, returns the name; otherwise `None`. + pub fn canister_name(&self) -> Option<&str> { + match self { + ControllerRef::CanisterName(n) => Some(n), + ControllerRef::Principal(_) => None, + } + } +} + +/// Partition a slice of controller references into resolved principals and unresolved canister +/// names, using `ids` for name lookup. +pub fn resolve_controllers( + crefs: &[ControllerRef], + ids: &crate::ids::IdMapping, +) -> (Vec, Vec) { + let mut resolved = Vec::new(); + let mut unresolved = Vec::new(); + for cref in crefs { + match cref.resolve(ids) { + Some(p) => resolved.push(p), + None => { + if let Some(name) = cref.canister_name() { + unresolved.push(name.to_owned()); + } + } + } + } + (resolved, unresolved) +} + +impl schemars::JsonSchema for ControllerRef { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::borrow::Cow::Borrowed("ControllerRef") + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "description": "A controller: either a principal text (e.g. '2vxsx-fae') or a canister name in this project (e.g. 'my_canister')" + }) + } +} + +/// An environment variable value as written in a manifest. +/// +/// A plain scalar is the value itself: +/// ```yaml +/// environment_variables: +/// API_ENDPOINT: https://api.example.com +/// ``` +/// +/// The object form reads the value from a file, relative to the canister's own +/// directory — including when an environment overrides the variable, matching how +/// an `init_args` override resolves its path: +/// ```yaml +/// environment_variables: +/// API_KEY: +/// path: ./secrets/api-key +/// ``` +#[derive(Clone, Debug, PartialEq, Deserialize, JsonSchema, Serialize)] +#[serde(untagged, expecting = "a string, or `{ path: }`")] +pub enum ManifestEnvVar { + /// The value, written inline. + Value(String), + /// A file holding the value. Surrounding whitespace is trimmed off the + /// file's contents, so a trailing newline does not become part of the value. + Path { + #[schemars(with = "String")] + path: PathBuf, + }, +} + +impl Default for ManifestEnvVar { + fn default() -> Self { + Self::Value(String::new()) + } +} + +/// Canister settings loaded from a manifest, before file-backed environment +/// variable values have been read. See [`Settings`] for the resolved form. +pub type ManifestSettings = Settings; + +/// Canister settings, such as compute and memory allocation. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Settings { + /// Controls who can read canister logs. + #[serde(skip_serializing_if = "Option::is_none")] + pub log_visibility: Option, + + /// Compute allocation (0 to 100). Represents guaranteed compute capacity. + #[serde(skip_serializing_if = "Option::is_none")] + pub compute_allocation: Option, + + /// Memory allocation in bytes. If unset, memory is allocated dynamically. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_allocation: Option, + + /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. + /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). + #[serde(skip_serializing_if = "Option::is_none")] + pub freezing_threshold: Option, + + /// Upper limit on cycles reserved for future resource payments. + /// Memory allocations that would push the reserved balance above this limit will fail. + /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reserved_cycles_limit: Option, + + /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub wasm_memory_limit: Option, + + /// Wasm memory threshold in bytes. Triggers a callback when exceeded. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub wasm_memory_threshold: Option, + + /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. + /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub log_memory_limit: Option, + + /// Environment variables for the canister as key-value pairs. + /// These variables are accessible within the canister and can be used to configure + /// behavior without hardcoding values in the WASM module. + /// A value may also be read from a file with `{ path: }`. + #[serde(skip_serializing_if = "Option::is_none")] + pub environment_variables: Option>, + + /// Controllers for this canister. Each entry is either a principal text + /// (e.g. "2vxsx-fae") or the name of another canister in this project. + /// Named canisters that do not yet exist will be set as controllers once created. + #[serde(default)] + pub controllers: Option>, +} + +impl From for ManifestSettings { + fn from(settings: Settings) -> Self { + let Settings { + log_visibility, + compute_allocation, + memory_allocation, + freezing_threshold, + reserved_cycles_limit, + wasm_memory_limit, + wasm_memory_threshold, + log_memory_limit, + environment_variables, + controllers, + } = settings; + + Self { + log_visibility, + compute_allocation, + memory_allocation, + freezing_threshold, + reserved_cycles_limit, + wasm_memory_limit, + wasm_memory_threshold, + log_memory_limit, + environment_variables: environment_variables.map(|vars| { + vars.into_iter() + .map(|(name, value)| (name, ManifestEnvVar::Value(value))) + .collect() + }), + controllers, + } + } +} + +impl From for CanisterSettings { + fn from(settings: Settings) -> Self { + CanisterSettings { + freezing_threshold: settings.freezing_threshold.map(|d| Nat::from(d.get())), + controllers: None, + reserved_cycles_limit: settings.reserved_cycles_limit.map(|c| Nat::from(c.get())), + log_visibility: settings.log_visibility.map(Into::into), + memory_allocation: settings.memory_allocation.map(|m| Nat::from(m.get())), + compute_allocation: settings.compute_allocation.map(Nat::from), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use indoc::indoc; + + use super::*; + + #[test] + fn log_visibility_deserialize_controllers() { + let yaml = "controllers"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) + ); + } + + #[test] + fn log_visibility_deserialize_public() { + let yaml = "public"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + LogVisibilityDef::Simple(LogVisibilitySimple::Public) + ); + } + + #[test] + fn log_visibility_deserialize_allowed_viewers() { + let yaml = r#" +allowed_viewers: + - "aaaaa-aa" + - "2vxsx-fae" +"#; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + match result { + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + assert_eq!(allowed_viewers.len(), 2); + assert_eq!( + allowed_viewers[0], + Principal::from_text("aaaaa-aa").unwrap() + ); + assert_eq!( + allowed_viewers[1], + Principal::from_text("2vxsx-fae").unwrap() + ); + } + _ => panic!("Expected AllowedViewers variant"), + } + } + + #[test] + fn log_visibility_deserialize_allowed_viewers_empty() { + let yaml = "allowed_viewers: []"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + match result { + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + assert!(allowed_viewers.is_empty()); + } + _ => panic!("Expected AllowedViewers variant"), + } + } + + #[test] + fn log_visibility_deserialize_invalid_string() { + let yaml = "invalid"; + let result: Result = serde_yaml::from_str(yaml); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("unknown log_visibility value")); + } + + #[test] + fn log_visibility_deserialize_invalid_field() { + let yaml = "unknown_field: []"; + let result: Result = serde_yaml::from_str(yaml); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("unknown field")); + } + + #[test] + fn log_visibility_serialize_controllers() { + let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert_eq!(yaml.trim(), "controllers"); + } + + #[test] + fn log_visibility_serialize_public() { + let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Public); + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert_eq!(yaml.trim(), "public"); + } + + #[test] + fn log_visibility_serialize_allowed_viewers() { + let log_vis = LogVisibilityDef::AllowedViewers { + allowed_viewers: vec![ + Principal::from_text("aaaaa-aa").unwrap(), + Principal::from_text("2vxsx-fae").unwrap(), + ], + }; + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert!(yaml.contains("allowed_viewers")); + assert!(yaml.contains("aaaaa-aa")); + assert!(yaml.contains("2vxsx-fae")); + } + + #[test] + fn settings_reserved_cycles_limit_parses_suffix() { + let yaml = "reserved_cycles_limit: 4.3t"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.reserved_cycles_limit.as_ref().map(|c| c.get()), + Some(4_300_000_000_000) + ); + } + + #[test] + fn settings_reserved_cycles_limit_parses_number() { + let yaml = "reserved_cycles_limit: 5000000000000"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.reserved_cycles_limit.as_ref().map(|c| c.get()), + Some(5_000_000_000_000) + ); + } + + #[test] + fn settings_memory_allocation_parses_suffix() { + let yaml = "memory_allocation: 4gib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.memory_allocation.as_ref().map(|m| m.get()), + Some(4 * 1024 * 1024 * 1024) + ); + } + + #[test] + fn settings_memory_allocation_parses_number() { + let yaml = "memory_allocation: 4294967296"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.memory_allocation.as_ref().map(|m| m.get()), + Some(4294967296) + ); + } + + #[test] + fn settings_wasm_memory_limit_parses_suffix() { + let yaml = "wasm_memory_limit: 1.5gib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.wasm_memory_limit.as_ref().map(|m| m.get()), + Some(1610612736) + ); + } + + #[test] + fn settings_log_memory_limit_parses_suffix() { + let yaml = "log_memory_limit: 256kib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.log_memory_limit.as_ref().map(|m| m.get()), + Some(256 * 1024) + ); + } + + #[test] + fn settings_log_memory_limit_parses_mib() { + let yaml = "log_memory_limit: 2mib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.log_memory_limit.as_ref().map(|m| m.get()), + Some(2 * 1024 * 1024) + ); + } + + #[test] + fn settings_environment_variables_take_values_or_files() { + let yaml = indoc! {r#" + environment_variables: + API_ENDPOINT: https://api.example.com + API_KEY: + path: ./secrets/api-key + "#}; + let settings: ManifestSettings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.environment_variables, + Some(HashMap::from([ + ( + "API_ENDPOINT".to_owned(), + ManifestEnvVar::Value("https://api.example.com".to_owned()), + ), + ( + "API_KEY".to_owned(), + ManifestEnvVar::Path { + path: "./secrets/api-key".into(), + }, + ), + ])), + ); + } + + #[test] + fn settings_environment_variable_rejects_unknown_object_form() { + let yaml = indoc! {r#" + environment_variables: + API_KEY: + file: ./secrets/api-key + "#}; + let err = serde_yaml::from_str::(yaml) + .expect_err("only the `path` object form is accepted"); + assert!( + err.to_string().contains("a string, or `{ path: }`"), + "unhelpful error: {err}" + ); + } + + /// A value of the wrong scalar type reports what is accepted, rather than + /// serde's default "did not match any variant" for an untagged enum. + #[test] + fn settings_environment_variable_rejects_non_string_scalar() { + let err = + serde_yaml::from_str::("environment_variables:\n PORT: 8080\n") + .expect_err("a bare integer is not a value"); + assert!( + err.to_string().contains("a string, or `{ path: }`"), + "unhelpful error: {err}" + ); + } + + #[test] + fn resolved_settings_serialize_environment_variables_inline() { + let settings = Settings { + environment_variables: Some(HashMap::from([( + "API_KEY".to_owned(), + "s3cret".to_owned(), + )])), + ..Default::default() + }; + let yaml = serde_yaml::to_string(&ManifestSettings::from(settings)).unwrap(); + assert!( + yaml.contains("environment_variables:\n API_KEY: s3cret\n"), + "unexpected yaml: {yaml}" + ); + } + + #[test] + fn controller_ref_deserializes_principal() { + let yaml = "\"2vxsx-fae\""; + let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + ControllerRef::Principal(Principal::from_text("2vxsx-fae").unwrap()) + ); + } + + #[test] + fn controller_ref_deserializes_canister_name() { + let yaml = "\"my_canister\""; + let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + ControllerRef::CanisterName("my_canister".to_owned()) + ); + } + + #[test] + fn controller_ref_resolve_principal() { + let p = Principal::from_text("aaaaa-aa").unwrap(); + let cref = ControllerRef::Principal(p); + let ids = crate::ids::IdMapping::new(); + assert_eq!(cref.resolve(&ids), Some(p)); + } + + #[test] + fn controller_ref_resolve_canister_name_present() { + let p = Principal::from_text("aaaaa-aa").unwrap(); + let cref = ControllerRef::CanisterName("backend".to_owned()); + let mut ids = crate::ids::IdMapping::new(); + ids.insert("backend".to_owned(), p); + assert_eq!(cref.resolve(&ids), Some(p)); + } + + #[test] + fn controller_ref_resolve_canister_name_absent() { + let cref = ControllerRef::CanisterName("backend".to_owned()); + let ids = crate::ids::IdMapping::new(); + assert_eq!(cref.resolve(&ids), None); + } + + #[test] + fn settings_controllers_parses_mixed() { + let yaml = r#" +controllers: + - "aaaaa-aa" + - "my_other_canister" +"#; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + let controllers = settings.controllers.unwrap(); + assert_eq!(controllers.len(), 2); + assert_eq!( + controllers[0], + ControllerRef::Principal(Principal::from_text("aaaaa-aa").unwrap()) + ); + assert_eq!( + controllers[1], + ControllerRef::CanisterName("my_other_canister".to_owned()) + ); + } + + #[test] + fn log_visibility_conversion_to_ic_type() { + let controllers = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); + let ic_controllers: LogVisibility = controllers.into(); + assert!(matches!(ic_controllers, LogVisibility::Controllers)); + + let public = LogVisibilityDef::Simple(LogVisibilitySimple::Public); + let ic_public: LogVisibility = public.into(); + assert!(matches!(ic_public, LogVisibility::Public)); + + let viewers = LogVisibilityDef::AllowedViewers { + allowed_viewers: vec![Principal::from_text("aaaaa-aa").unwrap()], + }; + let ic_viewers: LogVisibility = viewers.into(); + match ic_viewers { + LogVisibility::AllowedViewers(v) => { + assert_eq!(v.len(), 1); + } + _ => panic!("Expected AllowedViewers"), + } + } +} diff --git a/crates/icp-deploy-canister/src/canister/recipe/mod.rs b/crates/icp-deploy-canister/src/canister/recipe/mod.rs new file mode 100644 index 000000000..9ca7b5852 --- /dev/null +++ b/crates/icp-deploy-canister/src/canister/recipe/mod.rs @@ -0,0 +1,451 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use handlebars::{Context, Handlebars, Helper, HelperDef, HelperResult, Output, RenderContext}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use snafu::prelude::*; + +use crate::files::FileAccess; +use crate::manifest::{ + adapter::prebuilt::SourceField, + canister::{BuildSteps, SyncSteps}, + recipe::{Recipe, RecipeType}, +}; +use crate::prelude::*; +use crate::sync_exec::StepProgress; + +/// Context passed to a recipe resolver, describing the canister being built. +/// +/// Serializes to the shape injected into recipe templates under the `_` namespace: +/// +/// ```yaml +/// canister: +/// name: +/// ``` +pub struct RecipeContext { + pub canister_name: String, +} + +impl RecipeContext { + /// Builds the YAML value injected into recipe templates under the `_` namespace. + /// Constructing the mapping directly is infallible, unlike `serde` serialization. + pub fn to_yaml(&self) -> serde_yaml::Value { + use serde_yaml::{Mapping, Value}; + + let mut canister = Mapping::new(); + canister.insert("name".into(), Value::String(self.canister_name.clone())); + + let mut root = Mapping::new(); + root.insert("canister".into(), Value::Mapping(canister)); + + Value::Mapping(root) + } +} + +/// Fetches the remote resources a project references — recipe templates and +/// plugin wasms — retrieving them over HTTP and caching as needed. +/// +/// The concrete resolver (which owns the HTTP client and the package cache) +/// lives in the host `icp` crate; this crate defines the interface, and renders +/// fetched recipe templates itself (see [`render_recipe`]), so that +/// consolidation and sync can call an injected resolver. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait RemoteResourceResolve: Sync + Send { + /// Fetch a recipe's Handlebars template, returning its raw source. Callers + /// render it into build/sync steps with [`render_recipe`], then hand the + /// result back to [`commit_recipe`](Self::commit_recipe). + async fn resolve_recipe(&self, recipe: &Recipe) -> Result; + + /// Accept a template from [`resolve_recipe`](Self::resolve_recipe) once it + /// has rendered successfully, letting the resolver commit whatever it held + /// back — for the host resolver, writing a fresh download to the package + /// cache. A resolver that never sets [`FetchedRecipe::deferred`] has nothing + /// to commit and implements this as `Ok(())`. + async fn commit_recipe( + &self, + recipe: &Recipe, + fetched: &FetchedRecipe, + ) -> Result<(), ResolveError>; + + /// Resolve a plugin wasm `source` (relative to `base_dir`) to a location the + /// host's [`PluginExecutor`](crate::sync_exec::PluginExecutor) can load, + /// verifying `sha256` and caching a remote download. `progress` receives + /// status lines. + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + progress: Option<&dyn StepProgress>, + ) -> Result; +} + +/// A recipe template retrieved by a [`RemoteResourceResolve`]. +/// +/// A resolver that caches downloads must not commit one before the template is +/// known to render: for an unpinned URL a single malformed response would +/// otherwise become the cached entry that every later project load reuses. Such +/// a resolver returns the template with `deferred` set and waits for +/// [`RemoteResourceResolve::commit_recipe`]. +pub struct FetchedRecipe { + /// Raw Handlebars template source. + pub template: String, + + /// Whether the resolver is holding work back until the caller confirms the + /// template renders. + pub deferred: bool, +} + +#[derive(Debug, Snafu)] +pub enum ResolveError { + /// The injected resolver failed. The concrete source (e.g. a fetch/cache + /// error from the host resolver) is boxed because this crate does not depend + /// on the resolver's implementation. + #[snafu(display("failed to fetch recipe template"))] + Resolve { + source: Box, + }, + + #[snafu(display("failed to resolve plugin wasm"))] + ResolveWasm { + source: Box, + }, +} + +/// A [`RemoteResourceResolve`] that serves only local resources — recipe files +/// read through the injected [`FileAccess`], and plugin wasms already on hand — +/// rejecting anything that would have to be fetched. For environments with no +/// HTTP access. +pub struct NoResolve(pub F); + +#[derive(Debug, Snafu)] +#[snafu(display("remote recipes are not allowed"))] +pub struct NoResolveError; + +#[derive(Debug, Snafu)] +#[snafu(display("remote modules are not allowed"))] +pub struct NoResolveModuleError; + +#[derive(Debug, Snafu)] +#[snafu(display( + "sha256 checksum mismatch for plugin wasm '{path}': expected {expected}, actual {actual}" +))] +pub struct WasmChecksumMismatchError { + path: PathBuf, + expected: String, + actual: String, +} + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +impl RemoteResourceResolve for NoResolve { + async fn resolve_recipe(&self, recipe: &Recipe) -> Result { + match &recipe.recipe_type { + RecipeType::File(path) => { + let template = self.0.read_to_string(path.as_ref()).await.map_err(|e| { + ResolveError::Resolve { + source: Box::new(e), + } + })?; + Ok(FetchedRecipe { + template, + deferred: false, + }) + } + _ => Err(ResolveError::Resolve { + source: Box::new(NoResolveError), + }), + } + } + + /// Local reads defer nothing. + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + Ok(()) + } + + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + progress: Option<&dyn StepProgress>, + ) -> Result { + let SourceField::Local(source) = source else { + return Err(ResolveError::ResolveWasm { + source: Box::new(NoResolveModuleError), + }); + }; + let path = base_dir.join(&source.path); + + if let Some(expected) = sha256 { + if let Some(p) = progress { + p.line(format!("Reading wasm: {path}")); + } + let bytes = self + .0 + .read_file(&path) + .await + .map_err(|e| ResolveError::ResolveWasm { + source: Box::new(e), + })?; + if let Some(p) = progress { + p.line("Verifying checksum".to_string()); + } + let actual = hex::encode(Sha256::digest(&bytes)); + if actual != expected { + return Err(ResolveError::ResolveWasm { + source: Box::new( + WasmChecksumMismatchSnafu { + path, + expected, + actual, + } + .build(), + ), + }); + } + } + Ok(path) + } +} + +#[derive(Debug, Snafu)] +pub enum RenderRecipeError { + #[snafu(display("recipe template for '{recipe}' failed to render"))] + Render { + source: handlebars::RenderError, + recipe: RecipeType, + }, + + #[snafu(display("recipe '{recipe}' did not render into a valid build/sync manifest"))] + Parse { + source: serde_yaml::Error, + recipe: RecipeType, + }, +} + +/// Render a recipe's Handlebars `template` into concrete build/sync steps. +/// +/// The template is rendered with the recipe's `configuration` plus the reserved +/// `_` namespace (the `_` key always overrides any user-supplied value), then the +/// resulting YAML is parsed. A recipe may only produce `build` and `sync`. +#[allow(clippy::result_large_err)] +pub fn render_recipe( + template: &str, + recipe: &Recipe, + recipe_context: &RecipeContext, +) -> Result<(BuildSteps, SyncSteps), RenderRecipeError> { + let mut reg = Handlebars::new(); + // The output is YAML, not HTML, so disable HTML escaping. + reg.register_escape_fn(handlebars::no_escape); + reg.register_helper("replace", Box::new(ReplaceHelper)); + // Reject unset template variables. + reg.set_strict_mode(true); + + // User-provided configuration plus the injected `_.*` variables. The `_` key + // is reserved and always overrides any user-supplied value. + let mut render_context: HashMap = recipe.configuration.clone(); + render_context.insert("_".to_string(), recipe_context.to_yaml()); + + let out = reg + .render_template(template, &render_context) + .context(RenderSnafu { + recipe: recipe.recipe_type.clone(), + })?; + + // Recipes can only render `build`/`sync`. + #[derive(Deserialize)] + struct BuildSyncHelper { + build: BuildSteps, + #[serde(default)] + sync: SyncSteps, + } + + let helper: BuildSyncHelper = serde_yaml::from_str(&out).context(ParseSnafu { + recipe: recipe.recipe_type.clone(), + })?; + Ok((helper.build, helper.sync)) +} + +/// Handlebars helper for string replacement operations. +/// Usage: `{{ replace "from" "to" value }}` +#[derive(Clone, Copy)] +struct ReplaceHelper; + +impl HelperDef for ReplaceHelper { + fn call<'reg: 'rc, 'rc>( + &self, + h: &Helper, + _: &'reg Handlebars<'reg>, + _: &Context, + _: &mut RenderContext<'reg, 'rc>, + out: &mut dyn Output, + ) -> HelperResult { + let (from, to) = (h.param(0).unwrap().render(), h.param(1).unwrap().render()); + let v = h.param(2).unwrap().render(); + out.write(&v.replace(&from, &to))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::adapter::prebuilt::LocalSource; + use crate::manifest::canister::BuildStep; + use crate::testutil::HostFiles; + + fn recipe(config: &[(&str, &str)]) -> Recipe { + Recipe { + recipe_type: RecipeType::File("recipe.hbs".to_owned()), + configuration: config + .iter() + .map(|(k, v)| ((*k).to_owned(), serde_yaml::Value::String((*v).to_owned()))) + .collect(), + sha256: None, + } + } + + fn ctx(name: &str) -> RecipeContext { + RecipeContext { + canister_name: name.to_owned(), + } + } + + /// The only build step's command, for a recipe that renders a single script step. + fn rendered_command(template: &str, recipe: &Recipe, context: &RecipeContext) -> String { + let (build, _sync) = render_recipe(template, recipe, context).unwrap(); + match &build.steps[0] { + BuildStep::Script(adapter) => adapter.command.as_vec()[0].clone(), + other => panic!("expected a script build step, got {other:?}"), + } + } + + /// Interpolated values are not HTML-escaped (the output is YAML): `=` and `&` + /// must survive. + #[test] + fn template_values_are_not_html_escaped() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "{{ command }}" + "#}; + let r = recipe(&[("command", "SITE=https://example.com&foo=bar npm run build")]); + assert_eq!( + rendered_command(template, &r, &ctx("my-canister")), + "SITE=https://example.com&foo=bar npm run build" + ); + } + + /// The canister name is injected under the reserved `_` namespace. + #[test] + fn canister_name_is_injected() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "build {{_.canister.name}}" + "#}; + assert_eq!( + rendered_command(template, &recipe(&[]), &ctx("my-canister")), + "build my-canister" + ); + } + + /// The `_` namespace works through the `replace` helper. + #[test] + fn canister_name_works_with_replace_helper() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "cp {{ replace "-" "_" _.canister.name }}.wasm out.wasm" + "#}; + assert_eq!( + rendered_command(template, &recipe(&[]), &ctx("my-canister")), + "cp my_canister.wasm out.wasm" + ); + } + + /// User configuration cannot override the reserved `_` namespace. + #[test] + fn reserved_namespace_cannot_be_overridden_by_user_config() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "build {{_.canister.name}}" + "#}; + let mut r = recipe(&[]); + r.configuration.insert( + "_".to_owned(), + serde_yaml::from_str("canister:\n name: user-override").unwrap(), + ); + assert_eq!( + rendered_command(template, &r, &ctx("real-name")), + "build real-name" + ); + } + + /// A template referencing an unset variable is a `Render` error, because + /// strict mode is on. + #[test] + fn unset_template_variable_is_a_render_error() { + let template = indoc::indoc! {r#" + build: + steps: + - type: script + command: "{{ never_set }}" + "#}; + assert!(matches!( + render_recipe(template, &recipe(&[]), &ctx("c")), + Err(RenderRecipeError::Render { .. }) + )); + } + + /// A template that renders to invalid build/sync YAML is a `Parse` error, + /// not a panic. + #[test] + fn invalid_rendered_yaml_is_a_parse_error() { + let template = "not: a valid build manifest\n"; + assert!(matches!( + render_recipe(template, &recipe(&[]), &ctx("c")), + Err(RenderRecipeError::Parse { .. }) + )); + } + + /// A plugin wasm resolved locally is still checked against a configured + /// `sha256`, and the path is taken relative to `base_dir`. + #[tokio::test] + async fn no_resolve_verifies_local_wasm_checksum() { + let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); + std::fs::write(tmp.path().join("plugin.wasm"), b"plugin bytes").unwrap(); + + let resolver = NoResolve(HostFiles); + let source = SourceField::Local(LocalSource { + path: "plugin.wasm".into(), + }); + let good = hex::encode(Sha256::digest(b"plugin bytes")); + + assert_eq!( + resolver + .resolve_wasm(&source, tmp.path(), Some(&good), None) + .await + .unwrap(), + tmp.path().join("plugin.wasm") + ); + assert!( + resolver + .resolve_wasm(&source, tmp.path(), Some(&"00".repeat(32)), None) + .await + .is_err() + ); + } +} diff --git a/crates/icp-deploy-canister/src/deploy.rs b/crates/icp-deploy-canister/src/deploy.rs new file mode 100644 index 000000000..3e280324e --- /dev/null +++ b/crates/icp-deploy-canister/src/deploy.rs @@ -0,0 +1,1053 @@ +//! Canister installation, environment-variable wiring, syncing, and deploy +//! orchestration, expressed entirely over the injected IO traits so the core +//! can run inside a canister. + +use std::collections::BTreeMap; + +use candid::Principal; +use candid::utils::ArgumentEncoder; +use ic_management_canister_types::{ + CanisterIdRecord, CanisterInstallMode, CanisterSettings, CanisterStatusResult, + CanisterStatusType, ChunkHash, ClearChunkStoreArgs, EnvironmentVariable, + InstallChunkedCodeArgs, InstallCodeArgs, UpdateSettingsArgs, UpgradeFlags, UploadChunkArgs, + WasmMemoryPersistence, +}; +use sha2::{Digest, Sha256}; +use snafu::prelude::*; + +use crate::{ + Canister, Project, + files::{FileAccess, FileAccessError}, + icp_access::{IcpAccess, IcpAccessError}, + ids::IdStore, + manifest::canister::SyncStep, + network::Configuration, + prelude::*, + sync_exec::{ + PluginExecutor, PluginInvocation, ScriptInvocation, ScriptRunner, StepProgress, + SyncStepContext, + }, +}; + +/// EOP custom-section metadata key marking a Motoko enhanced-orthogonal-persistence canister. +const EOP_METADATA: &str = "enhanced-orthogonal-persistence"; + +/// Requested installation mode. `Auto` resolves to `Install` or `Upgrade` by +/// querying the canister's current status. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InstallMode { + Auto, + Install, + Reinstall, + Upgrade, +} + +// --------------------------------------------------------------------------- +// Management-canister transport +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum MgmtCallError { + #[snafu(display("failed to encode arguments for management method '{method}'"))] + Encode { + method: String, + source: candid::Error, + }, + + #[snafu(display("management call '{method}' failed"))] + Call { + method: String, + source: IcpAccessError, + }, + + #[snafu(display("failed to decode reply from management method '{method}'"))] + Decode { + method: String, + source: candid::Error, + }, +} + +/// Encode + issue a management-canister update call and decode its reply. +/// +/// The management canister has no routing of its own, so the effective canister +/// id is the `target`. Candid coding happens here; [`IcpAccess`] is dumb transport. +async fn mgmt_call( + icp: &dyn IcpAccess, + method: &str, + target: Principal, + args: A, + cycles: u128, +) -> Result +where + A: ArgumentEncoder, + R: for<'a> candid::utils::ArgumentDecoder<'a>, +{ + let arg = candid::encode_args(args).context(EncodeSnafu { method })?; + let raw = icp + .canister_update( + Principal::management_canister(), + method, + arg, + target, + cycles, + ) + .await + .context(CallSnafu { method })?; + candid::decode_args(&raw).context(DecodeSnafu { method }) +} + +// --------------------------------------------------------------------------- +// Install +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum InstallCanisterError { + #[snafu(display("failed to read the built artifact for canister '{canister}'"))] + ReadArtifact { + canister: String, + source: FileAccessError, + }, + + #[snafu(display("failed to query status of canister '{canister}'"))] + CanisterStatus { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to detect orthogonal-persistence metadata on canister '{canister}'"))] + DetectEop { + canister: String, + source: IcpAccessError, + }, + + #[snafu(display("failed to stop canister '{canister}' before upgrade"))] + StopCanister { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to start canister '{canister}' after upgrade"))] + StartCanister { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to clear the chunk store for canister '{canister}'"))] + ClearChunkStore { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to upload wasm chunk {index} for canister '{canister}'"))] + UploadChunk { + canister: String, + index: usize, + source: MgmtCallError, + }, + + #[snafu(display("failed to install code on canister '{canister}'"))] + InstallCode { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to install chunked code on canister '{canister}'"))] + InstallChunkedCode { + canister: String, + source: MgmtCallError, + }, +} + +/// Query `canister_status` and pick the concrete install mode for `Auto`. +async fn resolve_mode_and_status( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + mode: InstallMode, +) -> Result<(CanisterInstallMode, CanisterStatusType), InstallCanisterError> { + let (status,): (CanisterStatusResult,) = mgmt_call( + icp, + "canister_status", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(CanisterStatusSnafu { + canister: canister_name, + })?; + let install_mode = match mode { + InstallMode::Auto => { + if status.module_hash.is_some() { + CanisterInstallMode::Upgrade(None) + } else { + CanisterInstallMode::Install + } + } + InstallMode::Install => CanisterInstallMode::Install, + InstallMode::Reinstall => CanisterInstallMode::Reinstall, + InstallMode::Upgrade => CanisterInstallMode::Upgrade(None), + }; + Ok((install_mode, status.status)) +} + +/// Whether the canister exposes the `enhanced-orthogonal-persistence` metadata. +async fn is_eop_canister( + icp: &dyn IcpAccess, + canister_id: Principal, +) -> Result { + Ok(icp + .read_canister_metadata(canister_id, EOP_METADATA) + .await? + .is_some()) +} + +/// Install (or upgrade/reinstall) a single, already-built canister. +/// +/// Reads the wasm from `artifact_path` through `files`; resolves `Auto` mode and +/// EOP-upgrade flags through `icp`. Large wasm is installed via the chunk store. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister( + canister_name: &str, + canister_id: Principal, + artifact_path: &Path, + mode: InstallMode, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + files: &dyn FileAccess, + icp: &dyn IcpAccess, +) -> Result<(), InstallCanisterError> { + let (mode, status) = resolve_mode_and_status(icp, canister_name, canister_id, mode).await?; + install_canister_resolved( + canister_name, + canister_id, + artifact_path, + mode, + status, + init_args, + wasm_memory_persistence, + files, + icp, + ) + .await +} + +/// Like [`install_canister`], but with the install mode and current status +/// already resolved by the caller. Callers that need the resolved mode/status +/// for their own logic first (e.g. a Candid-compatibility gate before install) +/// resolve once via [`resolve_install_mode_and_status`] and pass the result here, +/// avoiding a second `canister_status` call. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister_resolved( + canister_name: &str, + canister_id: Principal, + artifact_path: &Path, + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + files: &dyn FileAccess, + icp: &dyn IcpAccess, +) -> Result<(), InstallCanisterError> { + let wasm = files + .read_file(artifact_path) + .await + .context(ReadArtifactSnafu { + canister: canister_name, + })?; + install_canister_wasm( + canister_name, + canister_id, + &wasm, + mode, + status, + init_args, + wasm_memory_persistence, + icp, + ) + .await +} + +/// Like [`install_canister_resolved`], but for a caller that already holds the +/// wasm bytes. Callers that inspect the module before installing (e.g. the +/// Candid-compatibility gate) pass those same bytes here, so the code that is +/// installed is exactly the code that was checked. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister_wasm( + canister_name: &str, + canister_id: Principal, + wasm: &[u8], + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + icp: &dyn IcpAccess, +) -> Result<(), InstallCanisterError> { + // For EOP Motoko canisters an upgrade must set `wasm_memory_persistence`. + // Trust an explicit caller override; otherwise auto-detect and default to Keep. + let mode = match mode { + CanisterInstallMode::Upgrade(_) => { + let persistence = match wasm_memory_persistence { + Some(p) => Some(p), + None => is_eop_canister(icp, canister_id) + .await + .context(DetectEopSnafu { + canister: canister_name, + })? + .then_some(WasmMemoryPersistence::Keep), + }; + if let Some(persistence) = persistence { + CanisterInstallMode::Upgrade(Some(UpgradeFlags { + skip_pre_upgrade: None, + wasm_memory_persistence: Some(persistence), + })) + } else { + mode + } + } + other => other, + }; + + do_install( + icp, + canister_name, + canister_id, + wasm, + mode, + status, + init_args, + ) + .await +} + +/// Resolve an [`InstallMode`] and the canister's current status via +/// `canister_status` (the resolution [`install_canister`] performs internally). +/// Exposed for callers that need the resolved mode before installing. +pub async fn resolve_install_mode_and_status( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + mode: InstallMode, +) -> Result<(CanisterInstallMode, CanisterStatusType), InstallCanisterError> { + resolve_mode_and_status(icp, canister_name, canister_id, mode).await +} + +#[allow(clippy::too_many_arguments)] +async fn do_install( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + wasm: &[u8], + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, +) -> Result<(), InstallCanisterError> { + // Raw install_code messages are limited to 2 MiB; larger wasm goes through + // the chunk store (spec limit is 1 MiB per chunk). + const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024; + const CHUNK_SIZE: usize = 1024 * 1024; + // Generous overhead for encoding, target canister ID, install mode, etc. + const ENCODING_OVERHEAD: usize = 500; + + let arg = init_args + .map(|a| a.to_vec()) + .unwrap_or_else(|| candid::encode_args(()).expect("encoding empty args is infallible")); + + let total_install_size = wasm.len() + arg.len() + ENCODING_OVERHEAD; + + if total_install_size <= CHUNK_THRESHOLD { + let install_args = InstallCodeArgs { + mode, + canister_id, + wasm_module: wasm.to_vec(), + arg, + sender_canister_version: None, + }; + stop_and_start_if_needed(icp, canister_name, canister_id, mode, status, async { + mgmt_call::<_, ()>(icp, "install_code", canister_id, (install_args,), 0) + .await + .context(InstallCodeSnafu { + canister: canister_name, + }) + }) + .await?; + } else { + // Clear any existing chunks to ensure a clean state. + clear_chunk_store(icp, canister_name, canister_id).await?; + + let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect(); + let mut chunk_hashes: Vec = Vec::new(); + for (i, chunk) in chunks.iter().enumerate() { + let (hash,): (ChunkHash,) = mgmt_call( + icp, + "upload_chunk", + canister_id, + (UploadChunkArgs { + canister_id, + chunk: chunk.to_vec(), + },), + 0, + ) + .await + .context(UploadChunkSnafu { + canister: canister_name, + index: i, + })?; + chunk_hashes.push(hash); + } + + let wasm_module_hash = Sha256::digest(wasm).to_vec(); + let chunked_args = InstallChunkedCodeArgs { + mode, + target_canister: canister_id, + store_canister: None, + chunk_hashes_list: chunk_hashes, + wasm_module_hash, + arg, + sender_canister_version: None, + }; + + let install_res = + stop_and_start_if_needed(icp, canister_name, canister_id, mode, status, async { + mgmt_call::<_, ()>(icp, "install_chunked_code", canister_id, (chunked_args,), 0) + .await + .context(InstallChunkedCodeSnafu { + canister: canister_name, + }) + }) + .await; + + // Always clear the chunk store afterwards to free storage. If the install + // failed, report that error in preference to a clear-store failure. + let clear_res = clear_chunk_store(icp, canister_name, canister_id).await; + install_res?; + clear_res?; + } + + Ok(()) +} + +async fn clear_chunk_store( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, +) -> Result<(), InstallCanisterError> { + mgmt_call::<_, ()>( + icp, + "clear_chunk_store", + canister_id, + (ClearChunkStoreArgs { canister_id },), + 0, + ) + .await + .context(ClearChunkStoreSnafu { + canister: canister_name, + }) +} + +/// Guard an upgrade/reinstall of a Running canister by stopping it first and +/// restarting it afterwards (whether or not the install succeeded). +async fn stop_and_start_if_needed( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + mode: CanisterInstallMode, + status: CanisterStatusType, + install: F, +) -> Result<(), InstallCanisterError> +where + F: Future>, +{ + let should_guard = matches!( + mode, + CanisterInstallMode::Upgrade(_) | CanisterInstallMode::Reinstall + ) && matches!(status, CanisterStatusType::Running); + + if should_guard { + mgmt_call::<_, ()>( + icp, + "stop_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StopCanisterSnafu { + canister: canister_name, + })?; + } + + let install_result = install.await; + + if !should_guard { + return install_result; + } + + let start_result = mgmt_call::<_, ()>( + icp, + "start_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StartCanisterSnafu { + canister: canister_name, + }); + + // Restart whether or not the install succeeded. If both failed, the install + // error is the more likely root cause. + match (install_result, start_result) { + (Err(install_err), _) => Err(install_err), + (Ok(()), Err(start_err)) => Err(start_err), + (Ok(()), Ok(())) => Ok(()), + } +} + +/// Start a canister (idempotent; a no-op if already Running). +pub async fn start_canister( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, +) -> Result<(), InstallCanisterError> { + mgmt_call::<_, ()>( + icp, + "start_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StartCanisterSnafu { + canister: canister_name, + }) +} + +// --------------------------------------------------------------------------- +// Environment variables + sync +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum SyncCanisterError { + #[snafu(display("failed to apply environment variables to canister '{canister}'"))] + ApplyEnvVars { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to run sync step for canister '{canister}'"))] + RunStep { + canister: String, + source: SyncStepError, + }, +} + +#[derive(Debug, Snafu)] +pub enum SyncStepError { + #[snafu(transparent)] + ResolvePlugin { + source: crate::sync_exec::UnknownCallableCanisterError, + }, + + #[snafu(transparent)] + Plugin { + source: crate::sync_exec::PluginExecutorError, + }, + + #[snafu(transparent)] + Script { + source: crate::sync_exec::ScriptRunError, + }, +} + +/// Compute the environment variables a canister should run with: its manifest +/// `settings` variables merged with the generated `PUBLIC_CANISTER_ID:` +/// variables, resolved against `canister_ids`. +/// +/// Each canister is wired only to the ids it declares in `bindings`, resolved to +/// the ids that exist in this environment; unresolved bindings are skipped. +pub fn binding_env_vars( + canister: &Canister, + canister_ids: &BTreeMap, +) -> Vec { + let mut env_vars = canister + .settings + .environment_variables + .clone() + .unwrap_or_default(); + + for (env_name, referenced_key) in &canister.bindings { + if let Some(principal) = canister_ids.get(referenced_key) { + env_vars.insert( + format!("PUBLIC_CANISTER_ID:{env_name}"), + principal.to_text(), + ); + } + } + + env_vars + .into_iter() + .map(|(name, value)| EnvironmentVariable { name, value }) + .collect() +} + +/// Apply the canister's environment variables (see [`binding_env_vars`]) via +/// `update_settings`. +/// +/// This is the piece that standalone `icp sync` previously skipped: the binding +/// ids must be (re)written whenever a canister is synced, not only on deploy. +pub async fn apply_binding_env_vars( + canister: &Canister, + canister_id: Principal, + canister_ids: &BTreeMap, + icp: &dyn IcpAccess, +) -> Result<(), SyncCanisterError> { + let environment_variables = binding_env_vars(canister, canister_ids); + + mgmt_call::<_, ()>( + icp, + "update_settings", + canister_id, + (UpdateSettingsArgs { + canister_id, + settings: CanisterSettings { + environment_variables: Some(environment_variables), + ..Default::default() + }, + sender_canister_version: None, + },), + 0, + ) + .await + .context(ApplyEnvVarsSnafu { + canister: canister.name.clone(), + }) +} + +/// Run a canister's configured sync steps through the injected executors, +/// collecting any retained stderr lines. Does not apply environment variables +/// (see [`apply_binding_env_vars`] / [`sync_canister`]). +pub async fn run_sync_steps( + canister: &Canister, + ctx: &SyncStepContext, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, + progress: Option<&dyn StepProgress>, +) -> Result, SyncCanisterError> { + let mut lines = Vec::new(); + for step in &canister.sync.steps { + // This crate owns dispatch and input derivation; the executors only run + // the fully-resolved invocation. + let step_lines = match step { + SyncStep::Plugin(adapter) => match PluginInvocation::new(adapter, ctx) { + Ok(invocation) => plugin_exec + .run_plugin(invocation, progress) + .await + .map_err(SyncStepError::from), + Err(err) => Err(SyncStepError::from(err)), + }, + SyncStep::Script(adapter) => script_runner + .run_script(ScriptInvocation::new(adapter, ctx), progress) + .await + .map_err(SyncStepError::from), + } + .context(RunStepSnafu { + canister: canister.name.clone(), + })?; + lines.extend(step_lines); + } + Ok(lines) +} + +/// Sync a single canister: (re)apply its binding environment variables, then run +/// its sync steps. Applying env vars here is what makes standalone `icp sync` +/// include the generated `PUBLIC_CANISTER_ID:*` variables. +#[allow(clippy::too_many_arguments)] +pub async fn sync_canister( + canister: &Canister, + canister_id: Principal, + ctx: &SyncStepContext, + icp: &dyn IcpAccess, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, + progress: Option<&dyn StepProgress>, +) -> Result, SyncCanisterError> { + apply_binding_env_vars(canister, canister_id, &ctx.canister_ids, icp).await?; + run_sync_steps(canister, ctx, plugin_exec, script_runner, progress).await +} + +// --------------------------------------------------------------------------- +// Deploy +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum DeployCanisterError { + #[snafu(display("could not find canister '{canister}' in environment '{environment}'"))] + UnknownCanister { + canister: String, + environment: String, + }, + + #[snafu(display("could not find an id for canister '{canister}'; create it first"))] + LookupId { + canister: String, + source: crate::ids::IdStoreError, + }, + + #[snafu(display("failed to encode init args for canister '{canister}'"))] + InitArgs { + canister: String, + source: crate::InitArgsToBytesError, + }, + + #[snafu(transparent)] + Install { source: InstallCanisterError }, + + #[snafu(transparent)] + Sync { source: SyncCanisterError }, +} + +/// Deploy (install then sync) a single already-built canister in `environment`. +/// +/// Assumes the canister already exists (its id is read from `ids`); creating +/// canisters is the caller's responsibility. +#[allow(clippy::too_many_arguments)] +pub async fn deploy_canister( + project: &Project, + canister_name: &str, + environment: &str, + artifact_path: &Path, + mode: InstallMode, + proxy: Option, + files: &dyn FileAccess, + icp: &dyn IcpAccess, + ids: &dyn IdStore, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, + progress: Option<&dyn StepProgress>, +) -> Result, DeployCanisterError> { + let env = project + .environments + .get(environment) + .context(UnknownCanisterSnafu { + canister: canister_name, + environment, + })?; + let is_cache = matches!(env.network.configuration, Configuration::Managed { .. }); + let network = env.network.name.clone(); + + let (canister_path, canister) = + env.canisters + .get(canister_name) + .context(UnknownCanisterSnafu { + canister: canister_name, + environment, + })?; + + let canister_id = ids + .lookup(is_cache, environment, canister_name) + .context(LookupIdSnafu { + canister: canister_name, + })?; + let canister_ids = ids + .lookup_by_environment(is_cache, environment) + .unwrap_or_default(); + + let init_args = canister + .init_args + .as_ref() + .map(|ia| ia.to_bytes()) + .transpose() + .context(InitArgsSnafu { + canister: canister_name, + })?; + + // Environment variables first, then install, then sync steps. + apply_binding_env_vars(canister, canister_id, &canister_ids, icp).await?; + install_canister( + canister_name, + canister_id, + artifact_path, + mode, + init_args.as_deref(), + None, + files, + icp, + ) + .await?; + // Asset sync requires a Running canister; install_code is status-preserving. + start_canister(icp, canister_name, canister_id).await?; + + let ctx = SyncStepContext { + canister_path: canister_path.clone(), + project_dir: project.dir.clone(), + canister_id, + canister_name: canister.name.clone(), + environment: environment.to_owned(), + network, + canister_ids, + proxy, + }; + let lines = run_sync_steps(canister, &ctx, plugin_exec, script_runner, progress).await?; + Ok(lines) +} + +#[derive(Debug, Snafu)] +#[snafu(display("failed to deploy canister(s): {}", names.join(", ")))] +pub struct DeployError { + pub names: Vec, + pub failures: Vec<(String, DeployCanisterError)>, +} + +/// Deploy the `selected` already-built canisters in `environment`, each through +/// [`deploy_canister`]. `artifact_paths` maps canister name → its built wasm +/// path. Per-canister failures are aggregated. This is a batch entry point with +/// no progress reporting; the CLI drives its own per-canister fan-out instead. +#[allow(clippy::too_many_arguments)] +pub async fn deploy( + project: &Project, + selected: &[String], + environment: &str, + mode: InstallMode, + proxy: Option, + artifact_paths: &BTreeMap, + files: &dyn FileAccess, + icp: &dyn IcpAccess, + ids: &dyn IdStore, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, +) -> Result<(), DeployError> { + let mut failures = Vec::new(); + for name in selected { + let Some(artifact_path) = artifact_paths.get(name) else { + failures.push(( + name.clone(), + DeployCanisterError::UnknownCanister { + canister: name.clone(), + environment: environment.to_owned(), + }, + )); + continue; + }; + if let Err(e) = deploy_canister( + project, + name, + environment, + artifact_path, + mode, + proxy, + files, + icp, + ids, + plugin_exec, + script_runner, + None, + ) + .await + { + failures.push((name.clone(), e)); + } + } + + if failures.is_empty() { + Ok(()) + } else { + let names = failures.iter().map(|(n, _)| n.clone()).collect(); + Err(DeployError { names, failures }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::canister::Settings; + use crate::manifest::adapter::script::{self, CommandField}; + use crate::manifest::canister::{BuildSteps, SyncStep, SyncSteps}; + use async_trait::async_trait; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + /// Test principal `2vxsx-fae` (the anonymous principal), used as a stand-in. + fn principal() -> Principal { + Principal::anonymous() + } + + /// Records the ordered sequence of interactions across the mock `IcpAccess` + /// and mock sync executors, plus the raw args of each management call. + #[derive(Default)] + struct Log { + events: Vec, + calls: Vec<(String, Vec)>, + } + + struct MockIcp { + log: Arc>, + } + + #[cfg_attr(target_family = "wasm", async_trait(?Send))] + #[cfg_attr(not(target_family = "wasm"), async_trait)] + impl IcpAccess for MockIcp { + async fn canister_update( + &self, + _canister: Principal, + method: &str, + arg: Vec, + _effective_canister_id: Principal, + _cycles: u128, + ) -> Result, IcpAccessError> { + let mut log = self.log.lock().unwrap(); + log.events.push(method.to_owned()); + log.calls.push((method.to_owned(), arg)); + // Every management method exercised here replies with unit. + Ok(candid::encode_args(()).unwrap()) + } + + async fn read_canister_metadata( + &self, + _canister: Principal, + _path: &str, + ) -> Result>, IcpAccessError> { + Ok(None) + } + + fn caller_principal(&self) -> Principal { + principal() + } + } + + struct MockExec { + log: Arc>, + } + + #[cfg_attr(target_family = "wasm", async_trait(?Send))] + #[cfg_attr(not(target_family = "wasm"), async_trait)] + impl PluginExecutor for MockExec { + async fn run_plugin( + &self, + _invocation: PluginInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, crate::sync_exec::PluginExecutorError> { + self.log + .lock() + .unwrap() + .events + .push("run_plugin".to_owned()); + Ok(vec![]) + } + } + + #[cfg_attr(target_family = "wasm", async_trait(?Send))] + #[cfg_attr(not(target_family = "wasm"), async_trait)] + impl ScriptRunner for MockExec { + async fn run_script( + &self, + _invocation: ScriptInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, crate::sync_exec::ScriptRunError> { + self.log + .lock() + .unwrap() + .events + .push("run_script".to_owned()); + Ok(vec![]) + } + } + + fn canister_with_binding() -> Canister { + Canister { + name: "backend".to_owned(), + settings: Settings { + environment_variables: Some(HashMap::from([("FOO".to_owned(), "bar".to_owned())])), + ..Default::default() + }, + build: BuildSteps { steps: vec![] }, + sync: SyncSteps { + steps: vec![SyncStep::Script(script::Adapter { + command: CommandField::Command("noop".to_owned()), + })], + }, + init_args: None, + registry_recipe: None, + bindings: BTreeMap::from([("backend".to_owned(), "backend".to_owned())]), + friendly_names: vec![], + environment_variable_files: BTreeMap::new(), + } + } + + fn ctx_for(canister_id: Principal) -> SyncStepContext { + SyncStepContext { + canister_path: PathBuf::from("/project"), + project_dir: PathBuf::from("/project"), + canister_id, + canister_name: "backend".to_owned(), + environment: "local".to_owned(), + network: "local".to_owned(), + canister_ids: BTreeMap::from([("backend".to_owned(), canister_id)]), + proxy: None, + } + } + + /// The bug fix: `sync_canister` must (re)apply the binding environment + /// variables *before* running any sync step, so standalone `icp sync` + /// includes the generated `PUBLIC_CANISTER_ID:*` variables. + #[tokio::test] + async fn sync_canister_applies_env_vars_before_steps() { + let log = Arc::new(Mutex::new(Log::default())); + let icp = MockIcp { log: log.clone() }; + let exec = MockExec { log: log.clone() }; + let canister = canister_with_binding(); + let cid = principal(); + let ctx = ctx_for(cid); + + sync_canister(&canister, cid, &ctx, &icp, &exec, &exec, None) + .await + .unwrap(); + + let events = &log.lock().unwrap().events; + assert_eq!( + events.as_slice(), + &["update_settings".to_owned(), "run_script".to_owned()], + "env vars must be applied before sync steps run" + ); + } + + /// `apply_binding_env_vars` merges manifest env vars with the generated + /// `PUBLIC_CANISTER_ID:` ids. + #[tokio::test] + async fn apply_binding_env_vars_merges_manifest_and_bindings() { + let log = Arc::new(Mutex::new(Log::default())); + let icp = MockIcp { log: log.clone() }; + let canister = canister_with_binding(); + let cid = principal(); + let canister_ids = BTreeMap::from([("backend".to_owned(), cid)]); + + apply_binding_env_vars(&canister, cid, &canister_ids, &icp) + .await + .unwrap(); + + let calls = &log.lock().unwrap().calls; + assert_eq!(calls.len(), 1); + let (method, arg) = &calls[0]; + assert_eq!(method, "update_settings"); + let (args,): (UpdateSettingsArgs,) = candid::decode_args(arg).unwrap(); + let vars: HashMap = args + .settings + .environment_variables + .unwrap() + .into_iter() + .map(|v| (v.name, v.value)) + .collect(); + assert_eq!(vars.get("FOO"), Some(&"bar".to_owned())); + assert_eq!(vars.get("PUBLIC_CANISTER_ID:backend"), Some(&cid.to_text())); + } + + /// A binding whose referenced canister has no id in this environment is + /// skipped rather than emitted with an empty value. + #[test] + fn unresolved_binding_is_skipped() { + let canister = canister_with_binding(); + let vars = binding_env_vars(&canister, &BTreeMap::new()); + let names: Vec<&str> = vars.iter().map(|v| v.name.as_str()).collect(); + assert_eq!(names, ["FOO"], "unresolved bindings must not be emitted"); + } +} diff --git a/crates/icp-deploy-canister/src/files.rs b/crates/icp-deploy-canister/src/files.rs new file mode 100644 index 000000000..546a11347 --- /dev/null +++ b/crates/icp-deploy-canister/src/files.rs @@ -0,0 +1,50 @@ +//! Abstracted filesystem access. +//! +//! Project loading, manifest consolidation, and reading built wasm artifacts all +//! go through [`FileAccess`] rather than touching the real filesystem, so the +//! same logic can run inside a canister (backed by, e.g., stable-memory blobs). +//! Paths are `camino` UTF-8 paths. + +use async_trait::async_trait; +use snafu::Snafu; + +use crate::prelude::*; + +#[derive(Debug, Snafu)] +pub enum FileAccessError { + #[snafu(display("failed to read file at '{path}': {message}"))] + Read { path: PathBuf, message: String }, + + #[snafu(display("failed to list directory at '{path}': {message}"))] + ReadDir { path: PathBuf, message: String }, +} + +/// Read-oriented filesystem access, rooted at the project directory. +/// +/// Predicate methods (`exists`/`is_file`/`is_dir`) return `false` on any error, +/// matching the `std::path` inherent methods they replace. `canonicalize` +/// returns `None` when the path cannot be resolved or is not valid UTF-8. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait FileAccess: Send + Sync { + /// Read the raw bytes of a file. + async fn read_file(&self, path: &Path) -> Result, FileAccessError>; + + /// Read a file as a UTF-8 string. + async fn read_to_string(&self, path: &Path) -> Result; + + async fn exists(&self, path: &Path) -> bool; + + async fn is_file(&self, path: &Path) -> bool; + + async fn is_dir(&self, path: &Path) -> bool; + + /// Non-recursive directory listing. Entries are returned as absolute paths + /// (the directory joined with each entry name). + async fn read_dir(&self, path: &Path) -> Result, FileAccessError>; + + /// Canonicalize a path (resolve `..` and symlinks). Returns `None` if the + /// path does not exist or does not resolve to valid UTF-8; callers treat + /// that as "cannot establish identity", which is safe for de-duplication. + async fn canonicalize(&self, path: &Path) -> Option; +} diff --git a/crates/icp-deploy-canister/src/icp_access.rs b/crates/icp-deploy-canister/src/icp_access.rs new file mode 100644 index 000000000..0cad5c597 --- /dev/null +++ b/crates/icp-deploy-canister/src/icp_access.rs @@ -0,0 +1,60 @@ +//! Abstracted access to the ICP API. +//! +//! [`IcpAccess`] is a dumb transport: this crate encodes/decodes Candid and +//! decides *what* to call (including all management-canister calls), while the +//! implementation only routes bytes. Proxy routing is owned by the impl +//! (constructed with the proxy principal); the caller never threads a proxy +//! through per call. + +use async_trait::async_trait; +use candid::Principal; +use snafu::Snafu; + +#[derive(Debug, Snafu)] +pub enum IcpAccessError { + #[snafu(display("update call to '{method}' on canister '{canister}' failed: {message}"))] + Update { + canister: Principal, + method: String, + message: String, + }, + + #[snafu(display("failed to read metadata '{path}' from canister '{canister}': {message}"))] + ReadMetadata { + canister: Principal, + path: String, + message: String, + }, +} + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait IcpAccess: Send + Sync { + /// Perform an update call and return the raw reply bytes. + /// + /// `effective_canister_id` is the canister used for request routing. For a + /// normal application call it equals `canister`; for a management-canister + /// call (`canister == aaaaa-aa`) it must be the *target* canister, since the + /// management canister has no routing of its own. `cycles` is attached to + /// the call (only meaningful for proxied/funded calls such as + /// `create_canister`). + async fn canister_update( + &self, + canister: Principal, + method: &str, + arg: Vec, + effective_canister_id: Principal, + cycles: u128, + ) -> Result, IcpAccessError>; + + /// Read a canister's custom-section metadata (via `read_state`). Returns + /// `None` when the section is absent. Used for EOP-upgrade detection. + async fn read_canister_metadata( + &self, + canister: Principal, + path: &str, + ) -> Result>, IcpAccessError>; + + /// The caller's (identity's) principal. + fn caller_principal(&self) -> Principal; +} diff --git a/crates/icp-deploy-canister/src/ids.rs b/crates/icp-deploy-canister/src/ids.rs new file mode 100644 index 000000000..962a11bfa --- /dev/null +++ b/crates/icp-deploy-canister/src/ids.rs @@ -0,0 +1,85 @@ +//! Canister-id store: per-environment `name → principal` mappings. + +use std::{collections::BTreeMap, sync::Mutex}; + +use candid::Principal; +use snafu::{OptionExt, Snafu}; + +/// Mapping of canister names to their principals within an environment. +pub type IdMapping = BTreeMap; + +#[derive(Debug, Snafu)] +pub enum IdStoreError { + #[snafu(display("could not find id for canister '{canister_name}' in environment '{env}'"))] + NotFound { env: String, canister_name: String }, + + #[snafu(display("failed to access canister id store for environment '{env}': {message}"))] + Access { env: String, message: String }, +} + +/// Read/write access to canister-id mappings. +/// +/// The `is_cache` flag lets an implementation that keeps two stores — a +/// managed-network cache and a connected-network data store — pick the right +/// one. `register` mutates through `&self`, so the store is interior-mutable. +pub trait IdStore: Send + Sync { + fn lookup( + &self, + is_cache: bool, + env: &str, + canister_name: &str, + ) -> Result; + + fn lookup_by_environment(&self, is_cache: bool, env: &str) -> Result; + + fn register( + &self, + is_cache: bool, + env: &str, + canister_name: &str, + canister_id: Principal, + ) -> Result<(), IdStoreError>; +} + +#[derive(Debug, Default)] +pub struct InMemoryIdStore(pub Mutex>); + +impl IdStore for InMemoryIdStore { + fn lookup( + &self, + _is_cache: bool, + env: &str, + canister_name: &str, + ) -> Result { + let mapping = self.lookup_by_environment(_is_cache, env)?; + mapping + .get(canister_name) + .cloned() + .context(NotFoundSnafu { env, canister_name }) + } + + fn lookup_by_environment(&self, _is_cache: bool, env: &str) -> Result { + self.0 + .lock() + .unwrap() + .get(env) + .cloned() + .context(AccessSnafu { + env, + message: "environment not found", + }) + } + + fn register( + &self, + _is_cache: bool, + env: &str, + canister_name: &str, + canister_id: Principal, + ) -> Result<(), IdStoreError> { + let mut store = self.0.lock().unwrap(); + let mapping = store.entry(env.to_string()).or_default(); + mapping.insert(canister_name.to_string(), canister_id); + Ok(()) + } +} diff --git a/crates/icp-deploy-canister/src/lib.rs b/crates/icp-deploy-canister/src/lib.rs new file mode 100644 index 000000000..33383194a --- /dev/null +++ b/crates/icp-deploy-canister/src/lib.rs @@ -0,0 +1,326 @@ +//! Canister installation, syncing, and the project model, with all host IO +//! abstracted behind trait objects so the core can run inside a canister. +//! +//! See the module-level docs on the IO traits ([`files`], [`icp_access`], +//! [`ids`]) for the abstraction boundary. + +use std::collections::{BTreeMap, HashMap}; + +use indexmap::IndexMap; +use serde::Serialize; +use snafu::prelude::*; + +use candid_parser::parse_idl_args; + +use crate::{ + canister::Settings, + manifest::{ + ArgsFormat, + canister::{BuildSteps, SyncSteps}, + }, + network::Configuration, + prelude::*, +}; + +pub mod canister; +pub mod deploy; +pub mod files; +pub mod icp_access; +pub mod ids; +pub mod manifest; +pub mod network; +pub mod parsers; +pub mod prelude; +pub mod project; +pub mod sync_exec; + +#[cfg(test)] +mod testutil; + +pub use deploy::{ + DeployCanisterError, DeployError, InstallCanisterError, InstallMode, SyncCanisterError, + SyncStepError, apply_binding_env_vars, binding_env_vars, deploy, deploy_canister, + install_canister, install_canister_resolved, install_canister_wasm, + resolve_install_mode_and_status, run_sync_steps, start_canister, sync_canister, +}; +pub use files::{FileAccess, FileAccessError}; +pub use icp_access::{IcpAccess, IcpAccessError}; +pub use ids::{IdMapping, IdStore, IdStoreError}; +pub use project::{consolidate_manifest, load_project, verify_sandbox}; +pub use sync_exec::{ + PluginExecutor, PluginExecutorError, PluginInvocation, ScriptInvocation, ScriptRunError, + ScriptRunner, StepProgress, SyncStepContext, system_env_vars, +}; + +/// Resolved initialization arguments, with any file references already loaded. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub enum InitArgs { + /// Text content (inline or loaded from file). Format is always known. + Text { content: String, format: ArgsFormat }, + /// Raw binary bytes (from a file with `format: bin`). Used directly. + Binary(Vec), +} + +#[derive(Debug, Snafu)] +pub enum InitArgsToBytesError { + #[snafu(display("failed to decode hex init args"))] + HexDecode { source: hex::FromHexError }, + + #[snafu(display("failed to parse Candid init args"))] + CandidParse { source: candid_parser::Error }, + + #[snafu(display("failed to encode Candid init args to bytes"))] + CandidEncode { source: candid::Error }, +} + +impl InitArgs { + /// Resolve to raw bytes according to the format. + pub fn to_bytes(&self) -> Result, InitArgsToBytesError> { + match self { + InitArgs::Binary(bytes) => Ok(bytes.clone()), + InitArgs::Text { content, format } => match format { + ArgsFormat::Hex => hex::decode(content.trim()).context(HexDecodeSnafu), + ArgsFormat::Candid => { + let args = parse_idl_args(content.trim()).context(CandidParseSnafu)?; + args.to_bytes().context(CandidEncodeSnafu) + } + ArgsFormat::Bin => { + unreachable!("binary format cannot appear in InitArgs::Text") + } + }, + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Canister { + pub name: String, + + /// Canister settings, such as memory constaints, etc. + pub settings: Settings, + + /// The build configuration specifying how to compile the canister's source + /// code into a WebAssembly module, including the adapter to use. + pub build: BuildSteps, + + /// The configuration specifying how to sync the canister + pub sync: SyncSteps, + + /// Initialization arguments passed to the canister during installation. + /// Resolved from the manifest — file contents are already loaded. + pub init_args: Option, + + /// If the canister was defined via a recipe reference, this holds the + /// original recipe specifier string (e.g. `@dfinity/motoko@v4.0.0`). + /// `None` when the canister uses explicit build/sync instructions. + pub registry_recipe: Option, + + /// Canister-discovery wiring. Maps the name this canister reads in a + /// `PUBLIC_CANISTER_ID:` environment variable to the store key of the + /// referenced canister. Computed during consolidation so each canister sees + /// the view its owning project expects: its own project's canisters under + /// their local names, plus any declared dependencies under their aliases + /// (`:`). For a project with no dependencies this maps every + /// canister's local name to itself, reproducing the flat "every canister sees + /// every sibling" behavior. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub bindings: BTreeMap, + + /// Subdomain prefixes for the canister's friendly URLs, most-specific label + /// first, e.g. `["backend"]` for an own canister or `["backend.openemail"]` + /// for a dependency canister (dot-nested by alias chain). A de-duplicated + /// shared dependency canister carries one entry per alias chain that reaches + /// it. Consumed only at deploy time to build `custom-domains.txt` entries and + /// the printed URLs; a runtime display aid that is always recomputed during + /// consolidation, so it is never serialized. + #[serde(skip)] + pub friendly_names: Vec, + + /// For each environment variable whose value came from a file, the file it + /// was read from. `settings.environment_variables` already holds the + /// contents; the paths are kept so `icp project bundle` can hold a file + /// backing a variable to the same containment rule it applies to every other + /// file a manifest points at. Bookkeeping for that check rather than part of + /// the resolved configuration, so it is never serialized. + #[serde(skip)] + pub environment_variable_files: BTreeMap, +} + +#[derive(Debug, Snafu)] +pub enum BundleModulePathError { + #[snafu(display( + "canister '{canister}' does not have a single build step (found {count}); a bundled \ + canister must be built by exactly one pre-built step" + ))] + NotSingleBuildStep { canister: String, count: usize }, + + #[snafu(display( + "canister '{canister}' is not built by a pre-built step; a bundled canister's module \ + must come from a `pre-built` build step" + ))] + NotPrebuilt { canister: String }, + + #[snafu(display("canister '{canister}' is built from a remote URL, not a local module path"))] + NotLocal { canister: String }, +} + +/// Extract the local wasm module path a bundled canister is built from. +/// +/// A bundle's canisters are each built by a single `pre-built` step pointing at +/// the module on disk; this returns that path, erroring if the build is not that +/// single-prebuilt-local-path shape. +pub fn bundle_get_canister_module_path( + canister: &Canister, +) -> Result<&Path, BundleModulePathError> { + let steps = &canister.build.steps; + let [step] = steps.as_slice() else { + return NotSingleBuildStepSnafu { + canister: canister.name.clone(), + count: steps.len(), + } + .fail(); + }; + let manifest::BuildStep::Prebuilt(adapter) = step else { + return NotPrebuiltSnafu { + canister: canister.name.clone(), + } + .fail(); + }; + match &adapter.source { + manifest::prebuilt::SourceField::Local(local) => Ok(&local.path), + manifest::prebuilt::SourceField::Remote(_) => NotLocalSnafu { + canister: canister.name.clone(), + } + .fail(), + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Network { + pub name: String, + pub configuration: Configuration, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Environment { + pub name: String, + pub network: Network, + pub canisters: IndexMap, +} + +impl Environment { + pub fn get_canister_names(&self) -> Vec { + self.canisters.keys().cloned().collect() + } + + pub fn contains_canister(&self, canister_name: &str) -> bool { + self.canisters.contains_key(canister_name) + } + + pub fn get_canister_info(&self, canister: &str) -> Result<(PathBuf, Canister), String> { + self.canisters + .get(canister) + .ok_or_else(|| { + format!( + "canister '{}' not declared in environment '{}'", + canister, self.name + ) + }) + .cloned() + } +} + +/// Consolidated project definition +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Project { + pub dir: PathBuf, + pub canisters: IndexMap, + pub networks: HashMap, + pub environments: HashMap, + + /// Environments the workspace defines that some vendored member does *not* + /// declare, keyed by environment name → the missing members' store-key + /// prefixes. Enforced when the environment is selected (strict rule). + /// Empty for standalone projects and workspaces whose members are complete. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub member_missing_envs: HashMap>, +} + +impl Project { + pub fn get_canister(&self, canister_name: &str) -> Option<&(PathBuf, Canister)> { + self.canisters.get(canister_name) + } +} + +#[cfg(test)] +mod bundle_tests { + use super::*; + use crate::canister::Settings; + use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; + use crate::manifest::adapter::{prebuilt, script}; + use crate::manifest::canister::{BuildStep, SyncSteps}; + + fn canister_with_build(steps: Vec) -> Canister { + Canister { + name: "backend".to_owned(), + settings: Settings::default(), + build: BuildSteps { steps }, + sync: SyncSteps { steps: vec![] }, + init_args: None, + registry_recipe: None, + bindings: BTreeMap::new(), + friendly_names: vec![], + environment_variable_files: BTreeMap::new(), + } + } + + fn prebuilt_local(path: &str) -> BuildStep { + BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Local(LocalSource { path: path.into() }), + sha256: None, + }) + } + + #[test] + fn extracts_the_single_prebuilt_local_path() { + let c = canister_with_build(vec![prebuilt_local("out/backend.wasm")]); + assert_eq!( + bundle_get_canister_module_path(&c).unwrap(), + Path::new("out/backend.wasm") + ); + } + + #[test] + fn rejects_zero_or_multiple_build_steps() { + let two = canister_with_build(vec![prebuilt_local("a.wasm"), prebuilt_local("b.wasm")]); + assert!(matches!( + bundle_get_canister_module_path(&two), + Err(BundleModulePathError::NotSingleBuildStep { count: 2, .. }) + )); + } + + #[test] + fn rejects_a_non_prebuilt_step() { + let c = canister_with_build(vec![BuildStep::Script(script::Adapter { + command: script::CommandField::Command("make".to_owned()), + })]); + assert!(matches!( + bundle_get_canister_module_path(&c), + Err(BundleModulePathError::NotPrebuilt { .. }) + )); + } + + #[test] + fn rejects_a_remote_prebuilt_source() { + let c = canister_with_build(vec![BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Remote(RemoteSource { + url: "https://example.com/backend.wasm".to_owned(), + }), + sha256: Some("abc".to_owned()), + })]); + assert!(matches!( + bundle_get_canister_module_path(&c), + Err(BundleModulePathError::NotLocal { .. }) + )); + } +} diff --git a/crates/icp/src/manifest/adapter/mod.rs b/crates/icp-deploy-canister/src/manifest/adapter/mod.rs similarity index 100% rename from crates/icp/src/manifest/adapter/mod.rs rename to crates/icp-deploy-canister/src/manifest/adapter/mod.rs diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp-deploy-canister/src/manifest/adapter/plugin.rs similarity index 98% rename from crates/icp/src/manifest/adapter/plugin.rs rename to crates/icp-deploy-canister/src/manifest/adapter/plugin.rs index 579562503..739d5b964 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp-deploy-canister/src/manifest/adapter/plugin.rs @@ -253,16 +253,18 @@ pub struct Adapter { #[schemars(with = "Option>")] pub fields: Option>, - /// Canisters this plugin may call, or read metadata from, in addition to - /// the canister being synced. Each entry is a canister name resolved against + /// Canisters this plugin may call, read metadata from, or set environment + /// variables on, in addition to the canister being synced. Each entry is a + /// canister name resolved against /// the project's canister ID table for the environment being synced, written /// as this project spells it: a bare local name for one of its own canisters /// (e.g. `backend`), or a `:` key for a canister of /// something it depends on (e.g. `vendor/ledger:ledger`). The same spellings /// hold when the project is a workspace member, so vendoring it does not /// change them. The plugin picks a target per request via the `call-target` - /// in its `canister-call` or `canister-metadata-section` request; a target - /// not listed here is rejected by the host. + /// in its `canister-call`, `canister-metadata-section`, or + /// `canister-set-environment-variable` request; a target not listed here is + /// rejected by the host. pub canisters: Option>, } diff --git a/crates/icp/src/manifest/adapter/prebuilt.rs b/crates/icp-deploy-canister/src/manifest/adapter/prebuilt.rs similarity index 100% rename from crates/icp/src/manifest/adapter/prebuilt.rs rename to crates/icp-deploy-canister/src/manifest/adapter/prebuilt.rs diff --git a/crates/icp/src/manifest/adapter/script.rs b/crates/icp-deploy-canister/src/manifest/adapter/script.rs similarity index 100% rename from crates/icp/src/manifest/adapter/script.rs rename to crates/icp-deploy-canister/src/manifest/adapter/script.rs diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp-deploy-canister/src/manifest/canister.rs similarity index 90% rename from crates/icp/src/manifest/canister.rs rename to crates/icp-deploy-canister/src/manifest/canister.rs index 8efab30de..fe19e41d6 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp-deploy-canister/src/manifest/canister.rs @@ -153,27 +153,22 @@ impl<'de> Deserialize<'de> for CanisterManifest { let has_build = temp_map.contains_key(&build_key); let has_sync = temp_map.contains_key(&sync_key); - match (has_recipe, has_build, has_sync) { - (true, true, _) => { + match (has_recipe, has_build) { + (true, true) => { // Can't have a recipe and a build Err(Error::custom(format!( "Canister {name} cannot have both a `recipe` and a `build` section" ))) } - (true, false, true) => { - // Can't have a recipe and a sync sections - Err(Error::custom(format!( - "Canister {name} cannot have both a `recipe` and a `sync` section" - ))) - } - (false, false, _) => { + (false, false) => { // We must have recipe or build Err(Error::custom(format!( "Canister {name} must have a `recipe` or a `build` section" ))) } - (true, false, false) => { - // We have a a recipe + (true, false) => { + // We have a a recipe, optionally with sync steps of its + // own to run after the recipe's let recipe: Recipe = serde_yaml::from_value( temp_map .remove(&recipe_key) @@ -184,6 +179,23 @@ impl<'de> Deserialize<'de> for CanisterManifest { Error::custom(format!("Canister {name} failed to parse recipe: {}", e)) })?; + let sync: Option = if has_sync { + Some( + serde_yaml::from_value( + temp_map + .remove(&sync_key) + .ok_or_else(|| Error::custom("sync field not found"))?, + ) + .map_err(|e| { + Error::custom(format!( + "Canister {name} failed to parse sync instructions: {e}" + )) + })?, + ) + } else { + None + }; + if !temp_map.is_empty() { return Err(Error::custom(format!( "Unrecognized fields in canister `{name}`." @@ -194,10 +206,10 @@ impl<'de> Deserialize<'de> for CanisterManifest { name, settings, init_args, - instructions: Instructions::Recipe { recipe }, + instructions: Instructions::Recipe { recipe, sync }, }) } - (false, true, _) => { + (false, true) => { // We have a build section // Try to deserialize as BuildSync variant @@ -241,6 +253,10 @@ impl<'de> Deserialize<'de> for CanisterManifest { pub enum Instructions { Recipe { recipe: Recipe, + + /// Additional sync steps, run after the ones the recipe renders. + #[serde(skip_serializing_if = "Option::is_none")] + sync: Option, }, BuildSync { @@ -607,7 +623,8 @@ mod tests { recipe_type: RecipeType::File("my-recipe".to_string()), configuration: HashMap::new(), sha256: None, - } + }, + sync: None, }, }, ); @@ -636,7 +653,8 @@ mod tests { ("key-2".to_string(), "value-2".into()) ]), sha256: None, - } + }, + sync: None, }, }, ); @@ -667,7 +685,8 @@ mod tests { "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" .to_string() ), - } + }, + sync: None, }, }, ); @@ -697,12 +716,68 @@ mod tests { recipe_type: RecipeType::File("my-recipe".to_string()), configuration: HashMap::new(), sha256: None, - } + }, + sync: None, }, }, ); } + #[test] + fn recipe_with_sync() { + assert_eq!( + validate_canister_yaml(indoc! {r#" + name: my-canister + recipe: + type: file://my-recipe + sync: + steps: + - type: script + command: echo hi + "#}), + CanisterManifest { + name: "my-canister".to_string(), + settings: ManifestSettings::default(), + init_args: None, + instructions: Instructions::Recipe { + recipe: Recipe { + recipe_type: RecipeType::File("my-recipe".to_string()), + configuration: HashMap::new(), + sha256: None, + }, + sync: Some(SyncSteps { + steps: vec![SyncStep::Script(script::Adapter { + command: script::CommandField::Command("echo hi".to_string()), + })] + }), + }, + }, + ); + } + + #[test] + fn recipe_with_invalid_sync() { + match serde_yaml::from_str::(indoc! {r#" + name: my-canister + recipe: + type: file://my-recipe + sync: + steps: + - type: nonsense + "#}) + { + Ok(_) => panic!("an unknown sync step type should not deserialize"), + Err(err) => { + let err_msg = format!("{err}"); + if !err_msg.contains("Canister my-canister failed to parse sync instructions") { + panic!( + "expected 'Canister my-canister failed to parse sync instructions' error but got: {err}" + ); + } + } + }; + } + #[test] fn build_steps() { assert_eq!( diff --git a/crates/icp/src/manifest/dependency.rs b/crates/icp-deploy-canister/src/manifest/dependency.rs similarity index 100% rename from crates/icp/src/manifest/dependency.rs rename to crates/icp-deploy-canister/src/manifest/dependency.rs diff --git a/crates/icp/src/manifest/environment.rs b/crates/icp-deploy-canister/src/manifest/environment.rs similarity index 100% rename from crates/icp/src/manifest/environment.rs rename to crates/icp-deploy-canister/src/manifest/environment.rs diff --git a/crates/icp-deploy-canister/src/manifest/mod.rs b/crates/icp-deploy-canister/src/manifest/mod.rs new file mode 100644 index 000000000..99bc680a7 --- /dev/null +++ b/crates/icp-deploy-canister/src/manifest/mod.rs @@ -0,0 +1,132 @@ +use std::marker::PhantomData; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use snafu::prelude::*; + +use crate::files::{FileAccess, FileAccessError}; +use crate::prelude::*; + +pub mod adapter; +pub mod canister; +pub mod dependency; +pub mod environment; +pub mod network; +pub mod project; +pub mod recipe; +pub mod serde_helpers; + +pub use { + adapter::plugin, + adapter::prebuilt, + canister::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, + SyncStep, SyncSteps, + }, + dependency::DependencyManifest, + environment::EnvironmentManifest, + network::{ManagedMode, Mode, NetworkManifest}, + project::ProjectManifest, +}; + +pub const PROJECT_MANIFEST: &str = "icp.yaml"; +pub const CANISTER_MANIFEST: &str = "canister.yaml"; + +#[derive(Debug, Snafu)] +pub enum LoadManifestError { + #[snafu(transparent)] + Read { source: FileAccessError }, + + #[snafu(display("failed to parse manifest at '{path}'"))] + Parse { + source: serde_yaml::Error, + path: PathBuf, + }, +} + +/// Load and parse a YAML manifest of type `T` through the injected [`FileAccess`]. +pub async fn load_manifest(files: &dyn FileAccess, path: &Path) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let content = files.read_file(path).await?; + let m = serde_yaml::from_slice::(&content).context(ParseSnafu { + path: path.to_path_buf(), + })?; + Ok(m) +} + +// A manifest item that can either be a path to another manifest file or the manifest itself. +// +// The valid path specifications are: +// - CanisterManifest: path or glob pattern to the directory containing "canister.yaml" +// - NetworkManifest: path to network manifest +// - EnvironmentManifest: path to environment manifest +#[derive(Clone, Debug, PartialEq, JsonSchema)] +#[serde(untagged)] +pub enum Item { + /// Path to a manifest + Path(String), + + /// The manifest + Manifest(T), +} + +/// Items in path form serialize back to a bare path string, *not* to the contents of the +/// referenced file. Callers that need a self-contained YAML output (e.g. `icp project bundle`) +/// must convert any `Item::Path` to `Item::Manifest` themselves by loading the referenced +/// manifest first. +impl Serialize for Item { + fn serialize(&self, serializer: S) -> Result { + match self { + Item::Path(p) => p.serialize(serializer), + Item::Manifest(m) => m.serialize(serializer), + } + } +} + +impl<'de, T> Deserialize<'de> for Item +where + T: Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::{MapAccess, Visitor, value::MapAccessDeserializer}; + use std::fmt; + + struct ItemVisitor(PhantomData); + + impl<'de, T: Deserialize<'de>> Visitor<'de> for ItemVisitor { + type Value = Item; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string path or a manifest object") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(Item::Path(v.to_owned())) + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + Ok(Item::Path(v)) + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + T::deserialize(MapAccessDeserializer::new(map)).map(Item::Manifest) + } + } + + deserializer.deserialize_any(ItemVisitor(PhantomData)) + } +} diff --git a/crates/icp/src/manifest/network.rs b/crates/icp-deploy-canister/src/manifest/network.rs similarity index 100% rename from crates/icp/src/manifest/network.rs rename to crates/icp-deploy-canister/src/manifest/network.rs diff --git a/crates/icp/src/manifest/project.rs b/crates/icp-deploy-canister/src/manifest/project.rs similarity index 100% rename from crates/icp/src/manifest/project.rs rename to crates/icp-deploy-canister/src/manifest/project.rs diff --git a/crates/icp/src/manifest/recipe.rs b/crates/icp-deploy-canister/src/manifest/recipe.rs similarity index 100% rename from crates/icp/src/manifest/recipe.rs rename to crates/icp-deploy-canister/src/manifest/recipe.rs diff --git a/crates/icp/src/manifest/serde_helpers.rs b/crates/icp-deploy-canister/src/manifest/serde_helpers.rs similarity index 100% rename from crates/icp/src/manifest/serde_helpers.rs rename to crates/icp-deploy-canister/src/manifest/serde_helpers.rs diff --git a/crates/icp-deploy-canister/src/network/mod.rs b/crates/icp-deploy-canister/src/network/mod.rs new file mode 100644 index 000000000..4f21d3e7f --- /dev/null +++ b/crates/icp-deploy-canister/src/network/mod.rs @@ -0,0 +1,368 @@ +//! Network *configuration* model (the manifest-derived view of a network). +//! +//! Runtime concerns — launching/stopping managed networks, network descriptors, +//! agent access — live in the host `icp` crate. + +use schemars::JsonSchema; +use serde::{Deserialize, Deserializer, Serialize}; +use strum::EnumString; +use url::Url; + +pub use crate::manifest::network::RootKeySpec; +use crate::manifest::network::{ + Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode, +}; + +pub const DEFAULT_LOCAL_NETWORK_BIND: &str = "127.0.0.1"; +pub const DEFAULT_LOCAL_NETWORK_PORT: u16 = 8000; + +#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] +pub enum Port { + Fixed(u16), + Random, +} + +impl Default for Port { + fn default() -> Self { + Port::Fixed(8000) + } +} + +impl<'de> Deserialize<'de> for Port { + fn deserialize>(d: D) -> Result { + Ok(match u16::deserialize(d)? { + 0 => Port::Random, + p => Port::Fixed(p), + }) + } +} + +fn default_bind() -> String { + "127.0.0.1".to_string() +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Gateway { + #[serde(default = "default_bind")] + pub bind: String, + + #[serde(default)] + pub port: Port, + + #[serde(default)] + pub domains: Vec, +} + +impl Default for Gateway { + fn default() -> Self { + Self { + bind: default_bind(), + port: Default::default(), + domains: Default::default(), + } + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Managed { + #[serde(flatten)] + pub mode: ManagedMode, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +#[serde(untagged)] +pub enum ManagedMode { + Image(Box), + Launcher(Box), +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct ManagedLauncherConfig { + pub gateway: Gateway, + pub artificial_delay_ms: Option, + pub ii: bool, + pub nns: bool, + pub subnets: Option>, + pub bitcoind_addr: Option>, + pub dogecoind_addr: Option>, + pub version: Option, +} + +#[derive( + Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize, EnumString, strum::Display, +)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum SubnetKind { + Application, + System, + VerifiedApplication, + Bitcoin, + Fiduciary, + Nns, + Sns, +} + +impl Default for ManagedMode { + fn default() -> Self { + Self::default_for_port(DEFAULT_LOCAL_NETWORK_PORT) + } +} + +impl ManagedMode { + pub fn default_for_port(port: u16) -> Self { + ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway: Gateway { + bind: default_bind(), + port: if port == 0 { + Port::Random + } else { + Port::Fixed(port) + }, + domains: vec![], + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + version: None, + })) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct ManagedImageConfig { + pub image: String, + pub port_mapping: Vec, + pub rm_on_exit: bool, + pub args: Vec, + pub entrypoint: Option>, + pub environment: Vec, + pub volumes: Vec, + pub platform: Option, + pub user: Option, + pub shm_size: Option, + pub status_dir: String, + pub mounts: Vec, + pub extra_hosts: Vec, +} + +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Connected { + /// The URL this network's API can be reached at. + pub api_url: Url, + + /// The URL this network's HTTP gateway can be reached at. + pub http_gateway_url: Option, + + /// How to obtain the root key used to verify responses from this network. + pub root_key: RootKeySpec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +#[serde(tag = "mode", rename_all = "lowercase")] +pub enum Configuration { + // Note: we must use struct variants to be able to flatten + // and make schemars generate the proper schema + /// A managed network is one which can be controlled and manipulated. + Managed { + #[serde(flatten)] + managed: Managed, + }, + + /// A connected network is one which can be interacted with + /// but cannot be controlled or manipulated. + Connected { + #[serde(flatten)] + connected: Connected, + }, +} + +impl Default for Configuration { + fn default() -> Self { + Configuration::Managed { + managed: Managed::default(), + } + } +} + +impl From for Gateway { + fn from(value: ManifestGateway) -> Self { + let ManifestGateway { + bind, + domains, + port, + } = value; + let bind = bind.unwrap_or("127.0.0.1".to_string()); + let port = match port { + Some(0) => Port::Random, + Some(p) => Port::Fixed(p), + None => Port::default(), + }; + let mut domains = domains.unwrap_or_default(); + if bind == "127.0.0.1" || bind == "0.0.0.0" || bind == "::1" || bind == "::" { + domains.insert(0, "localhost".to_string()); + } + Gateway { + bind, + port, + domains, + } + } +} + +impl From for Connected { + fn from(value: ManifestConnected) -> Self { + let root_key = value.root_key; + match value.endpoints { + Endpoints::Implicit { url } => Connected { + api_url: url.clone(), + http_gateway_url: Some(url), + root_key, + }, + Endpoints::Explicit { + api_url, + http_gateway_url, + } => Connected { + api_url, + http_gateway_url, + root_key, + }, + } + } +} + +impl From for Configuration { + fn from(value: Mode) -> Self { + match value { + Mode::Managed(managed) => match *managed.mode { + crate::manifest::network::ManagedMode::Launcher { + gateway, + artificial_delay_ms, + ii, + nns, + subnets, + bitcoind_addr, + dogecoind_addr, + version, + } => { + let gateway: Gateway = match gateway { + Some(g) => g.into(), + None => Gateway::default(), + }; + let version = match version { + Some(v) => { + if v.starts_with('v') { + Some(v) + } else { + Some(format!("v{v}")) + } + } + None => None, + }; + Configuration::Managed { + managed: Managed { + mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway, + artificial_delay_ms, + ii: ii.unwrap_or(false), + nns: nns.unwrap_or(false), + subnets, + bitcoind_addr, + dogecoind_addr, + version, + })), + }, + } + } + crate::manifest::network::ManagedMode::Image { + image, + port_mapping, + rm_on_exit, + args, + entrypoint, + environment, + volumes, + platform, + user, + shm_size, + status_dir, + mounts: mount, + extra_hosts, + } => Configuration::Managed { + managed: Managed { + mode: ManagedMode::Image(Box::new(ManagedImageConfig { + image, + port_mapping, + rm_on_exit: rm_on_exit.unwrap_or(false), + args: args.unwrap_or_default(), + entrypoint, + environment: environment.unwrap_or_default(), + volumes: volumes.unwrap_or_default(), + platform, + user, + shm_size, + status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), + mounts: mount.unwrap_or_default(), + extra_hosts: extra_hosts.unwrap_or_default(), + })), + }, + }, + }, + Mode::Connected(connected) => Configuration::Connected { + connected: connected.into(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::network::{ + Gateway as ManifestGateway, Managed as ManifestManaged, ManagedMode as ManifestManagedMode, + Mode, + }; + + #[test] + fn from_mode_launcher_with_bitcoind_addr() { + let mode = Mode::Managed(ManifestManaged { + mode: Box::new(ManifestManagedMode::Launcher { + gateway: Some(ManifestGateway { + bind: None, + port: Some(8000), + domains: None, + }), + artificial_delay_ms: None, + ii: None, + nns: None, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, + version: None, + }), + }); + + let config: Configuration = mode.into(); + match config { + Configuration::Managed { + managed: + Managed { + mode: ManagedMode::Launcher(launcher_config), + }, + } => { + assert_eq!( + launcher_config.bitcoind_addr, + Some(vec!["127.0.0.1:18444".to_string()]) + ); + assert_eq!(launcher_config.dogecoind_addr, None); + assert!(!launcher_config.ii); + assert!(!launcher_config.nns); + } + _ => panic!("expected ManagedMode::Launcher"), + } + } +} diff --git a/crates/icp-deploy-canister/src/parsers.rs b/crates/icp-deploy-canister/src/parsers.rs new file mode 100644 index 000000000..b0a74730f --- /dev/null +++ b/crates/icp-deploy-canister/src/parsers.rs @@ -0,0 +1,643 @@ +//! Parsing of token, cycle, memory, and duration amounts with support for suffixes and underscores. + +use bigdecimal::{BigDecimal, Signed}; +use num_bigint::BigUint; +use num_integer::Integer; +use num_traits::{ToPrimitive, Zero}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::str::FromStr; + +/// Parse a token amount with support for suffixes (k, m, b, t) and underscores. +/// +/// Examples: +/// - `1` -> 1 +/// - `1_000` -> 1000 +/// - `1k` or `1K` -> 1000 +/// - `1t` or `1T` -> 1000000000000 +/// - `0.5` -> 0.5 +/// - `0.5k` -> 500 +pub fn parse_token_amount(input: &str) -> Result { + let input = input.trim(); + + if input.is_empty() { + return Err("Token amount cannot be empty".to_string()); + } + + let (number_part, multiplier) = if let Some(last_char) = input.chars().last() { + match last_char { + 'k' | 'K' => (&input[..input.len() - 1], 1_000u128), + 'm' | 'M' => (&input[..input.len() - 1], 1_000_000u128), + 'b' | 'B' => (&input[..input.len() - 1], 1_000_000_000u128), + 't' | 'T' => (&input[..input.len() - 1], 1_000_000_000_000u128), + _ => (input, 1u128), + } + } else { + (input, 1u128) + }; + + let cleaned = number_part.replace('_', ""); + let base = + BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid token amount: '{}'", input))?; + + if base.is_negative() { + return Err(format!("Token amount cannot be negative: '{}'", input)); + } + + let multiplier_decimal = BigDecimal::from(multiplier); + Ok(base * multiplier_decimal) +} + +/// Convert a token amount to the smallest unit by multiplying by 10^token_decimals. +/// E.g. 1.5 with 8 decimals -> 150000000. Fails if the result would be fractional. +pub fn to_token_unit_amount( + token_amount: BigDecimal, + token_decimals: u8, +) -> Result { + use num_bigint::BigInt; + use num_traits::pow::Pow; + + let (mantissa, exponent) = token_amount.into_bigint_and_exponent(); + let scale_adjustment = token_decimals as i64 - exponent; + let ten = BigInt::from(10); + + let result = if scale_adjustment >= 0 { + let multiplier = ten.pow(scale_adjustment as u32); + mantissa * multiplier + } else { + let divisor = ten.pow((-scale_adjustment) as u32); + let (quotient, remainder) = mantissa.div_rem(&divisor); + if !remainder.is_zero() { + return Err(format!( + "Token amount cannot be represented with {} decimals (would result in fractional units)", + token_decimals + )); + } + quotient + }; + + result + .try_into() + .map_err(|_| "Token amount cannot be negative".to_string()) +} + +fn parse_cycles_str(s: &str) -> Result { + let token_amount = parse_token_amount(s)?; + let unit_amount = to_token_unit_amount(token_amount, 0)?; + unit_amount + .to_u128() + .ok_or_else(|| format!("Cycles amount too large: '{}'", s)) +} + +/// An amount of cycles. +/// +/// Deserializes from a number or a string with suffixes (k, m, b, t) and optional underscore separators. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum CyclesAmount { + Number(u64), // yaml only supports up to u64 + Str(String), +} + +impl CyclesAmount { + pub fn get(&self) -> u128 { + match self { + CyclesAmount::Number(n) => *n as u128, + CyclesAmount::Str(s) => parse_cycles_str(s) + .unwrap_or_else(|e| panic!("invalid cycles amount '{}': {}", s, e)), + } + } +} + +impl<'de> Deserialize<'de> for CyclesAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + // Identical enum to CyclesAmount. Needed to avoid a circular dependency. + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom("cycles amount must be a number or a string with optional suffix (k, m, b, t), e.g. 1000 or \"4t\"") + })?; + let c = match v { + Raw::Number(n) => CyclesAmount::Number(n), + Raw::Str(ref s) => { + parse_cycles_str(s).map_err(serde::de::Error::custom)?; // validate the string is a valid cycles amount + CyclesAmount::Str(s.clone()) + } + }; + Ok(c) + } +} + +impl Serialize for CyclesAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + CyclesAmount::Number(n) => serializer.serialize_u64(*n), + CyclesAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for CyclesAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_cycles_str(s)?; // validate the string is a valid cycles amount + Ok(CyclesAmount::Str(s.to_string())) + } +} + +impl fmt::Display for CyclesAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u128 { + fn from(c: CyclesAmount) -> Self { + c.get() + } +} + +impl From for CyclesAmount { + fn from(n: u128) -> Self { + if let Ok(n64) = u64::try_from(n) { + CyclesAmount::Number(n64) + } else { + CyclesAmount::Str(n.to_string()) + } + } +} + +const KB: u64 = 1000; +const KIB: u64 = 1024; +const MB: u64 = 1_000_000; +const MIB: u64 = 1024 * 1024; +const GB: u64 = 1_000_000_000; +const GIB: u64 = 1024 * 1024 * 1024; + +fn parse_memory_str(s: &str) -> Result { + let s = s.trim(); + if s.is_empty() { + return Err("Memory amount cannot be empty".to_string()); + } + let lower = s.to_lowercase(); + let (number_part, factor) = if lower.ends_with("gib") { + (&s[..s.len() - 3], GIB) + } else if lower.ends_with("gb") { + (&s[..s.len() - 2], GB) + } else if lower.ends_with("mib") { + (&s[..s.len() - 3], MIB) + } else if lower.ends_with("mb") { + (&s[..s.len() - 2], MB) + } else if lower.ends_with("kib") { + (&s[..s.len() - 3], KIB) + } else if lower.ends_with("kb") { + (&s[..s.len() - 2], KB) + } else { + (s, 1u64) + }; + let cleaned = number_part.trim().replace('_', ""); + let amount = + BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid memory amount: '{}'", s))?; + if amount.is_negative() { + return Err(format!("Memory amount cannot be negative: '{}'", s)); + } + let product = amount * BigDecimal::from(factor); + if !product.is_integer() { + return Err( + "Memory amount must be a whole number of bytes (fractional bytes not allowed)" + .to_string(), + ); + } + product + .to_u64() + .ok_or_else(|| format!("Memory amount too large: '{}'", s)) +} + +/// An amount of memory in bytes. +/// +/// Deserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib), +/// optional decimals, and optional underscore separators. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum MemoryAmount { + Number(u64), + Str(String), +} + +impl MemoryAmount { + pub fn get(&self) -> u64 { + match self { + MemoryAmount::Number(n) => *n, + MemoryAmount::Str(s) => parse_memory_str(s) + .unwrap_or_else(|e| panic!("invalid memory amount '{}': {}", s, e)), + } + } +} + +impl<'de> Deserialize<'de> for MemoryAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom( + "memory amount must be a number or a string with optional suffix (kb, kib, mb, mib, gb, gib), e.g. 1024 or \"2.5gib\"", + ) + })?; + let m = match v { + Raw::Number(n) => MemoryAmount::Number(n), + Raw::Str(ref s) => { + parse_memory_str(s).map_err(serde::de::Error::custom)?; + MemoryAmount::Str(s.clone()) + } + }; + Ok(m) + } +} + +impl Serialize for MemoryAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + MemoryAmount::Number(n) => serializer.serialize_u64(*n), + MemoryAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for MemoryAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_memory_str(s)?; + Ok(MemoryAmount::Str(s.to_string())) + } +} + +impl fmt::Display for MemoryAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u64 { + fn from(m: MemoryAmount) -> Self { + m.get() + } +} + +impl From for MemoryAmount { + fn from(n: u64) -> Self { + MemoryAmount::Number(n) + } +} + +const SECONDS_PER_MINUTE: u64 = 60; +const SECONDS_PER_HOUR: u64 = 3600; +const SECONDS_PER_DAY: u64 = 86400; +const SECONDS_PER_WEEK: u64 = 604800; + +fn parse_duration_str(s: &str) -> Result { + let s = s.trim(); + if s.is_empty() { + return Err("Duration cannot be empty".to_string()); + } + let lower = s.to_lowercase(); + let (number_part, factor) = if lower.ends_with('w') { + (&s[..s.len() - 1], SECONDS_PER_WEEK) + } else if lower.ends_with('d') { + (&s[..s.len() - 1], SECONDS_PER_DAY) + } else if lower.ends_with('h') { + (&s[..s.len() - 1], SECONDS_PER_HOUR) + } else if lower.ends_with('m') { + (&s[..s.len() - 1], SECONDS_PER_MINUTE) + } else if lower.ends_with('s') { + (&s[..s.len() - 1], 1u64) + } else { + (s, 1u64) + }; + let cleaned = number_part.trim().replace('_', ""); + if cleaned.is_empty() { + return Err(format!("Invalid duration: '{s}'")); + } + let value: u64 = cleaned + .parse() + .map_err(|_| format!("Invalid duration: '{s}'"))?; + value + .checked_mul(factor) + .ok_or_else(|| format!("Duration too large: '{s}'")) +} + +/// A duration in seconds. +/// +/// Deserializes from a number (seconds) or a string with duration suffix (s, m, h, d, w) +/// and optional underscore separators. +/// +/// Suffixes (case-insensitive): +/// - `s` — seconds +/// - `m` — minutes (×60) +/// - `h` — hours (×3600) +/// - `d` — days (×86400) +/// - `w` — weeks (×604800) +/// +/// A bare number without suffix is treated as seconds. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum DurationAmount { + Number(u64), + Str(String), +} + +impl DurationAmount { + pub fn get(&self) -> u64 { + match self { + DurationAmount::Number(n) => *n, + DurationAmount::Str(s) => { + parse_duration_str(s).unwrap_or_else(|e| panic!("invalid duration '{}': {}", s, e)) + } + } + } +} + +impl<'de> Deserialize<'de> for DurationAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom( + "duration must be a number (seconds) or a string with optional suffix (s, m, h, d, w), e.g. 2592000 or \"30d\"", + ) + })?; + let c = match v { + Raw::Number(n) => DurationAmount::Number(n), + Raw::Str(ref s) => { + parse_duration_str(s).map_err(serde::de::Error::custom)?; + DurationAmount::Str(s.clone()) + } + }; + Ok(c) + } +} + +impl Serialize for DurationAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + DurationAmount::Number(n) => serializer.serialize_u64(*n), + DurationAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for DurationAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_duration_str(s)?; + Ok(DurationAmount::Str(s.to_string())) + } +} + +impl fmt::Display for DurationAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u64 { + fn from(d: DurationAmount) -> Self { + d.get() + } +} + +impl From for DurationAmount { + fn from(n: u64) -> Self { + DurationAmount::Number(n) + } +} + +impl PartialEq for DurationAmount { + fn eq(&self, other: &u64) -> bool { + self.get() == *other + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cycles_amount_from_str_plain() { + assert_eq!("1".parse::().unwrap().get(), 1); + assert_eq!("1000".parse::().unwrap().get(), 1000); + } + + #[test] + fn cycles_amount_from_str_suffixes() { + assert_eq!("1k".parse::().unwrap().get(), 1000); + assert_eq!( + "1t".parse::().unwrap().get(), + 1_000_000_000_000 + ); + assert_eq!( + "4t".parse::().unwrap().get(), + 4_000_000_000_000 + ); + assert_eq!( + "0.5t".parse::().unwrap().get(), + 500_000_000_000 + ); + } + + #[test] + fn cycles_amount_from_str_underscores() { + assert_eq!("1_000".parse::().unwrap().get(), 1000); + } + + #[test] + fn cycles_amount_from_str_fractional_rejected() { + assert!("1.5".parse::().is_err()); + } + + #[test] + fn cycles_amount_deserialize() { + let yaml = "4t"; + let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(c.get(), 4_000_000_000_000); + + let yaml = "5000000000000"; + let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(c.get(), 5_000_000_000_000); + } + + #[test] + fn parse_token_amount_plain_and_suffixes() { + use std::str::FromStr; + assert_eq!( + parse_token_amount("1").unwrap(), + BigDecimal::from_str("1").unwrap() + ); + assert_eq!( + parse_token_amount("1k").unwrap(), + BigDecimal::from_str("1000").unwrap() + ); + assert_eq!( + parse_token_amount("0.5t").unwrap(), + BigDecimal::from_str("500000000000").unwrap() + ); + } + + #[test] + fn memory_amount_from_str_plain() { + assert_eq!("1".parse::().unwrap().get(), 1); + assert_eq!("1024".parse::().unwrap().get(), 1024); + } + + #[test] + fn memory_amount_from_str_suffixes() { + assert_eq!("1kb".parse::().unwrap().get(), 1000); + assert_eq!("1kib".parse::().unwrap().get(), 1024); + assert_eq!("1mb".parse::().unwrap().get(), 1_000_000); + assert_eq!("1mib".parse::().unwrap().get(), 1024 * 1024); + assert_eq!("1gb".parse::().unwrap().get(), 1_000_000_000); + assert_eq!( + "1gib".parse::().unwrap().get(), + 1024 * 1024 * 1024 + ); + assert_eq!( + "2 GiB".parse::().unwrap().get(), + 2 * 1024 * 1024 * 1024 + ); + } + + #[test] + fn memory_amount_from_str_decimals() { + assert_eq!("0.5kib".parse::().unwrap().get(), 512); + assert_eq!("1.5gib".parse::().unwrap().get(), 1610612736); + } + + #[test] + fn memory_amount_fractional_bytes_rejected() { + assert!("1.5".parse::().is_err()); // 1.5 bytes + assert!("0.3kib".parse::().is_err()); // 307.2 bytes + } + + #[test] + fn memory_amount_from_str_underscores() { + assert_eq!("1_024".parse::().unwrap().get(), 1024); + } + + #[test] + fn memory_amount_deserialize() { + let yaml = "2gib"; + let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(m.get(), 2 * 1024 * 1024 * 1024); + + let yaml = "4294967296"; + let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(m.get(), 4294967296); + } + + #[test] + fn duration_amount_from_str_plain() { + assert_eq!("60".parse::().unwrap().get(), 60); + assert_eq!("2592000".parse::().unwrap().get(), 2592000); + } + + #[test] + fn duration_amount_from_str_underscores() { + assert_eq!( + "2_592_000".parse::().unwrap().get(), + 2592000 + ); + } + + #[test] + fn duration_amount_from_str_suffixes() { + assert_eq!("60s".parse::().unwrap().get(), 60); + assert_eq!("90m".parse::().unwrap().get(), 5400); + assert_eq!("24h".parse::().unwrap().get(), 86400); + assert_eq!("30d".parse::().unwrap().get(), 2592000); + assert_eq!("4w".parse::().unwrap().get(), 2419200); + } + + #[test] + fn duration_amount_from_str_case_insensitive() { + assert_eq!("30D".parse::().unwrap().get(), 2592000); + assert_eq!("1W".parse::().unwrap().get(), 604800); + assert_eq!("24H".parse::().unwrap().get(), 86400); + assert_eq!("60S".parse::().unwrap().get(), 60); + assert_eq!("90M".parse::().unwrap().get(), 5400); + } + + #[test] + fn duration_amount_from_str_underscores_with_suffix() { + assert_eq!( + "2_592_000s".parse::().unwrap().get(), + 2592000 + ); + } + + #[test] + fn duration_amount_from_str_errors() { + assert!("abc".parse::().is_err()); + assert!("".parse::().is_err()); + assert!("1x".parse::().is_err()); + assert!("1.5d".parse::().is_err()); + assert!("-1d".parse::().is_err()); + } + + #[test] + fn duration_amount_deserialize() { + let yaml = "30d"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + + let yaml = "2592000"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + + let yaml = "2_592_000"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + } + + #[test] + fn duration_amount_partial_eq_u64() { + let d = DurationAmount::Number(2592000); + assert!(d == 2592000); + assert!(d != 0); + + let d = DurationAmount::Str("30d".to_string()); + assert!(d == 2592000); + } +} diff --git a/crates/icp-deploy-canister/src/prelude.rs b/crates/icp-deploy-canister/src/prelude.rs new file mode 100644 index 000000000..3cf6e17e0 --- /dev/null +++ b/crates/icp-deploy-canister/src/prelude.rs @@ -0,0 +1,13 @@ +pub use camino::{FromPathBufError, Utf8Path as Path, Utf8PathBuf as PathBuf}; + +pub const TRILLION: u128 = 1_000_000_000_000; + +pub const SECOND: u64 = 1; +pub const MINUTE: u64 = 60 * SECOND; + +pub const IC_MAINNET_NETWORK_API_URL: &str = "https://icp-api.io"; +pub const IC_MAINNET_NETWORK_GATEWAY_URL: &str = "https://icp.net"; +/// Name of the implicit IC mainnet network and its implicit environment +pub const IC: &str = "ic"; +/// Name of the implicit local managed network and its implicit environment +pub const LOCAL: &str = "local"; diff --git a/crates/icp-deploy-canister/src/project.rs b/crates/icp-deploy-canister/src/project.rs new file mode 100644 index 000000000..3e8c76615 --- /dev/null +++ b/crates/icp-deploy-canister/src/project.rs @@ -0,0 +1,2477 @@ +use std::collections::{BTreeMap, HashMap, HashSet, hash_map::Entry}; + +use indexmap::{IndexMap, map::Entry as IndexEntry}; + +use snafu::prelude::*; + +use crate::{ + Canister, Environment, InitArgs, Network, Project, + canister::{ControllerRef, ManifestEnvVar, ManifestSettings, Settings, recipe}, + files::{FileAccess, FileAccessError}, + manifest::{ + ArgsFormat, CANISTER_MANIFEST, CanisterManifest, DependencyManifest, EnvironmentManifest, + Item, LoadManifestError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, + ProjectManifest, + canister::{Instructions, SyncSteps}, + environment::CanisterSelection, + load_manifest, + network::RootKeySpec, + recipe::RecipeType, + }, + network::{ + Configuration, Connected, DEFAULT_LOCAL_NETWORK_BIND, DEFAULT_LOCAL_NETWORK_PORT, Gateway, + Managed, ManagedLauncherConfig, ManagedMode, Port, + }, + prelude::*, +}; + +#[derive(Debug, Snafu)] +pub enum EnvironmentError { + #[snafu(display("environment '{environment}' points to invalid network '{network}'"))] + InvalidNetwork { + environment: String, + network: String, + }, + + #[snafu(display("environment '{environment}' points to invalid canister '{canister}'"))] + InvalidCanister { + environment: String, + canister: String, + }, +} + +#[derive(Debug, Snafu)] +pub enum ConsolidateManifestError { + #[snafu(display("failed to parse glob pattern"))] + GlobParse { source: glob::PatternError }, + + #[snafu(display("failed to list directory while expanding a glob"))] + ListDir { source: FileAccessError }, + + #[snafu(display("failed to load canister manifest"))] + LoadCanister { source: LoadManifestError }, + + #[snafu(display("failed to load network manifest"))] + LoadNetwork { source: LoadManifestError }, + + #[snafu(display("failed to load environment manifest"))] + LoadEnvironment { source: LoadManifestError }, + + #[snafu(display("failed to load {kind} manifest at: {path}"))] + Failed { kind: String, path: String }, + + #[snafu(display("failed to fetch canister recipe: {recipe_type:?}"))] + FetchRecipe { + #[snafu(source(from(recipe::ResolveError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("failed to render canister recipe: {recipe_type:?}"))] + RenderRecipe { + #[snafu(source(from(recipe::RenderRecipeError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("failed to cache canister recipe: {recipe_type:?}"))] + CacheRecipe { + #[snafu(source(from(recipe::ResolveError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("project contains two similarly named {kind}s: '{name}'"))] + Duplicate { kind: String, name: String }, + + #[snafu(display("`{name}` is a reserved {kind} name."))] + Reserved { kind: String, name: String }, + + #[snafu(display("could not locate a {kind} manifest at: '{path}'"))] + NotFound { kind: String, path: String }, + + #[snafu(display("failed to read init_args file for canister '{canister}'"))] + ReadInitArgs { + source: FileAccessError, + canister: String, + }, + + #[snafu(display( + "failed to read the file backing environment variable '{variable}' of canister '{canister}'" + ))] + ReadEnvironmentVariable { + source: FileAccessError, + canister: String, + variable: String, + }, + + #[snafu(display( + "init_args for canister '{canister}' uses format 'bin' with inline content; \ + binary format requires a file path" + ))] + BinFormatInlineContent { canister: String }, + + #[snafu(display( + "canister '{canister}' lists controller '{controller}', but no canister with that \ + name is declared in the project" + ))] + UnknownControllerCanister { + canister: String, + controller: String, + }, + + #[snafu(display( + "canister name '{name}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ + (':' is reserved as the dependency namespace separator)" + ))] + InvalidCanisterName { name: String }, + + #[snafu(display( + "dependency alias '{alias}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ + (':' is reserved as the dependency namespace separator)" + ))] + InvalidDependencyAlias { alias: String }, + + #[snafu(display("project declares two dependencies with the same alias '{alias}'"))] + DuplicateDependencyAlias { alias: String }, + + #[snafu(display( + "dependency alias '{alias}' collides with a canister of the same name in the same project" + ))] + DependencyAliasCollision { alias: String }, + + #[snafu(display("could not find a project manifest for dependency '{alias}' at: '{path}'"))] + DependencyNotFound { alias: String, path: String }, + + #[snafu(display("failed to canonicalize path for dependency '{alias}' at: '{path}'"))] + DependencyCanonicalize { alias: String, path: String }, + + #[snafu(display("failed to load project manifest for dependency '{alias}'"))] + LoadDependencyManifest { + source: LoadManifestError, + alias: String, + }, + + #[snafu(display( + "dependency '{alias}' selects canister '{canister}', which the dependency does not declare" + ))] + UnknownDependencyCanister { alias: String, canister: String }, + + #[snafu(display("dependency cycle detected: {chain}"))] + CircularDependency { chain: String }, + + #[snafu(transparent)] + Environment { source: EnvironmentError }, +} + +/// Resolve a [`ManifestInitArgs`] into a canonical [`InitArgs`] by reading +/// any file references relative to `base_path`. +async fn resolve_manifest_init_args( + files: &dyn FileAccess, + manifest_init_args: &ManifestInitArgs, + base_path: &Path, + canister: &str, +) -> Result { + match manifest_init_args { + ManifestInitArgs::String(content) => Ok(InitArgs::Text { + content: content.trim().to_owned(), + format: ArgsFormat::Candid, + }), + ManifestInitArgs::Path { path, format } => { + let file_path = base_path.join(path); + match format { + ArgsFormat::Bin => { + let bytes = files + .read_file(&file_path) + .await + .context(ReadInitArgsSnafu { canister })?; + Ok(InitArgs::Binary(bytes)) + } + fmt => { + let content = files + .read_to_string(&file_path) + .await + .context(ReadInitArgsSnafu { canister })?; + Ok(InitArgs::Text { + content: content.trim().to_owned(), + format: fmt.clone(), + }) + } + } + } + ManifestInitArgs::Value { value, format } => match format { + ArgsFormat::Bin => BinFormatInlineContentSnafu { canister }.fail(), + fmt => Ok(InitArgs::Text { + content: value.trim().to_owned(), + format: fmt.clone(), + }), + }, + } +} + +/// Resolve a manifest's [`ManifestSettings`] into the model's [`Settings`] by +/// reading any file-backed environment variable values relative to `base_path`. +/// Also returns the file each such value came from, for +/// [`Canister::environment_variable_files`]. +async fn resolve_manifest_settings( + files: &dyn FileAccess, + manifest_settings: &ManifestSettings, + base_path: &Path, + canister: &str, +) -> Result<(Settings, BTreeMap), ConsolidateManifestError> { + let ManifestSettings { + log_visibility, + compute_allocation, + memory_allocation, + freezing_threshold, + reserved_cycles_limit, + wasm_memory_limit, + wasm_memory_threshold, + log_memory_limit, + environment_variables, + controllers, + } = manifest_settings; + + let mut env_files = BTreeMap::new(); + let mut resolved_vars = None; + if let Some(vars) = environment_variables { + let mut resolved = HashMap::with_capacity(vars.len()); + for (name, var) in vars { + let value = match var { + ManifestEnvVar::Value(value) => value.to_owned(), + ManifestEnvVar::Path { path } => { + let file = base_path.join(path); + let contents = files.read_to_string(&file).await.context( + ReadEnvironmentVariableSnafu { + canister, + variable: name, + }, + )?; + env_files.insert(name.to_owned(), file); + contents.trim().to_owned() + } + }; + resolved.insert(name.to_owned(), value); + } + resolved_vars = Some(resolved); + } + + let settings = Settings { + log_visibility: log_visibility.clone(), + compute_allocation: *compute_allocation, + memory_allocation: memory_allocation.clone(), + freezing_threshold: freezing_threshold.clone(), + reserved_cycles_limit: reserved_cycles_limit.clone(), + wasm_memory_limit: wasm_memory_limit.clone(), + wasm_memory_threshold: wasm_memory_threshold.clone(), + log_memory_limit: log_memory_limit.clone(), + environment_variables: resolved_vars, + controllers: controllers.clone(), + }; + Ok((settings, env_files)) +} + +fn is_glob(s: &str) -> bool { + s.contains('*') || s.contains('?') || s.contains('[') || s.contains('{') +} + +/// Collect `dir` and all of its descendant directories (recursively), used to +/// expand a `**` glob segment through the injected [`FileAccess`]. +async fn collect_descendant_dirs( + files: &dyn FileAccess, + dir: &Path, + out: &mut Vec, +) -> Result<(), ConsolidateManifestError> { + // Iterative BFS to avoid boxing a recursive async fn. + let mut queue = vec![dir.to_owned()]; + while let Some(d) = queue.pop() { + let entries = files.read_dir(&d).await.context(ListDirSnafu)?; + for entry in entries { + if files.is_dir(&entry).await { + out.push(entry.clone()); + queue.push(entry); + } + } + } + Ok(()) +} + +/// Expand a glob `pattern` (relative to `base`) into concrete paths, using the +/// injected [`FileAccess`] instead of the real filesystem. Supports literal +/// segments, single-segment wildcards (`*`, `?`, `[...]`, `{...}` via +/// [`glob::Pattern`]), and the `**` recursive segment. +async fn expand_glob( + files: &dyn FileAccess, + base: &Path, + pattern: &str, +) -> Result, ConsolidateManifestError> { + let mut current = vec![base.to_owned()]; + for seg in pattern.split('/') { + if seg.is_empty() { + continue; + } + let mut next = Vec::new(); + if seg == "**" { + for dir in ¤t { + next.push(dir.clone()); + collect_descendant_dirs(files, dir, &mut next).await?; + } + } else if is_glob(seg) { + let pat = glob::Pattern::new(seg).context(GlobParseSnafu)?; + for dir in ¤t { + if !files.is_dir(dir).await { + continue; + } + for entry in files.read_dir(dir).await.context(ListDirSnafu)? { + if let Some(name) = entry.file_name() + && pat.matches(name) + { + next.push(entry); + } + } + } + } else { + for dir in ¤t { + next.push(dir.join(seg)); + } + } + current = next; + } + Ok(current) +} + +/// Whether `name` is a valid canister name or dependency alias: non-empty and +/// containing only ASCII letters, digits, `_`, or `-`. +/// +/// A single strict rule keeps names safe for every purpose they are reused for — +/// store-key segments, `PUBLIC_CANISTER_ID:` env vars, DNS subdomains, and +/// archive paths — so no per-site sanitizing is needed. In particular `:` is the +/// dependency namespace separator, and `.` / `/` would be ambiguous in +/// subdomains and paths. +fn is_valid_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') +} + +/// Builds the canonical canisters declared directly in one project manifest, +/// resolving glob/path/inline entries, recipes, and init-args relative to +/// `pdir`. Returns `(local name, canister dir, canister)` with empty bindings; +/// callers assign store keys and bindings. Does not check for duplicate names +/// across projects — that is the caller's responsibility (via the global map). +async fn build_manifest_canisters( + files: &dyn FileAccess, + pdir: &Path, + manifest_canisters: &[Item], + recipe_resolver: &dyn recipe::RemoteResourceResolve, +) -> Result, ConsolidateManifestError> { + let mut result: Vec<(String, PathBuf, Canister)> = Vec::new(); + + for i in manifest_canisters { + let ms = match i { + Item::Path(pattern) => { + let is_glob_pattern = is_glob(pattern); + let paths = if is_glob_pattern { + expand_glob(files, pdir, pattern).await? + } else { + vec![pdir.join(pattern)] + }; + + let paths = if is_glob_pattern { + // For glob patterns, filter out non-directories and non-canister directories + let mut kept = Vec::new(); + for p in paths { + if files.is_dir(&p).await && files.exists(&p.join(CANISTER_MANIFEST)).await + { + kept.push(p); + } + } + kept + } else { + // For explicit paths, validate that they exist and contain canister.yaml + let mut validated_paths = vec![]; + for p in paths { + if !files.is_file(&p.join(CANISTER_MANIFEST)).await { + return NotFoundSnafu { + kind: "canister".to_string(), + path: pattern.to_string(), + } + .fail(); + } + validated_paths.push(p); + } + validated_paths + }; + + let mut ms = vec![]; + for p in paths { + ms.push(( + p.to_owned(), + load_manifest::(files, &p.join(CANISTER_MANIFEST)) + .await + .context(LoadCanisterSnafu)?, + )); + } + ms + } + + Item::Manifest(m) => vec![(pdir.to_owned(), m.to_owned())], + }; + + for (cdir, m) in ms { + if !is_valid_name(&m.name) { + return InvalidCanisterNameSnafu { + name: m.name.clone(), + } + .fail(); + } + + let registry_recipe = match &m.instructions { + Instructions::BuildSync { .. } => None, + Instructions::Recipe { recipe, .. } => match &recipe.recipe_type { + RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), + _ => None, + }, + }; + + let (build, sync) = + match &m.instructions { + // Build/Sync + Instructions::BuildSync { build, sync } => ( + build.to_owned(), + match sync { + Some(sync) => sync.to_owned(), + None => SyncSteps::default(), + }, + ), + + // Recipe: fetch the template through the resolver, then render + // and parse it into concrete steps. + Instructions::Recipe { + recipe, + sync: extra_sync, + } => { + let ctx = recipe::RecipeContext { + canister_name: m.name.clone(), + }; + let fetched = recipe_resolver.resolve_recipe(recipe).await.context( + FetchRecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + }, + )?; + let steps = recipe::render_recipe(&fetched.template, recipe, &ctx) + .context(RenderRecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + })?; + // The template rendered, so an unpinned download is now + // known good and safe to cache. Committing only here is what + // keeps a bad remote response from becoming sticky. + recipe_resolver + .commit_recipe(recipe, &fetched) + .await + .context(CacheRecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + })?; + + // The manifest's own sync steps run after the recipe's. + let (build, mut sync) = steps; + if let Some(extra_sync) = extra_sync { + sync.steps.extend(extra_sync.steps.iter().cloned()); + } + (build, sync) + } + }; + + let (settings, environment_variable_files) = + resolve_manifest_settings(files, &m.settings, &cdir, &m.name).await?; + + let init_args = match m.init_args.as_ref() { + Some(mia) => Some(resolve_manifest_init_args(files, mia, &cdir, &m.name).await?), + None => None, + }; + + result.push(( + m.name.clone(), + cdir, + Canister { + name: m.name.clone(), + settings, + build, + sync, + init_args, + registry_recipe, + bindings: BTreeMap::new(), + // Default to the bare local name; overwritten with the + // dot-nested alias form when the canister is imported as a + // dependency (see `import_dependency`). + friendly_names: vec![m.name.clone()], + environment_variable_files, + }, + )); + } + } + + Ok(result) +} + +/// A dependency instance imported into the workspace. Returned by +/// [`import_dependency`] and cached per canonical path so diamond dependencies +/// reuse the same instance. +#[derive(Clone)] +struct ImportedInstance { + /// This instance's own canisters, as `(local name, full store key)` — the + /// set exposable to the parent via `canisters:` selection. + own: Vec<(String, String)>, + /// Every canister in this instance's subtree (its own canisters plus all + /// transitively imported ones), as `(store key, local name, alias chain + /// from this instance down to the canister's owning project)`. Used to + /// register a friendly URL per alias chain when the instance is reached + /// again via de-duplication (a diamond), including for its descendants. + subtree: Vec<(String, String, Vec)>, +} + +/// A member environment's per-canister config, to be folded into the root's +/// same-named environment beneath any root overrides. +#[derive(Default, Clone)] +struct MemberCanisterOverride { + settings: Option, + init_args: Option, +} + +/// Per-environment member overrides: env name → store key → override. +type MemberEnvOverrides = HashMap>; + +/// A member's identity (store-key prefix) and the environment names it defines, +/// used to enforce that a member declares every environment the root targets +/// (strict rule). +struct MemberEnvInfo { + prefix: String, + defined: HashSet, +} + +/// Canonicalize a dependency root (resolving symlinks and `..`) for use as a +/// de-dup / cycle-detection identity. +async fn canonicalize_dep( + files: &dyn FileAccess, + alias: &str, + dep_root: &Path, +) -> Result { + files.canonicalize(dep_root).await.ok_or_else(|| { + DependencyCanonicalizeSnafu { + alias: alias.to_owned(), + path: dep_root.to_string(), + } + .build() + }) +} + +/// Store-key prefix for a dependency instance: its canonical directory relative +/// to the canonical app root, forward-slash separated so keys are stable across +/// platforms and independent of how each edge spells the path. +pub fn relative_prefix(app_root_canonical: &Path, dep_canonical: &Path) -> String { + let rel = pathdiff::diff_utf8_paths(dep_canonical, app_root_canonical) + .unwrap_or_else(|| dep_canonical.to_owned()); + rel.as_str().replace('\\', "/") +} + +/// Build a dependency canister's friendly-URL subdomain prefix: the canister's +/// local name as the most-specific label, followed by its alias chain reversed +/// (root-most alias last). E.g. local `backend` reached via `[service-a, +/// openemail]` → `backend.openemail.service-a`. Dot-nested so it stays a valid, +/// collision-free multi-label host; see DESIGN §17.2. +fn friendly_name_for(local: &str, alias_chain: &[String]) -> String { + let mut labels = Vec::with_capacity(alias_chain.len() + 1); + labels.push(local.to_string()); + labels.extend(alias_chain.iter().rev().cloned()); + labels.join(".") +} + +/// Rewrite `CanisterName` controller references from a dependency's local +/// canister names to their store keys, so global controller validation and +/// deploy-time id lookup operate uniformly on store keys. +fn translate_controllers(canister: &mut Canister, local_to_key: &BTreeMap) { + translate_settings_controllers(&mut canister.settings, local_to_key); +} + +/// Rewrite `CanisterName` controller references in a `Settings` from a +/// dependency's local canister names to their store keys. +fn translate_settings_controllers( + settings: &mut Settings, + local_to_key: &BTreeMap, +) { + if let Some(controllers) = &mut settings.controllers { + for cref in controllers.iter_mut() { + if let ControllerRef::CanisterName(name) = cref + && let Some(key) = local_to_key.get(name) + { + *name = key.clone(); + } + } + } +} + +/// Compute the `PUBLIC_CANISTER_ID` env-var wiring for canisters in one project +/// scope: its own canisters by local name, plus each dependency's exposed +/// canisters under `:`. +fn compute_bindings( + own: &[(String, String)], + edges: &[(String, Vec<(String, String)>)], +) -> BTreeMap { + let mut bindings = BTreeMap::new(); + for (local, key) in own { + bindings.insert(local.clone(), key.clone()); + } + for (alias, exposed) in edges { + for (dep_local, key) in exposed { + bindings.insert(format!("{alias}:{dep_local}"), key.clone()); + } + } + bindings +} + +/// Select which of a dependency instance's own canisters are exposed to the +/// parent, per the dependency's `canisters` selection. +fn select_exposed( + own: &[(String, String)], + selection: &CanisterSelection, + alias: &str, +) -> Result, ConsolidateManifestError> { + match selection { + CanisterSelection::Everything => Ok(own.to_vec()), + CanisterSelection::None => Ok(vec![]), + CanisterSelection::Named(names) => { + let mut out = Vec::new(); + for name in names { + match own.iter().find(|(local, _)| local == name) { + Some(pair) => out.push(pair.clone()), + None => { + return UnknownDependencyCanisterSnafu { + alias: alias.to_owned(), + canister: name.clone(), + } + .fail(); + } + } + } + Ok(out) + } + } +} + +/// Validate the dependency aliases declared in one project scope: no `:`, no +/// collision with a local canister name, and no duplicate alias. +fn validate_dependency_aliases( + deps: &[DependencyManifest], + own_canister_names: &HashSet, +) -> Result<(), ConsolidateManifestError> { + let mut seen: HashSet<&str> = HashSet::new(); + for d in deps { + if !is_valid_name(&d.name) { + return InvalidDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + if own_canister_names.contains(&d.name) { + return DependencyAliasCollisionSnafu { + alias: d.name.clone(), + } + .fail(); + } + if !seen.insert(&d.name) { + return DuplicateDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + } + Ok(()) +} + +/// Recursively import a dependency's canisters into `canisters`, keyed by their +/// app-root-relative store keys. De-duplicates instances by canonical path +/// (diamond dependencies deploy once) and detects cycles. Returns the imported +/// instance's prefix and its own canisters. +#[allow(clippy::too_many_arguments)] +async fn import_dependency( + files: &dyn FileAccess, + app_root_canonical: &Path, + parent_dir: &Path, + dep: &DependencyManifest, + recipe_resolver: &dyn recipe::RemoteResourceResolve, + canisters: &mut IndexMap, + registry: &mut HashMap, + stack: &mut Vec, + member_env_overrides: &mut MemberEnvOverrides, + members: &mut Vec, + // Alias chain from the workspace root to and including this dependency, + // used to build friendly-URL subdomains (§17.2). + alias_chain: &[String], +) -> Result { + let dep_root = parent_dir.join(&dep.path); + let manifest_path = dep_root.join(PROJECT_MANIFEST); + if !files.is_file(&manifest_path).await { + return DependencyNotFoundSnafu { + alias: dep.name.clone(), + path: dep_root.to_string(), + } + .fail(); + } + + let canonical = canonicalize_dep(files, &dep.name, &dep_root).await?; + + // Cycle detection. + if stack.contains(&canonical) { + let mut chain: Vec = stack.iter().map(|p| p.to_string()).collect(); + chain.push(canonical.to_string()); + return CircularDependencySnafu { + chain: chain.join(" -> "), + } + .fail(); + } + + // Diamond de-dup: same resolved directory means the same instance, deployed + // once. It is still reachable via this new alias chain, so register an + // additional friendly URL per chain (§17.3) rather than picking one — for + // the whole subtree (its own canisters *and* its transitive dependencies), + // each named by this chain extended with the canister's alias path below the + // instance. + if let Some(inst) = registry.get(&canonical) { + let inst = inst.clone(); + for (key, local, rel_chain) in &inst.subtree { + let mut chain = alias_chain.to_vec(); + chain.extend(rel_chain.iter().cloned()); + let fname = friendly_name_for(local, &chain); + if let Some((_, canister)) = canisters.get_mut(key) + && !canister.friendly_names.contains(&fname) + { + canister.friendly_names.push(fname); + } + } + return Ok(inst); + } + + stack.push(canonical.clone()); + + let prefix = relative_prefix(app_root_canonical, &canonical); + + let dep_manifest: ProjectManifest = + load_manifest(files, &manifest_path) + .await + .context(LoadDependencyManifestSnafu { + alias: dep.name.clone(), + })?; + + // Build the dependency's own canisters and key them under the prefix. All of + // them are imported (deploy-all); the `canisters` exposure subset is applied + // by the caller when wiring env vars. + let built = + build_manifest_canisters(files, &dep_root, &dep_manifest.canisters, recipe_resolver) + .await?; + + let mut own: Vec<(String, String)> = Vec::new(); + let mut local_to_key: BTreeMap = BTreeMap::new(); + for (local, cdir, mut canister) in built { + let store_key = format!("{prefix}:{local}"); + canister.name = store_key.clone(); + // Friendly URL from the alias chain, not the path-based store key. + canister.friendly_names = vec![friendly_name_for(&local, alias_chain)]; + own.push((local.clone(), store_key.clone())); + local_to_key.insert(local.clone(), store_key.clone()); + match canisters.entry(store_key.clone()) { + IndexEntry::Occupied(_) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: store_key, + } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); + } + } + } + + // Now that every sibling's store key is known, translate the dependency's + // controller references (local sibling name -> store key). + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + translate_controllers(canister, &local_to_key); + } + } + + // Capture the member's own environments so the parent can honor its + // per-canister settings/init_args for the same-named environment + // (standalone-equivalence). The network binding and canister selection are + // ignored; only overrides on the member's *own* canisters are + // folded in — keys naming its dependencies are left to those dependencies. + let mut defined_envs: HashSet = HashSet::new(); + for env_item in &dep_manifest.environments { + let em: EnvironmentManifest = match env_item { + Item::Manifest(m) => m.clone(), + Item::Path(path) => { + let p = dep_root.join(path); + if !files.is_file(&p).await { + return NotFoundSnafu { + kind: "environment".to_string(), + path: p.to_string(), + } + .fail(); + } + load_manifest::(files, &p) + .await + .context(LoadEnvironmentSnafu)? + } + }; + defined_envs.insert(em.name.clone()); + if let Some(settings) = &em.settings { + for (local, s) in settings { + if let Some(key) = local_to_key.get(local) { + // Translate the override's own controller references from the + // member's local names to store keys, so name-based controllers + // resolve against the workspace id map just like base settings. + let mut s = s.clone(); + translate_settings_controllers(&mut s, &local_to_key); + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .settings = Some(s); + } + } + } + if let Some(init_args) = &em.init_args { + for (local, ia) in init_args { + if let Some(key) = local_to_key.get(local) { + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .init_args = Some(ia.clone()); + } + } + } + } + members.push(MemberEnvInfo { + prefix: prefix.clone(), + defined: defined_envs, + }); + + // Recurse into the dependency's own dependencies. + let own_names: HashSet = own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&dep_manifest.dependencies, &own_names)?; + + // The instance's subtree, for diamond-hit friendly-URL propagation: its own + // canisters sit at the instance root (empty relative alias chain); each + // nested dependency contributes its subtree prefixed with the nested alias. + let mut subtree: Vec<(String, String, Vec)> = own + .iter() + .map(|(local, key)| (key.clone(), local.clone(), Vec::new())) + .collect(); + + let mut edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for nested in &dep_manifest.dependencies { + let mut nested_chain = alias_chain.to_vec(); + nested_chain.push(nested.name.clone()); + let inst = Box::pin(import_dependency( + files, + app_root_canonical, + &dep_root, + nested, + recipe_resolver, + canisters, + registry, + stack, + member_env_overrides, + members, + &nested_chain, + )) + .await?; + for (key, local, rel) in &inst.subtree { + let mut r = Vec::with_capacity(rel.len() + 1); + r.push(nested.name.clone()); + r.extend(rel.iter().cloned()); + subtree.push((key.clone(), local.clone(), r)); + } + let exposed = select_exposed(&inst.own, &nested.canisters, &nested.name)?; + edges.push((nested.name.clone(), exposed)); + } + + // Assign env-var bindings for this instance's own canisters. + let bindings = compute_bindings(&own, &edges); + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = bindings.clone(); + } + } + + stack.pop(); + let instance = ImportedInstance { own, subtree }; + registry.insert(canonical, instance.clone()); + Ok(instance) +} + +/// Build one environment's canister map: select from `canisters`, then apply the +/// member overrides for this environment (standalone-equivalence), then +/// the root's own overrides (highest precedence). Precedence is therefore +/// root-explicit > member-env > canister-base. +async fn build_environment_canisters( + files: &dyn FileAccess, + canisters: &IndexMap, + env_name: &str, + selection: &CanisterSelection, + member_overrides: Option<&HashMap>, + root_settings: Option<&HashMap>, + root_init_args: Option<&HashMap>, +) -> Result, ConsolidateManifestError> { + let mut cs = match selection { + CanisterSelection::None => IndexMap::new(), + CanisterSelection::Everything => canisters.clone(), + CanisterSelection::Named(names) => { + let mut cs: IndexMap = IndexMap::new(); + for name in names { + let v = canisters.get(name).ok_or( + InvalidCanisterSnafu { + environment: env_name.to_owned(), + canister: name.to_owned(), + } + .build(), + )?; + cs.insert(name.to_owned(), v.to_owned()); + } + cs + } + }; + + // Member overrides first (lower precedence than the root's own overrides). + if let Some(overrides) = member_overrides { + for (key, ov) in overrides { + if let Some((cpath, canister)) = cs.get_mut(key) { + if let Some(s) = &ov.settings { + (canister.settings, canister.environment_variable_files) = + resolve_manifest_settings(files, s, cpath, key).await?; + } + if let Some(ia) = &ov.init_args { + canister.init_args = + Some(resolve_manifest_init_args(files, ia, cpath, key).await?); + } + } + } + } + + // Root overrides last (highest precedence). + if let Some(settings) = root_settings { + for (name, s) in settings { + if let Some((cpath, canister)) = cs.get_mut(name) { + (canister.settings, canister.environment_variable_files) = + resolve_manifest_settings(files, s, cpath, name).await?; + } + } + } + if let Some(init_args) = root_init_args { + for (name, ia) in init_args { + if let Some((cpath, canister)) = cs.get_mut(name) { + canister.init_args = + Some(resolve_manifest_init_args(files, ia, cpath, name).await?); + } + } + } + + Ok(cs) +} + +/// Turns the ProjectManifest into a Project struct +/// - Adds the default Networks +/// - Adds the default Environment +/// - Imports any dependency projects' canisters +/// - Validates the manifest to make sure that: +/// - There are no duplicates +/// - All the environments have networks +/// - All the referenced canisters exist +/// - All the recipes have been resolved +pub async fn consolidate_manifest( + files: &dyn FileAccess, + pdir: &Path, + recipe_resolver: &dyn recipe::RemoteResourceResolve, + m: &ProjectManifest, +) -> Result { + // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved + // through to consumers like `icp project bundle`, which needs reproducible output. + let mut canisters: IndexMap = IndexMap::new(); + + // Canonical app root, used to derive stable, order-independent store-key + // prefixes for imported dependency canisters. + let app_root_canonical = files + .canonicalize(pdir) + .await + .unwrap_or_else(|| pdir.to_owned()); + + // This project's own canisters, keyed by their bare local names. + let app_built = build_manifest_canisters(files, pdir, &m.canisters, recipe_resolver).await?; + let mut app_own: Vec<(String, String)> = Vec::new(); + for (local, cdir, canister) in app_built { + app_own.push((local.clone(), local.clone())); + match canisters.entry(local.clone()) { + IndexEntry::Occupied(e) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); + } + } + } + + // Import dependency projects. Each dependency is deployed in full and keyed + // under its app-root-relative path; diamonds (the same directory reached via + // multiple edges) resolve to a single instance. + let mut registry: HashMap = HashMap::new(); + let mut stack: Vec = Vec::new(); + // Member environment config folded into the root's same-named environments, + // and the per-member set of declared environment names for the strict rule. + let mut member_env_overrides: MemberEnvOverrides = HashMap::new(); + let mut members: Vec = Vec::new(); + let app_own_names: HashSet = app_own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&m.dependencies, &app_own_names)?; + + let mut app_edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for dep in &m.dependencies { + let inst = import_dependency( + files, + &app_root_canonical, + pdir, + dep, + recipe_resolver, + &mut canisters, + &mut registry, + &mut stack, + &mut member_env_overrides, + &mut members, + std::slice::from_ref(&dep.name), + ) + .await?; + let exposed = select_exposed(&inst.own, &dep.canisters, &dep.name)?; + app_edges.push((dep.name.clone(), exposed)); + } + + // Assign env-var bindings for this project's own canisters (own canisters by + // local name plus each dependency's exposed canisters under `:`). + let app_bindings = compute_bindings(&app_own, &app_edges); + for (_, key) in &app_own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = app_bindings.clone(); + } + } + + // Friendly URLs need no de-collision pass: the strict name rule (no '.') makes + // own canisters single-label and dependency canisters multi-label (dot-nested + // by alias chain), so their hostnames are disjoint by construction (§17.2). + + // Validate that every canister-name controller reference points to a declared canister. + // Catching typos here turns "perpetual warning" into a clear load-time error. + for (canister_name, (_, canister)) in &canisters { + let Some(crefs) = &canister.settings.controllers else { + continue; + }; + for cref in crefs { + if let Some(ref_name) = cref.canister_name() + && !canisters.contains_key(ref_name) + { + return UnknownControllerCanisterSnafu { + canister: canister_name.to_owned(), + controller: ref_name.to_owned(), + } + .fail(); + } + } + } + + // Networks + let mut networks: HashMap = HashMap::new(); + + // Add IC network first - this is always protected and non-overridable + networks.insert( + IC.to_string(), + Network { + name: IC.to_string(), + configuration: Configuration::Connected { + connected: Connected { + api_url: IC_MAINNET_NETWORK_API_URL.parse().unwrap(), + http_gateway_url: Some(IC_MAINNET_NETWORK_GATEWAY_URL.parse().unwrap()), + root_key: RootKeySpec::Mainnet, + }, + }, + }, + ); + + // Track which network names are protected (only IC network) + let protected_network_names: HashSet = [IC.to_string()].into_iter().collect(); + + // Resolve NetworkManifests and add them (including user-defined "local" if provided) + for i in &m.networks { + let m = match i { + Item::Path(path) => { + let path = pdir.join(path); + if !files.is_file(&path).await { + return NotFoundSnafu { + kind: "network".to_string(), + path: path.to_string(), + } + .fail(); + } + load_manifest::(files, &path) + .await + .context(LoadNetworkSnafu)? + } + Item::Manifest(ms) => ms.clone(), + }; + + match networks.entry(m.name.to_owned()) { + // Duplicate + Entry::Occupied(e) => { + // Only error if trying to override a protected network + if protected_network_names.contains(&m.name) { + return ReservedSnafu { + kind: "network".to_string(), + name: m.name.to_string(), + } + .fail(); + } + + // For non-protected duplicates, this is a user error (defining same network twice) + return DuplicateSnafu { + kind: "network".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + + // Ok + Entry::Vacant(e) => { + e.insert(Network { + name: m.name.to_owned(), + configuration: m.configuration.into(), // Convert manifest to config struct + }); + } + } + } + + // After processing user networks, add default "local" if not already defined + // This provides backward compatibility for projects that don't define their own "local" network + if !networks.contains_key(LOCAL) { + networks.insert( + LOCAL.to_string(), + Network { + name: LOCAL.to_string(), + configuration: Configuration::Managed { + managed: Managed { + mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway: Gateway { + bind: DEFAULT_LOCAL_NETWORK_BIND.to_string(), + port: Port::Fixed(DEFAULT_LOCAL_NETWORK_PORT), + domains: vec![], + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + version: None, + })), + }, + }, + }, + ); + } + + // Environments + let mut environments: HashMap = HashMap::new(); + + for i in &m.environments { + let m = match i { + Item::Path(path) => { + let path = pdir.join(path); + if !files.is_file(&path).await { + return NotFoundSnafu { + kind: "environment".to_string(), + path: path.to_string(), + } + .fail(); + } + load_manifest::(files, &path) + .await + .context(LoadEnvironmentSnafu)? + } + Item::Manifest(ms) => ms.clone(), + }; + + match environments.entry(m.name.to_owned()) { + // Duplicate + Entry::Occupied(e) => { + return DuplicateSnafu { + kind: "environment".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + + // Ok + Entry::Vacant(e) => { + e.insert(Environment { + name: m.name.to_owned(), + + // Embed network in environment + network: { + let v = networks.get(&m.network).ok_or( + InvalidNetworkSnafu { + environment: m.name.to_owned(), + network: m.network.to_owned(), + } + .build(), + )?; + + v.to_owned() + }, + + // Embed canisters in environment, folding member overrides + // beneath the root's own settings/init_args overrides. + canisters: build_environment_canisters( + files, + &canisters, + &m.name, + &m.canisters, + member_env_overrides.get(&m.name), + m.settings.as_ref(), + m.init_args.as_ref(), + ) + .await?, + }); + } + } + } + + // We're done adding all the user environments + // Now we add the implicit `local` and `ic` environment if the user hasn't overriden it + if let Entry::Vacant(vacant_entry) = environments.entry(LOCAL.to_string()) { + let network = networks + .get(LOCAL) + .ok_or( + InvalidNetworkSnafu { + environment: LOCAL.to_owned(), + network: LOCAL.to_owned(), + } + .build(), + )? + .to_owned(); + vacant_entry.insert(Environment { + name: LOCAL.to_string(), + network, + canisters: build_environment_canisters( + files, + &canisters, + LOCAL, + &CanisterSelection::Everything, + member_env_overrides.get(LOCAL), + None, + None, + ) + .await?, + }); + } + if let Entry::Vacant(vacant_entry) = environments.entry(IC.to_string()) { + let network = networks + .get(IC) + .ok_or( + InvalidNetworkSnafu { + environment: IC.to_owned(), + network: IC.to_owned(), + } + .build(), + )? + .to_owned(); + vacant_entry.insert(Environment { + name: IC.to_string(), + network, + canisters: build_environment_canisters( + files, + &canisters, + IC, + &CanisterSelection::Everything, + member_env_overrides.get(IC), + None, + None, + ) + .await?, + }); + } + + // Strict rule: every member must declare each environment the root targets. + // `local`/`ic` are implicit for every project, so they never count + // as missing; other environments must be declared explicitly by the member. + // Recorded per-environment and enforced lazily when that environment is + // selected (so a missing `staging` never blocks `deploy -e local`). + let mut member_missing_envs: HashMap> = HashMap::new(); + for env_name in environments.keys() { + if env_name == LOCAL || env_name == IC { + continue; + } + for member in &members { + if !member.defined.contains(env_name) { + member_missing_envs + .entry(env_name.clone()) + .or_default() + .push(member.prefix.clone()); + } + } + } + + Ok(Project { + dir: pdir.into(), + canisters, + networks, + environments, + member_missing_envs, + }) +} + +#[derive(Debug, Snafu)] +pub enum LoadProjectError { + #[snafu(display("failed to load project manifest"))] + ProjectManifest { source: LoadManifestError }, + + #[snafu(transparent)] + Consolidate { source: ConsolidateManifestError }, +} + +/// Load and consolidate the project rooted at `project_dir` (already located by +/// the caller), reading all files through `files` and resolving recipes through +/// `recipe`. +pub async fn load_project( + files: &dyn FileAccess, + recipe: &dyn recipe::RemoteResourceResolve, + project_dir: &Path, +) -> Result { + let m: ProjectManifest = load_manifest(files, &project_dir.join(PROJECT_MANIFEST)) + .await + .context(ProjectManifestSnafu)?; + let p = consolidate_manifest(files, project_dir, recipe, &m).await?; + Ok(p) +} + +#[derive(Debug, Snafu)] +pub enum VerifySandboxError { + #[snafu(display( + "canister '{canister}' uses a script {phase} step, which cannot run in the sandbox; \ + only pre-built builds and plugin syncs are permitted" + ))] + ScriptStep { canister: String, phase: String }, +} + +/// Verify that a fully-resolved project (recipes already resolved into concrete +/// steps) contains no script steps. Script build/sync steps spawn host +/// subprocesses and therefore cannot run inside the sandbox; only pre-built +/// builds and plugin syncs are permitted. +pub fn verify_sandbox(project: &Project) -> Result<(), VerifySandboxError> { + use crate::manifest::canister::{BuildStep, SyncStep}; + + for (name, (_, canister)) in &project.canisters { + if canister + .build + .steps + .iter() + .any(|s| matches!(s, BuildStep::Script(_))) + { + return ScriptStepSnafu { + canister: name.clone(), + phase: "build", + } + .fail(); + } + if canister + .sync + .steps + .iter() + .any(|s| matches!(s, SyncStep::Script(_))) + { + return ScriptStepSnafu { + canister: name.clone(), + phase: "sync", + } + .fail(); + } + } + Ok(()) +} + +#[cfg(test)] +mod recipe_sync_tests { + use super::*; + use crate::canister::recipe::{FetchedRecipe, RemoteResourceResolve, ResolveError}; + use crate::manifest::adapter::prebuilt::SourceField; + use crate::manifest::canister::SyncStep; + use crate::manifest::recipe::Recipe; + use crate::sync_exec::StepProgress; + use crate::testutil::HostFiles; + use camino_tempfile::Utf8TempDir; + + /// Hands back one fixed template for every recipe, without touching the + /// network or the cache. + struct FixedResolver(&'static str); + + #[async_trait::async_trait] + impl RemoteResourceResolve for FixedResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + Ok(FetchedRecipe { + template: self.0.to_owned(), + deferred: false, + }) + } + + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + Ok(()) + } + + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _progress: Option<&dyn StepProgress>, + ) -> Result { + panic!("wasm resolver should not be called in recipe sync tests"); + } + } + + const TEMPLATE: &str = indoc::indoc! {r#" + build: + steps: + - type: script + command: build.sh + sync: + steps: + - type: script + command: echo recipe + "#}; + + async fn consolidate(pdir: &Path) -> Result { + let files = HostFiles; + let m: ProjectManifest = load_manifest(&files, &pdir.join(PROJECT_MANIFEST)) + .await + .expect("failed to parse project manifest"); + consolidate_manifest(&files, pdir, &FixedResolver(TEMPLATE), &m).await + } + + /// The commands of a canister's sync steps, which are all script steps here. + fn sync_commands(p: &Project, key: &str) -> Vec { + p.canisters + .get(key) + .expect("canister not found") + .1 + .sync + .steps + .iter() + .map(|s| match s { + SyncStep::Script(adapter) => adapter.command.as_vec().join(" "), + other => panic!("expected a script sync step, got {other:?}"), + }) + .collect() + } + + /// A canister may add sync steps of its own on top of a recipe's; they run + /// after the ones the recipe renders. + #[tokio::test] + async fn manifest_sync_steps_follow_the_recipes() { + let tmp = Utf8TempDir::new().unwrap(); + std::fs::write( + tmp.path().join(PROJECT_MANIFEST), + indoc::indoc! {r#" + canisters: + - name: backend + recipe: + type: file://recipe.hbs + sync: + steps: + - type: script + command: echo manifest + "#}, + ) + .unwrap(); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!( + sync_commands(&p, "backend"), + ["echo recipe", "echo manifest"] + ); + } + + /// Without a `sync` section, a recipe canister still gets exactly the + /// recipe's own sync steps. + #[tokio::test] + async fn recipe_sync_steps_alone_when_manifest_has_none() { + let tmp = Utf8TempDir::new().unwrap(); + std::fs::write( + tmp.path().join(PROJECT_MANIFEST), + indoc::indoc! {r#" + canisters: + - name: backend + recipe: + type: file://recipe.hbs + "#}, + ) + .unwrap(); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!(sync_commands(&p, "backend"), ["echo recipe"]); + } +} + +#[cfg(test)] +mod dependency_tests { + use super::*; + use crate::canister::recipe::{FetchedRecipe, RemoteResourceResolve, ResolveError}; + use crate::manifest::adapter::prebuilt::SourceField; + use crate::manifest::recipe::Recipe; + use crate::sync_exec::StepProgress; + use crate::testutil::HostFiles; + use camino_tempfile::Utf8TempDir; + + /// Recipes and plugins are never used in these tests; every canister is pre-built. + struct PanicResolver; + + #[async_trait::async_trait] + impl RemoteResourceResolve for PanicResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + panic!("recipe resolver should not be called in dependency tests"); + } + + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + panic!("recipe resolver should not be called in dependency tests"); + } + + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _progress: Option<&dyn StepProgress>, + ) -> Result { + panic!("wasm resolver should not be called in dependency tests"); + } + } + + fn write(dir: &Path, rel: &str, contents: &str) { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, contents).unwrap(); + } + + /// A minimal `icp.yaml` body declaring the given pre-built canisters, + /// followed by a raw `dependencies:` block (may be empty). + fn manifest(canisters: &[&str], deps: &str) -> String { + let mut s = String::new(); + if canisters.is_empty() { + s.push_str("canisters: []\n"); + } else { + s.push_str("canisters:\n"); + for c in canisters { + s.push_str(&format!( + " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" + )); + } + } + s.push_str(deps); + s + } + + async fn consolidate(pdir: &Path) -> Result { + let files = HostFiles; + let m: ProjectManifest = load_manifest(&files, &pdir.join(PROJECT_MANIFEST)) + .await + .expect("failed to parse project manifest"); + consolidate_manifest(&files, pdir, &PanicResolver, &m).await + } + + fn bindings_of<'a>(p: &'a Project, key: &str) -> &'a BTreeMap { + &p.canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + p.canisters.keys().collect::>() + ) + }) + .1 + .bindings + } + + fn friendly_names_of<'a>(p: &'a Project, key: &str) -> &'a [String] { + &p.canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + p.canisters.keys().collect::>() + ) + }) + .1 + .friendly_names + } + + #[tokio::test] + async fn single_project_bindings_are_self_and_siblings() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Flat behavior preserved: every canister maps every sibling (incl. self) + // to itself. + let expected = BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ("frontend".to_string(), "frontend".to_string()), + ]); + assert_eq!(bindings_of(&p, "backend"), &expected); + assert_eq!(bindings_of(&p, "frontend"), &expected); + } + + #[tokio::test] + async fn dependency_import_and_exposure_subset() { + let tmp = Utf8TempDir::new().unwrap(); + // Dependency nested inside the app (mirrors a submodule under the app). + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [backend]\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // The whole dependency is deployed (both canisters imported), keyed by path. + assert!(p.canisters.contains_key("backend")); + assert!(p.canisters.contains_key("openemail:backend")); + assert!(p.canisters.contains_key("openemail:frontend")); + + // App's own canister sees itself and only the *exposed* dependency canister. + assert_eq!( + bindings_of(&p, "backend"), + &BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ( + "openemail:backend".to_string(), + "openemail:backend".to_string() + ), + ]) + ); + + // The dependency's own canisters keep their standalone view (bare names). + assert_eq!( + bindings_of(&p, "openemail:backend"), + &BTreeMap::from([ + ("backend".to_string(), "openemail:backend".to_string()), + ("frontend".to_string(), "openemail:frontend".to_string()), + ]) + ); + } + + #[tokio::test] + async fn member_env_config_folds_in_with_root_override_winning() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail defines `staging` with per-canister settings for its own + // canisters. + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - name: backend + build: + steps: + - type: pre-built + path: backend.wasm + - name: frontend + build: + steps: + - type: pre-built + path: frontend.wasm +environments: + - name: staging + settings: + backend: + compute_allocation: 5 + frontend: + compute_allocation: 7 +"#, + ); + // The app declares openemail and also defines `staging`, overriding the + // imported backend's settings (the root override must win). + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging + settings: + "openemail:backend": + compute_allocation: 99 +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + // Root override wins over the member's config. + assert_eq!( + staging + .canisters + .get("openemail:backend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(99), + ); + // No root override → the member's own config applies (standalone-equivalence). + assert_eq!( + staging + .canisters + .get("openemail:frontend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(7), + ); + // Both projects declared staging, so nothing is recorded as missing. + assert!(p.member_missing_envs.is_empty()); + } + + #[tokio::test] + async fn missing_member_environment_is_recorded() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + // openemail does not declare `staging`, so it is recorded as missing. + assert_eq!( + p.member_missing_envs.get("staging"), + Some(&vec!["openemail".to_string()]), + ); + // Implicit environments are never recorded as missing. + assert!(!p.member_missing_envs.contains_key("local")); + assert!(!p.member_missing_envs.contains_key("ic")); + } + + #[tokio::test] + async fn diamond_dedups_to_single_instance() { + let tmp = Utf8TempDir::new().unwrap(); + // umbrella layout: service-a and service-b both depend on ../openemail. + write( + tmp.path(), + "umbrella/openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "umbrella/service-a/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "umbrella/service-b/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // openemail is imported exactly once despite two edges reaching it. + let openemail_keys: Vec<_> = p + .canisters + .keys() + .filter(|k| k.contains("openemail")) + .collect(); + assert_eq!( + openemail_keys, + vec![&"umbrella/openemail:backend".to_string()], + "expected a single shared openemail instance" + ); + + // Both services' code reads `openemail:backend`, resolving to the one instance. + assert_eq!( + bindings_of(&p, "umbrella/service-a:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + assert_eq!( + bindings_of(&p, "umbrella/service-b:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + + // The single shared instance is reachable at one friendly URL per alias + // chain (§17.3) — the store-key path (`umbrella/`) never appears. + assert_eq!( + friendly_names_of(&p, "umbrella/openemail:backend"), + &["backend.openemail.service-a", "backend.openemail.service-b"] + ); + // Each service's own canister is named by its own alias chain. + assert_eq!( + friendly_names_of(&p, "umbrella/service-a:backend"), + &["backend.service-a"] + ); + assert_eq!( + friendly_names_of(&p, "umbrella/service-b:backend"), + &["backend.service-b"] + ); + } + + #[tokio::test] + async fn diamond_transitive_dependency_gets_url_per_chain() { + let tmp = Utf8TempDir::new().unwrap(); + // The shared openemail itself depends on libfoo, and is reached via both + // service-a and service-b. + write( + tmp.path(), + "umbrella/openemail/libfoo/icp.yaml", + &manifest(&["bar"], ""), + ); + write( + tmp.path(), + "umbrella/openemail/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: libfoo\n path: ./libfoo\n", + ), + ); + write( + tmp.path(), + "umbrella/service-a/icp.yaml", + &manifest( + &["service"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "umbrella/service-b/icp.yaml", + &manifest( + &["service"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // The shared instance's own canister gets one URL per chain... + assert_eq!( + friendly_names_of(&p, "umbrella/openemail:backend"), + &["backend.openemail.service-a", "backend.openemail.service-b"] + ); + // ...and so does its *transitive* dependency (the subtree is revisited on + // the diamond hit, not just the instance's own canisters). + assert_eq!( + friendly_names_of(&p, "umbrella/openemail/libfoo:bar"), + &[ + "bar.libfoo.openemail.service-a", + "bar.libfoo.openemail.service-b" + ] + ); + } + + #[tokio::test] + async fn dot_in_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + // '.' is banned: it would be ambiguous in a dot-nested friendly subdomain + // (an own canister named `frontend.openemail` could collide with dependency + // `openemail`'s `frontend`). The strict name rule rejects it up front. + write( + tmp.path(), + "icp.yaml", + &manifest(&["frontend.openemail"], ""), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn invalid_dependency_alias_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["app"], + "dependencies:\n - name: open.email\n path: ./openemail\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidDependencyAlias { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn member_override_controllers_are_translated_to_store_keys() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail's `staging` override names a controller by its local name. + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - name: backend + build: + steps: + - type: pre-built + path: backend.wasm + - name: frontend + build: + steps: + - type: pre-built + path: frontend.wasm +environments: + - name: staging + settings: + backend: + controllers: ["frontend"] +"#, + ); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + let controllers = staging + .canisters + .get("openemail:backend") + .unwrap() + .1 + .settings + .controllers + .clone() + .expect("controllers set by the member override"); + + // The member-local `frontend` must be translated to its store key, so it + // resolves against the workspace id map at deploy time. + assert_eq!( + controllers, + vec![ControllerRef::CanisterName( + "openemail:frontend".to_string() + )] + ); + } + + fn env_vars_of<'a>( + canisters: &'a IndexMap, + key: &str, + ) -> &'a HashMap { + canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + canisters.keys().collect::>() + ) + }) + .1 + .settings + .environment_variables + .as_ref() + .expect("environment variables set") + } + + /// A canister manifest's file-backed environment variable resolves against + /// the canister's own directory, and the file's trailing newline is not part + /// of the value. + #[tokio::test] + async fn env_var_file_resolves_against_canister_dir() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "canisters/backend/canister.yaml", + r#" +name: backend +settings: + environment_variables: + API_KEY: + path: secrets/api-key +build: + steps: + - type: pre-built + path: backend.wasm +"#, + ); + write(tmp.path(), "canisters/backend/secrets/api-key", "s3cret\n"); + write( + tmp.path(), + "icp.yaml", + "canisters:\n - ./canisters/backend\n", + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!( + env_vars_of(&p.canisters, "backend"), + &HashMap::from([("API_KEY".to_string(), "s3cret".to_string())]), + ); + } + + /// A canister declared in its own directory, for the override tests below: + /// its directory is neither the project's nor an environment manifest's, so + /// the base a path resolves against is unambiguous. + fn write_backend_canister(dir: &Path, at: &str) { + write( + dir, + &format!("{at}/canister.yaml"), + r#" +name: backend +build: + steps: + - type: pre-built + path: backend.wasm +"#, + ); + } + + /// An environment override resolves a path against the *canister's* directory + /// — the same base an `init_args` override uses — not against the manifest + /// declaring the override, even when that is an environment manifest of its + /// own. + #[tokio::test] + async fn env_var_file_in_environment_override_resolves_against_canister_dir() { + let tmp = Utf8TempDir::new().unwrap(); + write_backend_canister(tmp.path(), "canisters/backend"); + write( + tmp.path(), + "icp.yaml", + "canisters:\n - ./canisters/backend\nenvironments:\n - ./environments/staging.yaml\n", + ); + write( + tmp.path(), + "environments/staging.yaml", + r#" +name: staging +settings: + backend: + environment_variables: + API_KEY: + path: secrets/api-key +"#, + ); + write( + tmp.path(), + "canisters/backend/secrets/api-key", + "staging-key\n", + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + assert_eq!( + env_vars_of(&staging.canisters, "backend"), + &HashMap::from([("API_KEY".to_string(), "staging-key".to_string())]), + ); + // The override applies to the environment only; the canister's own + // settings are untouched. + assert_eq!( + p.canisters + .get("backend") + .unwrap() + .1 + .settings + .environment_variables, + None, + ); + } + + /// A member's own environment override resolves against the member's + /// canister, not against the member's or the root's project directory. + #[tokio::test] + async fn env_var_file_in_member_environment_resolves_against_member_canister_dir() { + let tmp = Utf8TempDir::new().unwrap(); + write_backend_canister(tmp.path(), "openemail/canisters/backend"); + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - ./canisters/backend +environments: + - name: staging + settings: + backend: + environment_variables: + API_KEY: + path: secrets/api-key +"#, + ); + write( + tmp.path(), + "openemail/canisters/backend/secrets/api-key", + "member-key\n", + ); + write( + tmp.path(), + "icp.yaml", + &format!( + "{}environments:\n - name: staging\n", + manifest( + &["app"], + "dependencies:\n - name: openemail\n path: ./openemail\n" + ) + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + assert_eq!( + env_vars_of(&staging.canisters, "openemail:backend"), + &HashMap::from([("API_KEY".to_string(), "member-key".to_string())]), + ); + } + + #[tokio::test] + async fn missing_env_var_file_is_reported_with_the_variable_and_canister() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: backend + settings: + environment_variables: + API_KEY: + path: secrets/api-key + build: + steps: + - type: pre-built + path: backend.wasm +"#, + ); + + let err = consolidate(tmp.path()) + .await + .expect_err("the environment variable's file does not exist"); + + assert!( + matches!( + &err, + ConsolidateManifestError::ReadEnvironmentVariable { canister, variable, .. } + if canister == "backend" && variable == "API_KEY" + ), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn friendly_names_are_bare_for_own_and_dotted_for_dependencies() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail (with a transitive dep libfoo) vendored under the app. + write( + tmp.path(), + "openemail/libfoo/icp.yaml", + &manifest(&["bar"], ""), + ); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest( + &["backend", "frontend"], + "dependencies:\n - name: libfoo\n path: ./libfoo\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Own canister: bare name (unchanged from single-project behavior). + assert_eq!(friendly_names_of(&p, "backend"), &["backend"]); + // Direct dependency: dot-nested by alias (no `vendor/` path noise). + assert_eq!( + friendly_names_of(&p, "openemail:backend"), + &["backend.openemail"] + ); + assert_eq!( + friendly_names_of(&p, "openemail:frontend"), + &["frontend.openemail"] + ); + // Transitive dependency: full alias chain, canister-most-specific first. + assert_eq!( + friendly_names_of(&p, "openemail/libfoo:bar"), + &["bar.libfoo.openemail"] + ); + } + + #[tokio::test] + async fn cycle_is_detected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&[], "dependencies:\n - name: a\n path: ./a\n"), + ); + write( + tmp.path(), + "a/icp.yaml", + &manifest(&["x"], "dependencies:\n - name: b\n path: ../b\n"), + ); + write( + tmp.path(), + "b/icp.yaml", + &manifest(&["y"], "dependencies:\n - name: a\n path: ../a\n"), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::CircularDependency { .. }), + "expected CircularDependency, got {err:?}" + ); + } + + #[tokio::test] + async fn alias_colliding_with_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["openemail"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DependencyAliasCollision { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn duplicate_alias_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "one/icp.yaml", &manifest(&["backend"], "")); + write(tmp.path(), "two/icp.yaml", &manifest(&["backend"], "")); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: dup\n path: ./one\n - name: dup\n path: ./two\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DuplicateDependencyAlias { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn colon_in_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "icp.yaml", &manifest(&["foo:bar"], "")); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn unknown_exposed_canister_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [nope]\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::UnknownDependencyCanister { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn missing_dependency_path_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./does-not-exist\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::DependencyNotFound { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn imported_canisters_appear_in_implicit_environments() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Deploy-all: the implicit `local` environment includes the dependency. + let local = p.environments.get("local").unwrap(); + assert!(local.canisters.contains_key("backend")); + assert!(local.canisters.contains_key("openemail:backend")); + } +} diff --git a/crates/icp-deploy-canister/src/sync_exec.rs b/crates/icp-deploy-canister/src/sync_exec.rs new file mode 100644 index 000000000..18b3b678d --- /dev/null +++ b/crates/icp-deploy-canister/src/sync_exec.rs @@ -0,0 +1,634 @@ +//! Injected sync-step execution. +//! +//! A canister's sync steps run either a WASI plugin (wasmtime) or a subprocess +//! script — neither can run inside a canister — so their execution is provided +//! by the host through [`PluginExecutor`] and [`ScriptRunner`]. This crate keeps +//! *all* of the derivation, though: it dispatches on the step kind, resolves the +//! plugin inputs, and assembles the `ICP_CLI_*` system environment variables +//! scripts run with. The host implementations only perform the irreducible host +//! action — fetch-and-run-the-wasm, or spawn-the-subprocess — against a +//! fully-resolved [`PluginInvocation`] / [`ScriptInvocation`]. +//! +//! The two executors are separate traits because an environment can support one +//! without the other. Script steps are host-only, and are rejected by +//! [`crate::project::verify_sandbox`] before they reach an executor; [`NoScripts`] +//! is the ready-made [`ScriptRunner`] for a host that has no subprocesses. + +use std::collections::BTreeMap; + +use async_trait::async_trait; +use candid::Principal; +use snafu::prelude::*; + +use crate::manifest::adapter::{ + plugin::{self, NamedPaths}, + prebuilt::SourceField, + script, +}; +use crate::prelude::*; + +/// Resolved context for executing one canister's sync steps. +#[derive(Clone, Debug)] +pub struct SyncStepContext { + /// Directory the canister was declared in (base for relative plugin paths). + pub canister_path: PathBuf, + /// The project (workspace root) directory. It bounds what a sync plugin may + /// read: a declared `dirs`/`files` entry may rise out of the canister + /// directory into the rest of the project, but not out of the project. + pub project_dir: PathBuf, + /// The canister being synced. + pub canister_id: Principal, + /// Store key of the canister being synced (e.g. `backend`, or + /// `services/open-crm:backend` for a canister in a subproject) — the `name` + /// of its [`Canister`](crate::Canister). Its namespace prefix + /// identifies which other canisters are in the same subproject. + pub canister_name: String, + /// Name of the environment being synced (e.g. "local", "production"). + pub environment: String, + /// Name of the network (e.g. "local", "ic"). + pub network: String, + /// IDs of all named canisters in the project for this environment. + pub canister_ids: BTreeMap, + /// Proxy canister to route calls through, if `--proxy` was passed. + pub proxy: Option, +} + +/// A manifest-declared path, tagged with the map key it was declared under. +/// A plain-list entry carries no key. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KeyedPath { + /// The `dirs:`/`files:` map key this path sits under, or `None` for a + /// plain-list entry. Non-unique: the paths of a key that maps to a list all + /// share it. + pub key: Option, + /// The path itself, relative to the canister directory. + pub path: String, +} + +/// Convert a manifest [`NamedPaths`] (or its absence) into the key-tagged path +/// list the executor receives. A missing setting yields an empty list. +fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { + paths + .into_iter() + .flat_map(NamedPaths::entries) + .map(|entry| KeyedPath { + key: entry.key.map(str::to_string), + path: entry.path.to_string(), + }) + .collect() +} + +/// A fully-resolved WASI-plugin sync step. Everything the host needs to fetch +/// and run the plugin has been computed by this crate; the host supplies only +/// the wasm source resolution (through its +/// [`RemoteResourceResolve`](crate::canister::recipe::RemoteResourceResolve)) +/// and the wasmtime runtime, plus its own identity/agent state. +#[derive(Clone, Debug)] +pub struct PluginInvocation { + /// Where the plugin wasm comes from (local path or remote URL). + pub source: SourceField, + /// Optional sha256 the host verifies the wasm against (required for remote). + pub sha256: Option, + /// Canister directory; base for the relative `dirs`/`files` and the source. + pub base_dir: PathBuf, + /// The project directory: the sandbox boundary. A declared path may rise out + /// of `base_dir` with `..` and reach anything inside the project, but + /// nothing above it. + pub project_dir: PathBuf, + /// Directories preopened read-only into the WASI sandbox. + pub dirs: Vec, + /// Files the host reads and passes inline to the plugin. + pub files: Vec, + /// Key-value fields passed inline to the plugin. + pub fields: BTreeMap, + /// The canister being synced, which the plugin may always call. + pub canister_id: Principal, + /// Environment name exposed to the plugin via its `SyncExecInput`. + pub environment: String, + /// The canister ID table exposed to the plugin: every named canister in + /// the project, plus a bare-local-name alias for each canister in the same + /// subproject as the one being synced. + pub canister_ids: BTreeMap, + /// The canisters the step's `canisters:` list named, resolved to ids. These + /// are callable in addition to [`canister_id`](Self::canister_id). + pub callable: BTreeMap, + /// Proxy canister to route the plugin's canister calls through, if any. + pub proxy: Option, +} + +/// A plugin step named a canister in its `canisters:` list that the environment +/// does not have. +#[derive(Debug, Snafu)] +#[snafu(display( + "sync plugin lists canister '{name}' as callable, but no canister by that name \ + is known in environment '{environment}'" +))] +pub struct UnknownCallableCanisterError { + name: String, + environment: String, +} + +impl PluginInvocation { + /// Resolve a plugin step's adapter against the sync context. Fails if the + /// step declares a callable canister the environment does not have. + pub fn new( + adapter: &plugin::Adapter, + ctx: &SyncStepContext, + ) -> Result { + let canister_ids = exposed_canister_ids(ctx); + let callable = resolve_callable(adapter, &canister_ids, &ctx.environment)?; + Ok(Self { + source: adapter.source.clone(), + sha256: adapter.sha256.clone(), + base_dir: ctx.canister_path.clone(), + project_dir: ctx.project_dir.clone(), + dirs: keyed_paths(adapter.dirs.as_ref()), + files: keyed_paths(adapter.files.as_ref()), + fields: adapter.fields.clone().unwrap_or_default(), + canister_id: ctx.canister_id, + environment: ctx.environment.clone(), + canister_ids, + callable, + proxy: ctx.proxy, + }) + } +} + +/// The canister ID table exposed to a sync plugin: every named canister in the +/// project, plus — for every canister in the subproject the synced canister +/// belongs to, or in a subproject nested below it — a duplicate entry under the +/// name that subproject itself uses. A store key is `:` for a +/// canister in a subproject and a bare local name for a canister defined +/// directly in the app root, so the syncing canister's namespace is the prefix +/// of its own key. +/// +/// The aliases exist so a subproject's manifest and plugins keep working when it +/// is vendored into a workspace: both the step's `canisters:` list and the name a +/// plugin passes back as a call target are written where the subproject's own +/// names apply, but store keys are relative to the app root, which moves. See +/// [`member_relative_alias`] for the names produced. +/// +/// The aliases take precedence over a canister elsewhere in the workspace whose +/// store key happens to be spelled the same way: a plugin resolving such a name +/// is naming what its own subproject calls it. +fn exposed_canister_ids(ctx: &SyncStepContext) -> BTreeMap { + // A canister in the app root is in the subproject the store keys are already + // relative to, so its names need no translation. + let Some((syncing_namespace, _)) = ctx.canister_name.rsplit_once(':') else { + return ctx.canister_ids.clone(); + }; + + let mut table = ctx.canister_ids.clone(); + for (key, id) in &ctx.canister_ids { + if let Some(alias) = member_relative_alias(syncing_namespace, key) { + table.insert(alias.to_owned(), *id); + } + } + table +} + +/// The name a canister has *within* the subproject at `namespace`: its store key +/// with that subproject's prefix removed. A canister of the subproject itself +/// comes back under its bare local name; one belonging to a subproject nested +/// below it comes back under the `:` key it would have if that +/// subproject were the app root. `None` for a canister the subproject has no name +/// of its own for. +/// +/// A local name never contains a colon but a subproject directory may, so a key +/// splits on its *last* colon. +fn member_relative_alias<'a>(namespace: &str, key: &'a str) -> Option<&'a str> { + let (key_namespace, _) = key.rsplit_once(':')?; + let rest = key.strip_prefix(namespace)?; + match key_namespace == namespace { + // The colon separating the subproject from a local name of its own. + true => rest.strip_prefix(':'), + // Otherwise the key's subproject must sit *below* this one. Demanding + // the path separator is what keeps `services/crm-legacy:backend` out of + // `services/crm`, which it merely shares a spelling prefix with. + false => rest.strip_prefix('/'), + } +} + +/// Resolve the step's `canisters:` list against `canister_ids`. A name that does +/// not resolve is a manifest error. +fn resolve_callable( + adapter: &plugin::Adapter, + canister_ids: &BTreeMap, + environment: &str, +) -> Result, UnknownCallableCanisterError> { + let mut by_name = BTreeMap::new(); + for name in adapter.canisters.iter().flatten() { + let principal = canister_ids + .get(name) + .copied() + .context(UnknownCallableCanisterSnafu { + name: name.clone(), + environment: environment.to_owned(), + })?; + by_name.insert(name.clone(), principal); + } + Ok(by_name) +} + +/// A fully-resolved script sync step. This crate has already assembled the +/// working directory and the complete environment the subprocess runs with +/// (see [`system_env_vars`]); the host only spawns the command(s). +#[derive(Clone, Debug)] +pub struct ScriptInvocation { + /// Shell command(s) to run in order. + pub commands: Vec, + /// Working directory (the canister directory). + pub cwd: PathBuf, + /// Environment variables the subprocess inherits, in insertion order. + pub env: Vec<(String, String)>, +} + +impl ScriptInvocation { + /// Resolve a script step's adapter against the sync context, assembling the + /// `ICP_CLI_*` system environment variables the command runs with. + pub fn new(adapter: &script::Adapter, ctx: &SyncStepContext) -> Self { + Self { + commands: adapter.command.as_vec(), + cwd: ctx.canister_path.clone(), + env: system_env_vars(ctx), + } + } +} + +/// The `ICP_CLI_*` system environment variables every script sync step runs +/// with: the environment and network names, the target canister id, and one +/// `ICP_CLI_CID_` per known canister in the environment (name uppercased, +/// non-alphanumerics replaced with `_`). +pub fn system_env_vars(ctx: &SyncStepContext) -> Vec<(String, String)> { + let mut envs = vec![ + ("ICP_CLI_ENVIRONMENT".to_owned(), ctx.environment.clone()), + ("ICP_CLI_NETWORK".to_owned(), ctx.network.clone()), + ("ICP_CLI_CID".to_owned(), ctx.canister_id.to_text()), + ]; + for (name, id) in &ctx.canister_ids { + let key = format!( + "ICP_CLI_CID_{}", + name.to_uppercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect::() + ); + envs.push((key, id.to_text())); + } + envs +} + +/// A sink for streamed sync-step output lines (a presentation concern the host +/// implements, e.g. over a progress bar). +pub trait StepProgress: Send + Sync { + fn line(&self, line: String); +} + +/// A plugin step failed. The concrete cause (a host wasm/runtime error) is boxed +/// because this crate does not depend on the executor's implementation; callers +/// can still walk `source()`. +#[derive(Debug, Snafu)] +#[snafu(display("plugin sync step failed"))] +pub struct PluginExecutorError { + pub source: Box, +} + +/// Host execution of WASI-plugin sync steps. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait PluginExecutor: Send + Sync { + /// Fetch and run a WASI plugin against a canister, returning any stderr + /// lines the plugin emitted that should be retained past the streamed view. + async fn run_plugin( + &self, + invocation: PluginInvocation, + progress: Option<&dyn StepProgress>, + ) -> Result, PluginExecutorError>; +} + +/// A script step failed. Boxed for the same reason as [`PluginExecutorError`]. +#[derive(Debug, Snafu)] +#[snafu(display("script sync step failed"))] +pub struct ScriptRunError { + pub source: Box, +} + +/// Host execution of subprocess script sync steps. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait ScriptRunner: Send + Sync { + /// Run a resolved script step, returning any stderr lines to retain past the + /// streamed view. + async fn run_script( + &self, + invocation: ScriptInvocation, + progress: Option<&dyn StepProgress>, + ) -> Result, ScriptRunError>; +} + +/// A [`ScriptRunner`] that always fails, for environments that don't support +/// subprocess script sync steps. +pub struct NoScripts; + +#[derive(Debug, Snafu)] +#[snafu(display("script sync steps are not supported in this environment"))] +pub struct NoScriptsError; + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +impl ScriptRunner for NoScripts { + async fn run_script( + &self, + _invocation: ScriptInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, ScriptRunError> { + Err(ScriptRunError { + source: Box::new(NoScriptsError), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::adapter::prebuilt::{LocalSource, SourceField}; + + fn principal(byte: u8) -> Principal { + Principal::from_slice(&[byte; 4]) + } + + fn ctx_named(name: &str, ids: &[(&str, Principal)]) -> SyncStepContext { + SyncStepContext { + canister_path: "/work".into(), + project_dir: "/work".into(), + canister_id: principal(0), + canister_name: name.to_owned(), + environment: "demo".to_owned(), + network: "ic".to_owned(), + canister_ids: ids.iter().map(|(n, p)| ((*n).to_owned(), *p)).collect(), + proxy: None, + } + } + + fn adapter_with(canisters: Option>) -> plugin::Adapter { + plugin::Adapter { + source: SourceField::Local(LocalSource { + path: "plugin.wasm".into(), + }), + sha256: None, + dirs: None, + files: None, + fields: None, + canisters, + } + } + + /// Canisters sharing the syncing canister's subproject are additionally + /// exposed under their bare local name; canisters in other subprojects are + /// not. + #[test] + fn exposed_ids_add_bare_names_for_same_subproject() { + let backend = principal(1); + let frontend = principal(2); + let foreign = principal(3); + let ctx = ctx_named( + "services/open-accounts:backend", + &[ + ("services/open-accounts:backend", backend), + ("services/open-accounts:frontend", frontend), + ("services/open-crm:backend", foreign), + ], + ); + + let table = exposed_canister_ids(&ctx); + + // Same-subproject canisters gain a bare-local duplicate... + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + // ...while the fully-qualified keys are still present for everyone. + assert_eq!( + table.get("services/open-accounts:frontend"), + Some(&frontend) + ); + assert_eq!(table.get("services/open-crm:backend"), Some(&foreign)); + } + + /// A canister of a subproject nested below the syncing canister's own is + /// exposed under the key that subproject has relative to it — the very key + /// its manifest and plugins use when it is built standalone, so vendoring it + /// into a workspace leaves both spellings working. + #[test] + fn exposed_ids_add_member_relative_names_for_nested_subprojects() { + let ledger = principal(1); + let deep = principal(2); + let ctx = ctx_named( + "services/crm:backend", + &[ + ("services/crm:backend", principal(9)), + ("services/crm/vendor/ledger:ledger", ledger), + ("services/crm/vendor/ledger/vendor/util:util", deep), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("vendor/ledger:ledger"), Some(&ledger)); + // Nesting is not limited to one level: the whole subtree below the + // syncing canister's subproject is renamed relative to it. + assert_eq!(table.get("vendor/ledger/vendor/util:util"), Some(&deep)); + // The workspace-absolute keys remain. + assert_eq!( + table.get("services/crm/vendor/ledger:ledger"), + Some(&ledger) + ); + } + + /// A subproject whose path merely starts with the same characters is not + /// nested below the syncing canister's, so it contributes no alias. + #[test] + fn exposed_ids_ignore_a_subproject_sharing_a_spelling_prefix() { + let legacy = principal(1); + let ctx = ctx_named( + "services/crm:backend", + &[ + ("services/crm:backend", principal(9)), + ("services/crm-legacy:backend", legacy), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("services/crm-legacy:backend"), Some(&legacy)); + // Its local name belongs to the syncing canister, not to it. + assert_eq!(table.get("backend"), Some(&principal(9))); + assert_eq!(table.get("-legacy:backend"), None); + } + + /// A member-relative alias wins over a workspace canister whose store key is + /// spelled the same way, for the same reason a bare sibling name does: the + /// name is being read where the subproject's own names apply. + #[test] + fn exposed_ids_member_relative_alias_overrides_a_root_dependency_key() { + let root_ledger = principal(1); + let own_ledger = principal(2); + let ctx = ctx_named( + "services/crm:backend", + &[ + ("services/crm:backend", principal(9)), + ("services/crm/vendor/ledger:ledger", own_ledger), + ("vendor/ledger:ledger", root_ledger), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("vendor/ledger:ledger"), Some(&own_ledger)); + // The root's own dependency is still reachable, by its store key. + assert_eq!( + table.get("services/crm/vendor/ledger:ledger"), + Some(&own_ledger) + ); + assert!(!table.values().any(|id| *id == root_ledger)); + } + + /// An app-root canister sharing a local name with a sibling of the syncing + /// canister does not keep the bare name: the syncing subproject's own + /// canister is what that name means to the plugin. + #[test] + fn exposed_ids_sibling_alias_overrides_the_app_root_name() { + let root_backend = principal(1); + let sibling_backend = principal(2); + let ctx = ctx_named( + "services/open-accounts:frontend", + &[ + ("backend", root_backend), + ("services/open-accounts:backend", sibling_backend), + ("services/open-accounts:frontend", principal(3)), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("backend"), Some(&sibling_backend)); + // The app-root canister's only key was that bare name, so it drops out + // of the table entirely rather than answering to a sibling's name. + assert!(!table.values().any(|id| *id == root_backend)); + } + + /// A subproject directory may itself contain a colon, so keys are split on + /// their last one — the same rule bundling uses. + #[test] + fn exposed_ids_split_subproject_prefix_at_the_last_colon() { + let backend = principal(1); + let frontend = principal(2); + let nested = principal(3); + let ctx = ctx_named( + "services/odd:name:backend", + &[ + ("services/odd:name:backend", backend), + ("services/odd:name:frontend", frontend), + ("services/odd:name/vendor/ledger:ledger", nested), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + assert_eq!(table.get("vendor/ledger:ledger"), Some(&nested)); + } + + /// The mirror image: a directory whose *name* contains a colon is not a + /// subproject nested below the part before it. `services/odd:name` holds one + /// directory named `odd:name`, so from `services/odd` it is nothing at all. + #[test] + fn exposed_ids_do_not_read_a_colon_in_a_directory_name_as_nesting() { + let odd = principal(1); + let ctx = ctx_named( + "services/odd:backend", + &[ + ("services/odd:backend", principal(9)), + ("services/odd:name:frontend", odd), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("services/odd:name:frontend"), Some(&odd)); + assert_eq!(table.get("name:frontend"), None); + } + + /// A single-project layout keys canisters by bare local name already, so no + /// duplicates are added. + #[test] + fn exposed_ids_unchanged_without_a_subproject() { + let backend = principal(1); + let ctx = ctx_named("backend", &[("backend", backend)]); + let table = exposed_canister_ids(&ctx); + assert_eq!(table.len(), 1); + assert_eq!(table.get("backend"), Some(&backend)); + } + + #[test] + fn resolve_callable_resolves_names() { + let dep = principal(1); + let sibling = principal(2); + let table = BTreeMap::from([ + ("backend".to_owned(), sibling), + ("services/open-crm:backend".to_owned(), dep), + ]); + let adapter = adapter_with(Some(vec![ + "backend".to_owned(), + "services/open-crm:backend".to_owned(), + ])); + + let callable = resolve_callable(&adapter, &table, "demo").unwrap(); + + assert_eq!(callable.get("backend"), Some(&sibling)); + assert_eq!(callable.get("services/open-crm:backend"), Some(&dep)); + } + + #[test] + fn resolve_callable_rejects_unknown_name() { + let adapter = adapter_with(Some(vec!["nope".to_owned()])); + resolve_callable(&adapter, &BTreeMap::new(), "demo") + .expect_err("an undeclared name must fail"); + } + + /// A script step resolves to the manifest's commands, the canister directory + /// as cwd, and the `ICP_CLI_*` environment assembled from the context. + #[test] + fn script_invocation_resolves_commands_cwd_and_env() { + use crate::manifest::adapter::script::{Adapter, CommandField}; + + let cid = principal(7); + let frontend = principal(8); + let ctx = SyncStepContext { + canister_path: "/work/backend".into(), + project_dir: "/work".into(), + canister_id: cid, + canister_name: "backend".to_owned(), + environment: "production".to_owned(), + network: "ic".to_owned(), + canister_ids: BTreeMap::from([("my-frontend".to_owned(), frontend)]), + proxy: None, + }; + let adapter = Adapter { + command: CommandField::Command("./deploy.sh".to_owned()), + }; + + let invocation = ScriptInvocation::new(&adapter, &ctx); + + assert_eq!(invocation.commands, vec!["./deploy.sh"]); + assert_eq!(invocation.cwd, PathBuf::from("/work/backend")); + assert_eq!( + invocation.env, + vec![ + ("ICP_CLI_ENVIRONMENT".to_owned(), "production".to_owned()), + ("ICP_CLI_NETWORK".to_owned(), "ic".to_owned()), + ("ICP_CLI_CID".to_owned(), cid.to_text()), + ("ICP_CLI_CID_MY_FRONTEND".to_owned(), frontend.to_text()), + ] + ); + } +} diff --git a/crates/icp-deploy-canister/src/testutil.rs b/crates/icp-deploy-canister/src/testutil.rs new file mode 100644 index 000000000..cdafc37bd --- /dev/null +++ b/crates/icp-deploy-canister/src/testutil.rs @@ -0,0 +1,63 @@ +//! Test-only helpers. + +use async_trait::async_trait; + +use crate::files::{FileAccess, FileAccessError}; +use crate::prelude::*; + +/// A [`FileAccess`] backed by the real host filesystem, for unit tests that +/// write manifests to a temp dir and consolidate them. +pub struct HostFiles; + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +impl FileAccess for HostFiles { + async fn read_file(&self, path: &Path) -> Result, FileAccessError> { + std::fs::read(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn read_to_string(&self, path: &Path) -> Result { + std::fs::read_to_string(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn exists(&self, path: &Path) -> bool { + path.exists() + } + + async fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + async fn is_dir(&self, path: &Path) -> bool { + path.is_dir() + } + + async fn read_dir(&self, path: &Path) -> Result, FileAccessError> { + let rd = std::fs::read_dir(path).map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + let mut out = Vec::new(); + for entry in rd { + let entry = entry.map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + if let Ok(p) = PathBuf::from_path_buf(entry.path()) { + out.push(p); + } + } + Ok(out) + } + + async fn canonicalize(&self, path: &Path) -> Option { + let c = std::fs::canonicalize(path).ok()?; + PathBuf::from_path_buf(c).ok() + } +} diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index f80f04c26..30d99d516 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -43,6 +43,13 @@ docs; the *reasons* behind those choices are recorded here. a plugin probing for an optional section, not a failure it must recognize by parsing error text. The host pays for that guarantee on the proxied path — see *Metadata reads* below. +- **`canister-set-environment-variable` sets one variable, not a list** — the + management canister replaces a canister's environment variables wholesale, so + *some* read-modify-write has to happen; putting it in the host means a plugin + that wants to add one variable does not have to first learn the target's other + ones, which reading them itself would tell it. The cost is a round trip per + variable, paid by a plugin setting several. It takes the same `call-target` + and `direct` flag as the other two imports, for the same one mental model. - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still @@ -188,10 +195,13 @@ exactly as `canister-call` chooses one: *proven* by the certificate rather than asserted. It requests `controllers` alongside the metadata path, since only that distinguishes a canister with no such section from one that was never created. -- **Proxied** — `ProxyArgs` aimed at the management canister's - `canister_metadata`, so the controller check runs against the proxy. This is - the same shape the CLI's own management calls take through - `update_or_proxy_raw`; the runtime inlines it rather than depending on the CLI. +- **Proxied** — `management_call` aimed at the management canister's + `canister_metadata`, so the controller check runs against the proxy. + +`management_call` is the shape the CLI's own management calls take through +`update_or_proxy_raw` — proxied via `ProxyArgs`, or direct with the target as +the effective canister ID — inlined here rather than depended on, and shared +with the environment-variable write below. Only a certificate can make a read `none`. The management canister answers a section that isn't there and one private to someone else with the same @@ -201,6 +211,27 @@ A plugin then sees one answer either way: no section by that name and no module installed at all are `none`; a private section it may not have, a canister that does not exist, and any other failure are errors. +### Environment-variable writes (read-modify-write) + +`update_settings` has no per-variable form: naming `environment_variables` at all +replaces the target's whole list, and omitting a setting is what leaves it +unchanged. So `canister-set-environment-variable` reads the target's current +settings with `canister_status`, overlays the one variable, and writes the list +back with every other field of `CanisterSettings` left `None`. + +Both calls go through `management_call` on the *same* route, chosen by `direct`. +They have to: each is controller-gated against whoever makes it, so a read as +the sync identity followed by a write as the proxy would demand both control the +target and buy nothing for it. The pair is not atomic — a settings update landing +between them is overwritten — which is inherent to the wholesale-replace API and +is documented in the WIT rather than papered over. + +The variable lives in the canister's settings, not in the manifest, so a later +`icp deploy` drops it: `set_binding_env_vars_many` rewrites the list from the +manifest's variables plus the `PUBLIC_CANISTER_ID:*` bindings without reading +what is there. A sync step that sets the variable on every sync restores it, +which is the ordinary case, since deploy runs the sync phase after that pass. + ### Interface versioning (parallel v0.1.0 / v0.2.0 support) A component built with wit-bindgen imports the interface it `use`s as a diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 6e5cba9a4..f9e69a964 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -26,7 +26,10 @@ use camino::Utf8PathBuf; use candid::{Encode, Principal}; use ic_agent::Agent; use ic_agent::hash_tree::{Label, LookupResult}; -use ic_management_canister_types::{CanisterMetadataArgs, CanisterMetadataResult}; +use ic_management_canister_types::{ + CanisterIdRecord, CanisterMetadataArgs, CanisterMetadataResult, CanisterSettings, + CanisterStatusResult, EnvironmentVariable, UpdateSettingsArgs, +}; use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; use semver::{Version, VersionReq}; use snafu::prelude::*; @@ -155,6 +158,51 @@ async fn certified_metadata_section( } } +/// Call a controller-gated management-canister method about `target`, either +/// signed by the sync identity or made by the proxy canister on its behalf. +/// +/// This is the shape the CLI's own management calls take through +/// `update_or_proxy_raw`; the runtime inlines it rather than depending on the +/// CLI. Which caller the target sees is the whole point of the choice: the +/// method checks *it* against the target's controllers, so a plugin reaches a +/// canister the proxy controls but the sync identity does not, or the other way +/// around. +async fn management_call( + agent: &Agent, + proxy: Option, + target: Principal, + method: &str, + arg: Vec, +) -> Result, String> { + let Some(proxy_cid) = proxy else { + return agent + .update(&Principal::management_canister(), method) + .with_arg(arg) + .with_effective_canister_id(target) + .await + .map_err(|e| format!("{method} call failed: {e}")); + }; + + let proxy_args = ProxyArgs { + canister_id: Principal::management_canister(), + method: method.to_string(), + args: arg, + cycles: candid::Nat::from(0u8), + }; + let encoded = Encode!(&proxy_args).map_err(|e| format!("proxy encode failed: {e}"))?; + let raw = agent + .update(&proxy_cid, "proxy") + .with_arg(encoded) + .await + .map_err(|e| format!("proxy call failed: {e}"))?; + let (result,): (ProxyResult,) = + candid::decode_args(&raw).map_err(|e| format!("proxy decode failed: {e}"))?; + match result { + ProxyResult::Ok(ok) => Ok(ok.result), + ProxyResult::Err(err) => Err(err.format_error()), + } +} + /// Whether the management canister rejected a metadata read by claiming the /// target has no such section, rather than because the read itself failed. /// @@ -327,28 +375,21 @@ impl HostState { name: name.clone(), }) .map_err(|e| format!("metadata encode failed: {e}"))?; - let proxy_args = ProxyArgs { - canister_id: Principal::management_canister(), - method: "canister_metadata".to_string(), - args: metadata_args, - cycles: candid::Nat::from(0u8), - }; - let encoded = Encode!(&proxy_args).map_err(|e| format!("proxy encode failed: {e}"))?; - let raw = agent - .update(&proxy_cid, "proxy") - .with_arg(encoded) - .await - .map_err(|e| format!("proxy call failed: {e}"))?; - let (result,): (ProxyResult,) = - candid::decode_args(&raw).map_err(|e| format!("proxy decode failed: {e}"))?; - match result { - ProxyResult::Ok(ok) => { - let (metadata,): (CanisterMetadataResult,) = candid::decode_args(&ok.result) + match management_call( + &agent, + Some(proxy_cid), + target, + "canister_metadata", + metadata_args, + ) + .await + { + Ok(raw) => { + let (metadata,): (CanisterMetadataResult,) = candid::decode_args(&raw) .map_err(|e| format!("metadata decode failed: {e}"))?; Ok(Some(metadata.value)) } - ProxyResult::Err(err) => { - let message = err.format_error(); + Err(message) => { if !rejected_as_no_such_section(&message, target, &name) { return Err(format!("metadata read failed: {message}")); } @@ -370,6 +411,68 @@ impl HostState { result } + /// Set one environment variable on an already-resolved target principal, + /// leaving its other variables and the rest of its settings alone. + /// + /// The management canister has no per-variable update — `update_settings` + /// replaces a canister's environment variables wholesale — so this is a + /// read-modify-write: read the target's current settings, overlay the one + /// variable, write the whole list back. It is not atomic, so a settings + /// update by another party landing between the two calls is overwritten. + /// + /// Both halves take the same route. `direct` picks who the target sees + /// asking, and both `canister_status` and `update_settings` check that + /// caller against its controllers: reading as one principal and writing as + /// another would need both to control the target and would still be the + /// same round trip, so there is nothing to gain by splitting them. + fn do_set_environment_variable( + &mut self, + target: Principal, + name: String, + value: String, + direct: bool, + ) -> Result<(), String> { + let agent = Arc::clone(&self.agent); + let proxy = if direct { None } else { self.proxy }; + + let start = Instant::now(); + let result = tokio::runtime::Handle::current().block_on(async move { + let status_args = Encode!(&CanisterIdRecord { + canister_id: target + }) + .map_err(|e| format!("canister_status encode failed: {e}"))?; + let raw = management_call(&agent, proxy, target, "canister_status", status_args) + .await + .map_err(|e| format!("reading the target's environment variables failed: {e}"))?; + let (status,): (CanisterStatusResult,) = candid::decode_args(&raw) + .map_err(|e| format!("canister_status decode failed: {e}"))?; + + let mut variables = status.settings.environment_variables; + match variables.iter_mut().find(|variable| variable.name == name) { + Some(existing) => existing.value = value, + None => variables.push(EnvironmentVariable { name, value }), + } + + let update_args = Encode!(&UpdateSettingsArgs { + canister_id: target, + settings: CanisterSettings { + environment_variables: Some(variables), + // Every other setting is left `None`, which the management + // canister reads as "leave it as it is". + ..CanisterSettings::default() + }, + sender_canister_version: None, + }) + .map_err(|e| format!("update_settings encode failed: {e}"))?; + management_call(&agent, proxy, target, "update_settings", update_args) + .await + .map_err(|e| format!("setting the environment variable failed: {e}"))?; + Ok(()) + }); + self.refund_host_call_time(start); + result + } + /// Return the wall-clock time a host call spent off-wasm to the compute /// budget, so network latency doesn't count against the plugin's limit. fn refund_host_call_time(&self, start: Instant) { @@ -407,6 +510,14 @@ impl v2::SyncPluginImports for HostState { let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; self.do_canister_metadata_section(target, req.name, req.direct) } + + fn canister_set_environment_variable( + &mut self, + req: v2::icp::sync_plugin::types::SetEnvironmentVariableRequest, + ) -> Result<(), String> { + let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; + self.do_set_environment_variable(target, req.name, req.value, req.direct) + } } // -- v0.1.0 interface: calls always go to the canister being synced. ----------- @@ -1489,6 +1600,25 @@ mod tests { ); } + /// Setting an environment variable names its target the same way a call + /// does, so an undeclared target is refused before the host reads any + /// settings — no live canister needed. + #[test] + fn setting_env_var_on_undeclared_canister_is_rejected() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let lines = + run_plugin(invocation(wasm_path, "set-env-undeclared")).expect("plugin should succeed"); + let [refusal] = &lines[..] else { + panic!("expected one refusal line, got: {lines:?}"); + }; + assert!( + refusal.contains("not permitted") && refusal.contains("undeclared"), + "got: {refusal}" + ); + } + /// The replica's own wording for the two ways a target reports it has no /// section, copied from `CanisterManagerError` in the IC repo. Both are /// absence, not failure, so both must reach the plugin as `none`. diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 4af24d363..cb1107d14 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -165,11 +165,31 @@ interface types { /// configured the read goes directly either way. direct: bool, } + + /// A request to set one of a canister's environment variables. + record set-environment-variable-request { + /// Which canister to set the variable on. The same rule as + /// `canister-call-request.target` applies: `host` is always permitted, + /// a `name` must appear in the sync step's `canisters` list. + target: call-target, + /// Name of the environment variable, spelled as the canister reads it. + name: string, + /// Value to set it to, replacing whatever value the target currently + /// has under this name. + value: string, + /// When true, the update is signed by the sync identity, which must + /// control the target for it to be accepted. When false (the default), + /// it is made by the proxy canister configured via `--proxy`, and it is + /// the proxy that must control the target — the same arrangement + /// proxied update calls rely on. With no proxy configured the update is + /// signed by the sync identity either way. + direct: bool, + } } /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, metadata-section-request, call-target, canister-id-entry, dir-input, file-input, field-input}; + use types.{sync-exec-input, canister-call-request, metadata-section-request, set-environment-variable-request, call-target, canister-id-entry, dir-input, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin @@ -198,6 +218,14 @@ world sync-plugin { /// read. The plugin is responsible for interpreting the bytes. import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; + /// Set an environment variable on a canister, leaving the target's other + /// environment variables — and the rest of its settings — as they are. + /// The `req.target` selects the canister under the same rule as + /// `canister-call`: the canister being synced (`host`), or one listed in + /// the sync step's `canisters` list, by name. + /// Returns an error message on failure. + import canister-set-environment-variable: func(req: set-environment-variable-request) -> result<_, string>; + // The plugin's stdout is captured and shown as transient progress in // the rolling step view of icp-cli; it is discarded when the step ends. // diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 116eb0bc8..b33bd64e8 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -76,6 +76,20 @@ impl Guest for TestPlugin { eprintln!("{err}"); Ok(()) } + // Ask to set an environment variable on a canister the step did not + // declare. Rejected on the same rule as a call or a metadata read, + // before any settings are read; echo the refusal. + "set-env-undeclared" => { + let err = canister_set_environment_variable(&SetEnvironmentVariableRequest { + target: CallTarget::Name("undeclared".to_string()), + name: "API_URL".to_string(), + value: "https://example.com".to_string(), + direct: true, + }) + .expect_err("host must reject an undeclared target"); + eprintln!("{err}"); + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/Cargo.toml b/crates/icp/Cargo.toml index 40f310b15..e0a3253e2 100644 --- a/crates/icp/Cargo.toml +++ b/crates/icp/Cargo.toml @@ -25,7 +25,6 @@ elliptic-curve = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } glob = { workspace = true } -handlebars = { workspace = true } hex = { workspace = true } hmac = { workspace = true } hybrid-array = { workspace = true } @@ -36,6 +35,7 @@ ic-ledger-types = { workspace = true } ic-management-canister-types = { workspace = true } ic-utils = { workspace = true } icp-canister-interfaces = { workspace = true } +icp-deploy-canister = { workspace = true } icp-sync-plugin = { workspace = true } icrc-ledger-types = { workspace = true } indexmap = { workspace = true } @@ -80,6 +80,11 @@ uuid = { workspace = true } wslpath2 = { workspace = true } zeroize = { workspace = true } +[features] +# Enables `clap::ValueEnum` derives on CLI-facing enums, including the manifest +# enums now defined in `icp-deploy-canister`. +clap = ["dep:clap", "icp-deploy-canister/clap"] + [target.'cfg(windows)'.dependencies] winreg = { workspace = true } diff --git a/crates/icp/src/canister/build/prebuilt.rs b/crates/icp/src/canister/build/prebuilt.rs index 774a102f9..c54164143 100644 --- a/crates/icp/src/canister/build/prebuilt.rs +++ b/crates/icp/src/canister/build/prebuilt.rs @@ -1,7 +1,12 @@ use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{canister::wasm, fs, manifest::adapter::prebuilt::Adapter, package::PackageCache}; +use crate::{ + canister::{ChannelProgress, wasm}, + fs, + manifest::adapter::prebuilt::Adapter, + package::PackageCache, +}; use super::Params; @@ -20,11 +25,12 @@ pub(super) async fn build( stdio: Option>, pkg_cache: &PackageCache, ) -> Result<(), PrebuiltError> { + let progress = ChannelProgress::wrap(stdio.as_ref()); let src = wasm::resolve( &adapter.source, ¶ms.path, adapter.sha256.as_deref(), - stdio.as_ref(), + ChannelProgress::as_dyn(progress.as_ref()), pkg_cache, ) .await?; diff --git a/crates/icp/src/canister/mod.rs b/crates/icp/src/canister/mod.rs index e5277333d..a1abf66a9 100644 --- a/crates/icp/src/canister/mod.rs +++ b/crates/icp/src/canister/mod.rs @@ -1,13 +1,17 @@ -use std::collections::HashMap; - -use candid::{Nat, Principal}; -use ic_management_canister_types::{CanisterSettings, LogVisibility}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::{ - parsers::{CyclesAmount, DurationAmount, MemoryAmount}, - prelude::*, +//! Host-side canister facade. +//! +//! The canister *model* (`Settings`, `ControllerRef`, `resolve_controllers`, +//! log-visibility types, and the `RemoteResourceResolve` interface) lives in +//! `icp_deploy_canister::canister` and is re-exported here. The build/sync/wasm +//! *executors* (which spawn processes, run wasmtime, and fetch over HTTP) stay +//! here. + +use icp_deploy_canister::sync_exec::StepProgress; +use tokio::sync::mpsc::Sender; + +pub use icp_deploy_canister::canister::{ + ControllerRef, LogVisibilityDef, LogVisibilitySimple, ManifestEnvVar, ManifestSettings, + Settings, resolve_controllers, }; pub mod build; @@ -17,673 +21,27 @@ pub mod sync; mod script; pub mod wasm; -/// Controls who can read canister logs. -/// Supports both string format ("controllers", "public") and object format ({ allowed_viewers: [...] }). -#[derive(Clone, Debug, PartialEq, Serialize)] -#[serde(untagged)] -pub enum LogVisibilityDef { - /// Simple string variants for controllers or public - Simple(LogVisibilitySimple), - /// Object format with allowed_viewers list - AllowedViewers { allowed_viewers: Vec }, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum LogVisibilitySimple { - Controllers, - Public, -} - -impl<'de> Deserialize<'de> for LogVisibilityDef { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::{Error, MapAccess, Visitor}; - use std::fmt; - - struct LogVisibilityVisitor; - - impl<'de> Visitor<'de> for LogVisibilityVisitor { - type Value = LogVisibilityDef; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("'controllers', 'public', or object with 'allowed_viewers'") - } - - fn visit_str(self, value: &str) -> Result { - LogVisibilitySimple::deserialize( - serde::de::value::StrDeserializer::::new(value), - ) - .map(LogVisibilityDef::Simple) - .map_err(|_| { - E::custom(format!( - "unknown log_visibility value: '{}', expected 'controllers' or 'public'", - value - )) - }) - } - - fn visit_map(self, mut map: M) -> Result - where - M: MapAccess<'de>, - { - let mut allowed_viewers: Option> = None; - - while let Some(key) = map.next_key::()? { - match key.as_str() { - "allowed_viewers" => { - if allowed_viewers.is_some() { - return Err(Error::duplicate_field("allowed_viewers")); - } - allowed_viewers = Some(map.next_value()?); - } - _ => { - return Err(Error::unknown_field(&key, &["allowed_viewers"])); - } - } - } - - allowed_viewers - .map(|v| LogVisibilityDef::AllowedViewers { allowed_viewers: v }) - .ok_or_else(|| Error::missing_field("allowed_viewers")) - } - } - - deserializer.deserialize_any(LogVisibilityVisitor) - } -} - -impl JsonSchema for LogVisibilityDef { - fn schema_name() -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed("LogVisibility") - } - - fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "description": "Controls who can read canister logs.", - "oneOf": [ - { - "type": "string", - "enum": ["controllers", "public"], - "description": "Simple log visibility: 'controllers' (only controllers can view) or 'public' (anyone can view)" - }, - { - "type": "object", - "properties": { - "allowed_viewers": { - "type": "array", - "items": { - "type": "string", - "description": "A principal ID that can view logs" - }, - "description": "List of principal IDs that can view canister logs" - } - }, - "required": ["allowed_viewers"], - "additionalProperties": false, - "description": "Specific principals that can view logs" - } - ] - }) - } -} - -impl From for LogVisibility { - fn from(value: LogVisibilityDef) -> Self { - match value { - LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) => { - LogVisibility::Controllers - } - LogVisibilityDef::Simple(LogVisibilitySimple::Public) => LogVisibility::Public, - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - LogVisibility::AllowedViewers(allowed_viewers) - } - } - } -} - -/// A reference to a controller: either an explicit principal or a canister name in this project. -/// -/// During deserialization, principal text format is tried first; strings that don't parse as a -/// principal are treated as canister names. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ControllerRef { - /// An explicitly specified principal (e.g. "2vxsx-fae") - Principal(candid::Principal), - /// A canister name from the same project (e.g. "my_canister") - CanisterName(String), -} - -impl ControllerRef { - /// Resolve to a `Principal` using the provided ID mapping. - /// Returns `None` if this is a `CanisterName` not present in `ids`. - pub fn resolve(&self, ids: &crate::store_id::IdMapping) -> Option { - match self { - ControllerRef::Principal(p) => Some(*p), - ControllerRef::CanisterName(name) => ids.get(name).copied(), - } - } - - /// If this is a `CanisterName`, returns the name; otherwise `None`. - pub fn canister_name(&self) -> Option<&str> { - match self { - ControllerRef::CanisterName(n) => Some(n), - ControllerRef::Principal(_) => None, - } - } -} - -/// Partition a slice of controller references into resolved principals and unresolved canister -/// names, using `ids` for name lookup. -pub fn resolve_controllers( - crefs: &[ControllerRef], - ids: &crate::store_id::IdMapping, -) -> (Vec, Vec) { - let mut resolved = Vec::new(); - let mut unresolved = Vec::new(); - for cref in crefs { - match cref.resolve(ids) { - Some(p) => resolved.push(p), - None => { - if let Some(name) = cref.canister_name() { - unresolved.push(name.to_owned()); - } - } - } - } - (resolved, unresolved) -} - -impl schemars::JsonSchema for ControllerRef { - fn schema_name() -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed("ControllerRef") - } - - fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "type": "string", - "description": "A controller: either a principal text (e.g. '2vxsx-fae') or a canister name in this project (e.g. 'my_canister')" - }) - } -} - -/// An environment variable value as written in a manifest. -/// -/// A plain scalar is the value itself: -/// ```yaml -/// environment_variables: -/// API_ENDPOINT: https://api.example.com -/// ``` -/// -/// The object form reads the value from a file, relative to the canister's own -/// directory — including when an environment overrides the variable, matching how -/// an `init_args` override resolves its path: -/// ```yaml -/// environment_variables: -/// API_KEY: -/// path: ./secrets/api-key -/// ``` -#[derive(Clone, Debug, PartialEq, Deserialize, JsonSchema, Serialize)] -#[serde(untagged, expecting = "a string, or `{ path: }`")] -pub enum ManifestEnvVar { - /// The value, written inline. - Value(String), - /// A file holding the value. Surrounding whitespace is trimmed off the - /// file's contents, so a trailing newline does not become part of the value. - Path { - #[schemars(with = "String")] - path: PathBuf, - }, -} - -impl Default for ManifestEnvVar { - fn default() -> Self { - Self::Value(String::new()) - } -} - -/// Canister settings loaded from a manifest, before file-backed environment -/// variable values have been read. See [`Settings`] for the resolved form. -pub type ManifestSettings = Settings; - -/// Canister settings, such as compute and memory allocation. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Settings { - /// Controls who can read canister logs. - #[serde(skip_serializing_if = "Option::is_none")] - pub log_visibility: Option, - - /// Compute allocation (0 to 100). Represents guaranteed compute capacity. - #[serde(skip_serializing_if = "Option::is_none")] - pub compute_allocation: Option, - - /// Memory allocation in bytes. If unset, memory is allocated dynamically. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_allocation: Option, - - /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. - /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). - #[serde(skip_serializing_if = "Option::is_none")] - pub freezing_threshold: Option, - - /// Upper limit on cycles reserved for future resource payments. - /// Memory allocations that would push the reserved balance above this limit will fail. - /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reserved_cycles_limit: Option, - - /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub wasm_memory_limit: Option, - - /// Wasm memory threshold in bytes. Triggers a callback when exceeded. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub wasm_memory_threshold: Option, - - /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. - /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. - #[serde(skip_serializing_if = "Option::is_none")] - pub log_memory_limit: Option, - - /// Environment variables for the canister as key-value pairs. - /// These variables are accessible within the canister and can be used to configure - /// behavior without hardcoding values in the WASM module. - /// A value may also be read from a file with `{ path: }`. - #[serde(skip_serializing_if = "Option::is_none")] - pub environment_variables: Option>, - - /// Controllers for this canister. Each entry is either a principal text - /// (e.g. "2vxsx-fae") or the name of another canister in this project. - /// Named canisters that do not yet exist will be set as controllers once created. - #[serde(default)] - pub controllers: Option>, -} - -impl From for ManifestSettings { - fn from(settings: Settings) -> Self { - let Settings { - log_visibility, - compute_allocation, - memory_allocation, - freezing_threshold, - reserved_cycles_limit, - wasm_memory_limit, - wasm_memory_threshold, - log_memory_limit, - environment_variables, - controllers, - } = settings; +/// Adapts a streamed-output channel to the library's [`StepProgress`] line sink, +/// so host code that already owns a channel can hand one to the library's IO +/// traits. +pub struct ChannelProgress(pub Sender); - Self { - log_visibility, - compute_allocation, - memory_allocation, - freezing_threshold, - reserved_cycles_limit, - wasm_memory_limit, - wasm_memory_threshold, - log_memory_limit, - environment_variables: environment_variables.map(|vars| { - vars.into_iter() - .map(|(name, value)| (name, ManifestEnvVar::Value(value))) - .collect() - }), - controllers, - } +impl StepProgress for ChannelProgress { + fn line(&self, line: String) { + // Status lines are advisory: drop them rather than block the caller if + // the display has fallen behind. + let _ = self.0.try_send(line); } } -impl From for CanisterSettings { - fn from(settings: Settings) -> Self { - CanisterSettings { - freezing_threshold: settings.freezing_threshold.map(|d| Nat::from(d.get())), - controllers: None, - reserved_cycles_limit: settings.reserved_cycles_limit.map(|c| Nat::from(c.get())), - log_visibility: settings.log_visibility.map(Into::into), - memory_allocation: settings.memory_allocation.map(|m| Nat::from(m.get())), - compute_allocation: settings.compute_allocation.map(Nat::from), - ..Default::default() - } +impl ChannelProgress { + /// Wrap an optional channel, as the library's IO traits take it. + pub fn wrap(stdio: Option<&Sender>) -> Option { + stdio.cloned().map(Self) } -} - -#[cfg(test)] -mod tests { - use indoc::indoc; - - use super::*; - - #[test] - fn log_visibility_deserialize_controllers() { - let yaml = "controllers"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) - ); - } - - #[test] - fn log_visibility_deserialize_public() { - let yaml = "public"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - LogVisibilityDef::Simple(LogVisibilitySimple::Public) - ); - } - - #[test] - fn log_visibility_deserialize_allowed_viewers() { - let yaml = r#" -allowed_viewers: - - "aaaaa-aa" - - "2vxsx-fae" -"#; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - match result { - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - assert_eq!(allowed_viewers.len(), 2); - assert_eq!( - allowed_viewers[0], - Principal::from_text("aaaaa-aa").unwrap() - ); - assert_eq!( - allowed_viewers[1], - Principal::from_text("2vxsx-fae").unwrap() - ); - } - _ => panic!("Expected AllowedViewers variant"), - } - } - - #[test] - fn log_visibility_deserialize_allowed_viewers_empty() { - let yaml = "allowed_viewers: []"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - match result { - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - assert!(allowed_viewers.is_empty()); - } - _ => panic!("Expected AllowedViewers variant"), - } - } - - #[test] - fn log_visibility_deserialize_invalid_string() { - let yaml = "invalid"; - let result: Result = serde_yaml::from_str(yaml); - assert!(result.is_err()); - let err = result.unwrap_err().to_string(); - assert!(err.contains("unknown log_visibility value")); - } - - #[test] - fn log_visibility_deserialize_invalid_field() { - let yaml = "unknown_field: []"; - let result: Result = serde_yaml::from_str(yaml); - assert!(result.is_err()); - let err = result.unwrap_err().to_string(); - assert!(err.contains("unknown field")); - } - - #[test] - fn log_visibility_serialize_controllers() { - let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert_eq!(yaml.trim(), "controllers"); - } - - #[test] - fn log_visibility_serialize_public() { - let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Public); - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert_eq!(yaml.trim(), "public"); - } - - #[test] - fn log_visibility_serialize_allowed_viewers() { - let log_vis = LogVisibilityDef::AllowedViewers { - allowed_viewers: vec![ - Principal::from_text("aaaaa-aa").unwrap(), - Principal::from_text("2vxsx-fae").unwrap(), - ], - }; - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert!(yaml.contains("allowed_viewers")); - assert!(yaml.contains("aaaaa-aa")); - assert!(yaml.contains("2vxsx-fae")); - } - - #[test] - fn settings_reserved_cycles_limit_parses_suffix() { - let yaml = "reserved_cycles_limit: 4.3t"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.reserved_cycles_limit.as_ref().map(|c| c.get()), - Some(4_300_000_000_000) - ); - } - - #[test] - fn settings_reserved_cycles_limit_parses_number() { - let yaml = "reserved_cycles_limit: 5000000000000"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.reserved_cycles_limit.as_ref().map(|c| c.get()), - Some(5_000_000_000_000) - ); - } - - #[test] - fn settings_memory_allocation_parses_suffix() { - let yaml = "memory_allocation: 4gib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.memory_allocation.as_ref().map(|m| m.get()), - Some(4 * 1024 * 1024 * 1024) - ); - } - - #[test] - fn settings_memory_allocation_parses_number() { - let yaml = "memory_allocation: 4294967296"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.memory_allocation.as_ref().map(|m| m.get()), - Some(4294967296) - ); - } - - #[test] - fn settings_wasm_memory_limit_parses_suffix() { - let yaml = "wasm_memory_limit: 1.5gib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.wasm_memory_limit.as_ref().map(|m| m.get()), - Some(1610612736) - ); - } - - #[test] - fn settings_log_memory_limit_parses_suffix() { - let yaml = "log_memory_limit: 256kib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.log_memory_limit.as_ref().map(|m| m.get()), - Some(256 * 1024) - ); - } - - #[test] - fn settings_log_memory_limit_parses_mib() { - let yaml = "log_memory_limit: 2mib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.log_memory_limit.as_ref().map(|m| m.get()), - Some(2 * 1024 * 1024) - ); - } - - #[test] - fn settings_environment_variables_take_values_or_files() { - let yaml = indoc! {r#" - environment_variables: - API_ENDPOINT: https://api.example.com - API_KEY: - path: ./secrets/api-key - "#}; - let settings: ManifestSettings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.environment_variables, - Some(HashMap::from([ - ( - "API_ENDPOINT".to_owned(), - ManifestEnvVar::Value("https://api.example.com".to_owned()), - ), - ( - "API_KEY".to_owned(), - ManifestEnvVar::Path { - path: "./secrets/api-key".into(), - }, - ), - ])), - ); - } - - #[test] - fn settings_environment_variable_rejects_unknown_object_form() { - let yaml = indoc! {r#" - environment_variables: - API_KEY: - file: ./secrets/api-key - "#}; - let err = serde_yaml::from_str::(yaml) - .expect_err("only the `path` object form is accepted"); - assert!( - err.to_string().contains("a string, or `{ path: }`"), - "unhelpful error: {err}" - ); - } - - /// A value of the wrong scalar type reports what is accepted, rather than - /// serde's default "did not match any variant" for an untagged enum. - #[test] - fn settings_environment_variable_rejects_non_string_scalar() { - let err = - serde_yaml::from_str::("environment_variables:\n PORT: 8080\n") - .expect_err("a bare integer is not a value"); - assert!( - err.to_string().contains("a string, or `{ path: }`"), - "unhelpful error: {err}" - ); - } - - #[test] - fn resolved_settings_serialize_environment_variables_inline() { - let settings = Settings { - environment_variables: Some(HashMap::from([( - "API_KEY".to_owned(), - "s3cret".to_owned(), - )])), - ..Default::default() - }; - let yaml = serde_yaml::to_string(&ManifestSettings::from(settings)).unwrap(); - assert!( - yaml.contains("environment_variables:\n API_KEY: s3cret\n"), - "unexpected yaml: {yaml}" - ); - } - - #[test] - fn controller_ref_deserializes_principal() { - let yaml = "\"2vxsx-fae\""; - let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - ControllerRef::Principal(Principal::from_text("2vxsx-fae").unwrap()) - ); - } - - #[test] - fn controller_ref_deserializes_canister_name() { - let yaml = "\"my_canister\""; - let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - ControllerRef::CanisterName("my_canister".to_owned()) - ); - } - - #[test] - fn controller_ref_resolve_principal() { - let p = Principal::from_text("aaaaa-aa").unwrap(); - let cref = ControllerRef::Principal(p); - let ids = crate::store_id::IdMapping::new(); - assert_eq!(cref.resolve(&ids), Some(p)); - } - - #[test] - fn controller_ref_resolve_canister_name_present() { - let p = Principal::from_text("aaaaa-aa").unwrap(); - let cref = ControllerRef::CanisterName("backend".to_owned()); - let mut ids = crate::store_id::IdMapping::new(); - ids.insert("backend".to_owned(), p); - assert_eq!(cref.resolve(&ids), Some(p)); - } - - #[test] - fn controller_ref_resolve_canister_name_absent() { - let cref = ControllerRef::CanisterName("backend".to_owned()); - let ids = crate::store_id::IdMapping::new(); - assert_eq!(cref.resolve(&ids), None); - } - - #[test] - fn settings_controllers_parses_mixed() { - let yaml = r#" -controllers: - - "aaaaa-aa" - - "my_other_canister" -"#; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - let controllers = settings.controllers.unwrap(); - assert_eq!(controllers.len(), 2); - assert_eq!( - controllers[0], - ControllerRef::Principal(Principal::from_text("aaaaa-aa").unwrap()) - ); - assert_eq!( - controllers[1], - ControllerRef::CanisterName("my_other_canister".to_owned()) - ); - } - - #[test] - fn log_visibility_conversion_to_ic_type() { - let controllers = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); - let ic_controllers: LogVisibility = controllers.into(); - assert!(matches!(ic_controllers, LogVisibility::Controllers)); - - let public = LogVisibilityDef::Simple(LogVisibilitySimple::Public); - let ic_public: LogVisibility = public.into(); - assert!(matches!(ic_public, LogVisibility::Public)); - let viewers = LogVisibilityDef::AllowedViewers { - allowed_viewers: vec![Principal::from_text("aaaaa-aa").unwrap()], - }; - let ic_viewers: LogVisibility = viewers.into(); - match ic_viewers { - LogVisibility::AllowedViewers(v) => { - assert_eq!(v.len(), 1); - } - _ => panic!("Expected AllowedViewers"), - } + /// Borrow as the trait object the library's IO traits take. + pub fn as_dyn(this: Option<&Self>) -> Option<&dyn StepProgress> { + this.map(|p| p as &dyn StepProgress) } } diff --git a/crates/icp/src/canister/recipe/mod.rs b/crates/icp/src/canister/recipe/mod.rs index edb92caf3..834d19c65 100644 --- a/crates/icp/src/canister/recipe/mod.rs +++ b/crates/icp/src/canister/recipe/mod.rs @@ -1,59 +1,17 @@ -//! Recipe resolution, split into two stages. +//! Host-side recipe facade. //! -//! [`fetch`] retrieves a recipe's Handlebars template — reading a local file, or -//! downloading a remote URL or registry recipe — and returns the raw template -//! text. [`render`] turns that text into concrete build/sync steps. The first -//! stage does I/O and nothing else; the second is a pure function. +//! The [`RemoteResourceResolve`] interface, recipe rendering, and the +//! context/error types live in [`icp_deploy_canister::canister::recipe`]; the +//! concrete resolver — which fetches templates and plugin wasms over HTTP and +//! caches them — stays here in [`resolver`]. //! -//! The [`Resolve`] seam therefore covers only the fetching half, so a caller that -//! already has a template (or must not touch the network) can render without -//! going through a resolver at all. -//! -//! Caching a download is a third step, because whether a template is worth -//! keeping is not known until it renders. A download that carried a `sha256` is -//! cached during the fetch — the checksum already proves the bytes are the ones -//! that were asked for. An *unpinned* download is held back as a -//! [`PendingCache`] and only committed by the caller once rendering succeeds, so -//! that one bad remote response cannot become sticky in the cache. The full -//! sequence is therefore fetch → render → [`Resolve::commit`]. - -use async_trait::async_trait; -use snafu::prelude::*; - -use crate::manifest::recipe::Recipe; - -pub mod fetch; -pub mod render; - -pub use fetch::{Fetched, PendingCache}; -pub use render::{RecipeContext, RenderRecipeError, render_recipe}; - -/// Retrieves the recipe templates a project references. -/// -/// Only *fetching* is behind this trait: rendering a fetched template into build -/// and sync steps is [`render_recipe`], which needs no I/O and so needs no seam. -#[async_trait] -pub trait Resolve: Sync + Send { - /// Fetch the Handlebars template for `recipe`, returning its raw source and - /// any cache write held back until the template is known to render. - async fn resolve(&self, recipe: &Recipe) -> Result; - - /// Write a held-back download to the cache, now that it has rendered. - /// - /// Defaults to doing nothing: only [`fetch::RecipeFetcher`] caches, and only - /// it can construct the [`PendingCache`] that reaches this method, so a - /// resolver that never defers a write never has one to commit. - async fn commit(&self, pending: PendingCache) -> Result<(), ResolveError> { - let _ = pending; - Ok(()) - } -} +//! Resolution is therefore staged: fetch the template, render it, then commit +//! whatever the resolver held back. See [`resolver::ResourceResolver`] for why +//! the cache write waits. -#[derive(Debug, Snafu)] -pub enum ResolveError { - #[snafu(display("failed to fetch recipe template"))] - Fetch { source: fetch::RecipeFetchError }, +pub use icp_deploy_canister::canister::recipe::{ + FetchedRecipe, NoResolve, RecipeContext, RemoteResourceResolve, RenderRecipeError, + ResolveError, render_recipe, +}; - #[snafu(display("failed to cache recipe template"))] - Commit { source: fetch::RecipeFetchError }, -} +pub mod resolver; diff --git a/crates/icp/src/canister/recipe/render.rs b/crates/icp/src/canister/recipe/render.rs deleted file mode 100644 index f59941c73..000000000 --- a/crates/icp/src/canister/recipe/render.rs +++ /dev/null @@ -1,264 +0,0 @@ -//! Stage two of recipe resolution: turn template text into build/sync steps. - -use std::collections::HashMap; - -use handlebars::{Context, Handlebars, Helper, HelperDef, HelperResult, Output, RenderContext}; -use serde::Deserialize; -use snafu::prelude::*; -use tracing::debug; - -use crate::manifest::{ - canister::{BuildSteps, SyncSteps}, - recipe::{Recipe, RecipeType}, -}; - -/// Describes the canister being built, for the render stage. -/// -/// Belongs to rendering alone: [`Resolve::resolve`](super::Resolve::resolve) no -/// longer takes it, since fetching a template does not depend on which canister -/// the template is for. Only [`render_recipe`] consumes it. -/// -/// Serializes to the shape injected into recipe templates under the `_` namespace: -/// -/// ```yaml -/// canister: -/// name: -/// ``` -pub struct RecipeContext { - pub canister_name: String, -} - -impl RecipeContext { - /// Builds the YAML value injected into recipe templates under the `_` namespace. - /// Constructing the mapping directly is infallible, unlike `serde` serialization. - pub fn to_yaml(&self) -> serde_yaml::Value { - use serde_yaml::{Mapping, Value}; - - let mut canister = Mapping::new(); - canister.insert("name".into(), Value::String(self.canister_name.clone())); - - let mut root = Mapping::new(); - root.insert("canister".into(), Value::Mapping(canister)); - - Value::Mapping(root) - } -} - -#[derive(Debug, Snafu)] -pub enum RenderRecipeError { - #[snafu(display("recipe template for '{recipe}' failed to render"))] - Render { - // Boxed to keep `Result<_, RenderRecipeError>` small; `RenderError` - // alone is well over a hundred bytes. - #[snafu(source(from(handlebars::RenderError, Box::new)))] - source: Box, - recipe: RecipeType, - }, - - #[snafu(display("recipe '{recipe}' did not render into a valid build/sync manifest"))] - Parse { - source: serde_yaml::Error, - recipe: RecipeType, - }, -} - -/// Render a recipe's Handlebars `template` into concrete build/sync steps. -/// -/// The template is rendered with the recipe's `configuration` plus the reserved -/// `_` namespace (the `_` key always overrides any user-supplied value), then the -/// resulting YAML is parsed. A recipe may only produce `build` and `sync`. -pub fn render_recipe( - template: &str, - recipe: &Recipe, - recipe_context: &RecipeContext, -) -> Result<(BuildSteps, SyncSteps), RenderRecipeError> { - let mut reg = Handlebars::new(); - // The output is YAML, not HTML, so disable HTML escaping. - reg.register_escape_fn(handlebars::no_escape); - reg.register_helper("replace", Box::new(ReplaceHelper)); - // Reject unset template variables. - reg.set_strict_mode(true); - - // User-provided configuration plus the injected `_.*` variables. The `_` key - // is reserved and always overrides any user-supplied value. - let mut render_context: HashMap = recipe.configuration.clone(); - render_context.insert("_".to_string(), recipe_context.to_yaml()); - - debug!("Rendering recipe template:\n------\n{template}\n------"); - - let out = reg - .render_template(template, &render_context) - .context(RenderSnafu { - recipe: recipe.recipe_type.clone(), - })?; - - // Logged rather than carried in `Parse` below: a recipe author debugging a - // malformed render needs the whole document, which is too much for an error - // message. - debug!("Rendered recipe template:\n------\n{out}\n------"); - - // Recipes can only render `build`/`sync`. - #[derive(Deserialize)] - struct BuildSyncHelper { - build: BuildSteps, - #[serde(default)] - sync: SyncSteps, - } - - let helper: BuildSyncHelper = serde_yaml::from_str(&out).context(ParseSnafu { - recipe: recipe.recipe_type.clone(), - })?; - Ok((helper.build, helper.sync)) -} - -/// Handlebars helper for string replacement operations. -/// Usage: `{{ replace "from" "to" value }}` -#[derive(Clone, Copy)] -struct ReplaceHelper; - -impl HelperDef for ReplaceHelper { - fn call<'reg: 'rc, 'rc>( - &self, - h: &Helper, - _: &'reg Handlebars<'reg>, - _: &Context, - _: &mut RenderContext<'reg, 'rc>, - out: &mut dyn Output, - ) -> HelperResult { - let (from, to) = ( - h.param(0).unwrap().render(), // from - h.param(1).unwrap().render(), // to - ); - - let v = h.param(2).unwrap().render(); - out.write(&v.replace(&from, &to))?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::manifest::canister::BuildStep; - - fn recipe(config: &[(&str, &str)]) -> Recipe { - Recipe { - recipe_type: RecipeType::File("recipe.hbs".to_owned()), - configuration: config - .iter() - .map(|(k, v)| ((*k).to_owned(), serde_yaml::Value::String((*v).to_owned()))) - .collect(), - sha256: None, - } - } - - fn ctx(name: &str) -> RecipeContext { - RecipeContext { - canister_name: name.to_owned(), - } - } - - /// The only build step's command, for a recipe that renders a single script step. - fn rendered_command(template: &str, recipe: &Recipe, context: &RecipeContext) -> String { - let (build, _sync) = render_recipe(template, recipe, context).unwrap(); - match &build.steps[0] { - BuildStep::Script(adapter) => adapter.command.as_vec()[0].clone(), - other => panic!("expected a script build step, got {other:?}"), - } - } - - /// Interpolated values are not HTML-escaped (the output is YAML): `=` and `&` - /// must survive. - #[test] - fn template_values_are_not_html_escaped() { - let template = indoc::indoc! {r#" - build: - steps: - - type: script - command: "{{ command }}" - "#}; - let r = recipe(&[("command", "SITE=https://example.com&foo=bar npm run build")]); - assert_eq!( - rendered_command(template, &r, &ctx("my-canister")), - "SITE=https://example.com&foo=bar npm run build" - ); - } - - /// The canister name is injected under the reserved `_` namespace. - #[test] - fn canister_name_is_injected() { - let template = indoc::indoc! {r#" - build: - steps: - - type: script - command: "build {{_.canister.name}}" - "#}; - assert_eq!( - rendered_command(template, &recipe(&[]), &ctx("my-canister")), - "build my-canister" - ); - } - - /// The `_` namespace works through the `replace` helper. - #[test] - fn canister_name_works_with_replace_helper() { - let template = indoc::indoc! {r#" - build: - steps: - - type: script - command: "cp {{ replace "-" "_" _.canister.name }}.wasm out.wasm" - "#}; - assert_eq!( - rendered_command(template, &recipe(&[]), &ctx("my-canister")), - "cp my_canister.wasm out.wasm" - ); - } - - /// User configuration cannot override the reserved `_` namespace. - #[test] - fn reserved_namespace_cannot_be_overridden_by_user_config() { - let template = indoc::indoc! {r#" - build: - steps: - - type: script - command: "build {{_.canister.name}}" - "#}; - let mut r = recipe(&[]); - r.configuration.insert( - "_".to_owned(), - serde_yaml::from_str("canister:\n name: user-override").unwrap(), - ); - assert_eq!( - rendered_command(template, &r, &ctx("real-name")), - "build real-name" - ); - } - - /// A template referencing an unset variable is a `Render` error, because - /// strict mode is on. - #[test] - fn unset_template_variable_is_a_render_error() { - let template = indoc::indoc! {r#" - build: - steps: - - type: script - command: "{{ never_set }}" - "#}; - assert!(matches!( - render_recipe(template, &recipe(&[]), &ctx("c")), - Err(RenderRecipeError::Render { .. }) - )); - } - - /// A template that renders to invalid build/sync YAML is a `Parse` error, - /// not a panic. - #[test] - fn invalid_rendered_yaml_is_a_parse_error() { - let template = "not: a valid build manifest\n"; - assert!(matches!( - render_recipe(template, &recipe(&[]), &ctx("c")), - Err(RenderRecipeError::Parse { .. }) - )); - } -} diff --git a/crates/icp/src/canister/recipe/fetch.rs b/crates/icp/src/canister/recipe/resolver.rs similarity index 69% rename from crates/icp/src/canister/recipe/fetch.rs rename to crates/icp/src/canister/recipe/resolver.rs index cb6ca5c2d..cb1601abb 100644 --- a/crates/icp/src/canister/recipe/fetch.rs +++ b/crates/icp/src/canister/recipe/resolver.rs @@ -1,8 +1,7 @@ -//! Stage one of recipe resolution: get the template text. - use std::{str::FromStr, string::FromUtf8Error}; use async_trait::async_trait; +use icp_deploy_canister::sync_exec::StepProgress; use reqwest::{Method, Request, Url}; use sha2::{Digest, Sha256}; use snafu::prelude::*; @@ -19,53 +18,27 @@ use crate::{ prelude::*, }; -use super::{CommitSnafu, FetchSnafu, Resolve, ResolveError}; +use super::{FetchedRecipe, RemoteResourceResolve, ResolveError}; +use crate::manifest::adapter::prebuilt::SourceField; -/// Fetches recipe templates over HTTP, caching downloads in the package cache. -/// Template *rendering* is a separate stage -/// ([`render_recipe`](super::render_recipe)); this only produces the raw template -/// text. -pub struct RecipeFetcher { +/// Fetches recipe templates and plugin wasms over HTTP, caching downloads in the +/// package cache. Template *rendering* is the library's job +/// ([`icp_deploy_canister::canister::recipe::render_recipe`]); this only produces +/// the raw template text. +/// +/// Whether a download is worth keeping is not known until it renders, so an +/// *unpinned* download is marked [`FetchedRecipe::deferred`] and only written to +/// the cache when the caller reports back through +/// [`commit_recipe`](RemoteResourceResolve::commit_recipe). A checksummed +/// download is cached during the fetch — the checksum already proves the bytes +/// are the ones that were asked for. +pub struct ResourceResolver { /// Http client for fetching remote recipe templates pub http_client: reqwest::Client, /// Package cache for caching downloaded recipe templates pub pkg_cache: PackageCache, } -/// The result of the fetch stage. -pub struct Fetched { - /// Raw Handlebars template source. - pub template: String, - - /// A cache write deliberately held back until the template is known to - /// render; `None` when there is nothing to cache (a local file or a cache - /// hit) or when the download was already cached because it was checksummed. - /// - /// Pass to [`Resolve::commit`] after [`render_recipe`](super::render_recipe) - /// succeeds. - pub pending_cache: Option, -} - -/// A cache write for an unpinned download, held until the template renders. -/// -/// A checksummed download is cached the moment its checksum verifies: the -/// checksum is what establishes the bytes are the ones that were asked for, and -/// refetching would only produce the same bytes again. An unpinned download has -/// no such guarantee — caching it before it is known good would let a single bad -/// response become sticky, and every later resolution would read those bytes -/// back instead of refetching. -pub struct PendingCache { - target: CacheTarget, - hash: [u8; 32], - template: String, -} - -/// Where a fetched template belongs in the package cache. -enum CacheTarget { - Uri(String), - Registry { package: String, version: String }, -} - enum TemplateSource { LocalPath(PathBuf), RemoteUrl(String), @@ -110,27 +83,18 @@ pub enum RecipeFetchError { LockCache { source: crate::fs::lock::LockError }, } -impl RecipeFetcher { +impl ResourceResolver { /// Fetch a recipe's Handlebars template text: read a local file, or fetch a - /// remote URL or registry recipe. Verifies `sha256` when set. + /// remote URL or registry recipe (serving it from the package cache when + /// possible). Verifies `sha256` when set. /// - /// A checksummed download is cached here. An unpinned one is returned as a - /// [`PendingCache`] for the caller to commit once it renders — see - /// [`PendingCache`] for why. - async fn fetch_recipe(&self, recipe: &Recipe) -> Result { - // Determine the template source - let tmpl_source = match &recipe.recipe_type { - RecipeType::File(path) => TemplateSource::LocalPath(Path::new(&path).into()), - RecipeType::Url(url) => TemplateSource::RemoteUrl(url.to_owned()), - RecipeType::Registry { - name, - recipe, - version, - } => TemplateSource::Registry(name.to_owned(), recipe.to_owned(), version.to_owned()), - }; - - // Retrieve the template, using cache for remote/registry sources - let (tmpl, should_cache) = match &tmpl_source { + /// A checksummed download is cached here. An unpinned one is returned with + /// [`FetchedRecipe::deferred`] set, for the caller to commit once it renders. + async fn fetch_recipe(&self, recipe: &Recipe) -> Result { + // Retrieve the template, using cache for remote/registry sources. The + // flag says whether the bytes were freshly downloaded, and so are the + // only ones that could still need caching. + let (tmpl, downloaded) = match &template_source(&recipe.recipe_type) { TemplateSource::LocalPath(path) => { let bytes = read(path).context(ReadFileSnafu)?; (parse_bytes_to_string(bytes)?, false) @@ -188,69 +152,46 @@ impl RecipeFetcher { } }; - let hash = if let Some(sha256) = &recipe.sha256 { - verify_checksum(tmpl.as_bytes(), sha256)? - } else { - Sha256::digest(tmpl.as_bytes()).into() - }; - - // Nothing was downloaded (local file, or a cache hit): nothing to cache. - if !should_cache { - return Ok(Fetched { + if let Some(sha256) = &recipe.sha256 { + verify_checksum(tmpl.as_bytes(), sha256)?; + // The checksum matched, so refetching could only produce these same + // bytes: there is nothing to gain by waiting for a render that may + // never succeed. + if downloaded { + self.cache_recipe(recipe, &tmpl).await?; + } + return Ok(FetchedRecipe { template: tmpl, - pending_cache: None, + deferred: false, }); } - let target = match tmpl_source { - TemplateSource::LocalPath(_) => unreachable!("local files are never cached"), - TemplateSource::RemoteUrl(u) => CacheTarget::Uri(u), - TemplateSource::Registry(registry, recipe_name, version) => CacheTarget::Registry { - package: format!("@{registry}/{recipe_name}"), - version, - }, - }; - - let pending = PendingCache { - target, - hash, + Ok(FetchedRecipe { template: tmpl, - }; - - // A checksummed download is trustworthy the moment the checksum matches, - // so cache it now. An unpinned one waits for a successful render. - if recipe.sha256.is_some() { - self.write_cache(&pending).await?; - return Ok(Fetched { - template: pending.template, - pending_cache: None, - }); - } - - Ok(Fetched { - template: pending.template.clone(), - pending_cache: Some(pending), + deferred: downloaded, }) } - /// Write a fetched template into the package cache. - async fn write_cache(&self, pending: &PendingCache) -> Result<(), RecipeFetchError> { - let hash = hex::encode(pending.hash); - let bytes = pending.template.as_bytes(); - match &pending.target { - CacheTarget::Uri(u) => { + /// Cache a template downloaded by [`Self::fetch_recipe`]. For an unpinned + /// download this runs only after the caller has rendered it, so a malformed + /// response never becomes the entry that later project loads reuse. + async fn cache_recipe(&self, recipe: &Recipe, tmpl: &str) -> Result<(), RecipeFetchError> { + let hash = hex::encode(Sha256::digest(tmpl.as_bytes())); + match template_source(&recipe.recipe_type) { + TemplateSource::LocalPath(_) => unreachable!("local files are never cached"), + TemplateSource::RemoteUrl(u) => { self.pkg_cache .with_write(async |w| { - cache_uri_recipe(w, u, &hash, bytes).context(CacheRecipeSnafu)?; - Ok(()) + cache_uri_recipe(w, &u, &hash, tmpl.as_bytes()).context(CacheRecipeSnafu) }) .await .context(LockCacheSnafu)??; } - CacheTarget::Registry { package, version } => { + TemplateSource::Registry(registry, recipe_name, version) => { + let package = format!("@{registry}/{recipe_name}"); self.pkg_cache .with_write(async |w| { - cache_registry_recipe(w, package, version, &hash, bytes) + cache_registry_recipe(w, &package, &version, &hash, tmpl.as_bytes()) .context(CacheRecipeSnafu) }) .await @@ -284,24 +225,61 @@ impl RecipeFetcher { } #[async_trait] -impl Resolve for RecipeFetcher { - async fn resolve(&self, recipe: &Recipe) -> Result { - self.fetch_recipe(recipe).await.context(FetchSnafu) +impl RemoteResourceResolve for ResourceResolver { + async fn resolve_recipe(&self, recipe: &Recipe) -> Result { + self.fetch_recipe(recipe) + .await + .map_err(|source| ResolveError::Resolve { + source: Box::new(source), + }) + } + + async fn commit_recipe( + &self, + recipe: &Recipe, + fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + if !fetched.deferred { + return Ok(()); + } + self.cache_recipe(recipe, &fetched.template) + .await + .map_err(|source| ResolveError::Resolve { + source: Box::new(source), + }) + } + + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + progress: Option<&dyn StepProgress>, + ) -> Result { + crate::canister::wasm::resolve(source, base_dir, sha256, progress, &self.pkg_cache) + .await + .map_err(|source| ResolveError::ResolveWasm { + source: Box::new(source), + }) } +} - async fn commit(&self, pending: PendingCache) -> Result<(), ResolveError> { - self.write_cache(&pending).await.context(CommitSnafu) +/// Classify where a recipe's template comes from. +fn template_source(recipe_type: &RecipeType) -> TemplateSource { + match recipe_type { + RecipeType::File(path) => TemplateSource::LocalPath(Path::new(&path).into()), + RecipeType::Url(url) => TemplateSource::RemoteUrl(url.to_owned()), + RecipeType::Registry { + name, + recipe, + version, + } => TemplateSource::Registry(name.to_owned(), recipe.to_owned(), version.to_owned()), } } /// Helper function to verify sha256 checksum of recipe template bytes -fn verify_checksum(bytes: &[u8], expected: &str) -> Result<[u8; 32], RecipeFetchError> { - let actual_hash = { - let mut h = Sha256::new(); - h.update(bytes); - h.finalize() - }; - let actual = hex::encode(actual_hash); +fn verify_checksum(bytes: &[u8], expected: &str) -> Result<(), RecipeFetchError> { + let actual = hex::encode(Sha256::digest(bytes)); if actual != expected { return ChecksumMismatchSnafu { expected: expected.to_string(), @@ -309,7 +287,7 @@ fn verify_checksum(bytes: &[u8], expected: &str) -> Result<[u8; 32], RecipeFetch } .fail(); } - Ok(actual_hash.into()) + Ok(()) } /// Helper function to parse bytes into a UTF-8 string @@ -320,10 +298,11 @@ fn parse_bytes_to_string(bytes: Vec) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::canister::recipe::{RecipeContext, render_recipe}; use crate::manifest::recipe::{Recipe, RecipeType}; - fn fetcher(cache_dir: &Path) -> RecipeFetcher { - RecipeFetcher { + fn resolver(cache_dir: &Path) -> ResourceResolver { + ResourceResolver { http_client: reqwest::Client::new(), pkg_cache: PackageCache::new(cache_dir.to_owned()).unwrap(), } @@ -349,15 +328,12 @@ mod tests { sha256: None, }; - let fetched = fetcher(&tmp.path().join("pkg")) + let fetched = resolver(&tmp.path().join("pkg")) .fetch_recipe(&recipe) .await .unwrap(); assert_eq!(fetched.template, body); - assert!( - fetched.pending_cache.is_none(), - "local files are never cached" - ); + assert!(!fetched.deferred, "a local file has nothing to cache"); } /// A sha256 that does not match the template contents is rejected. @@ -374,7 +350,9 @@ mod tests { }; assert!(matches!( - fetcher(&tmp.path().join("pkg")).fetch_recipe(&recipe).await, + resolver(&tmp.path().join("pkg")) + .fetch_recipe(&recipe) + .await, Err(RecipeFetchError::ChecksumMismatch { .. }) )); } @@ -431,7 +409,7 @@ mod tests { let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); let cache_dir = tmp.path().join("pkg"); - let f = fetcher(&cache_dir); + let r = resolver(&cache_dir); let recipe = Recipe { recipe_type: RecipeType::Url(url), configuration: Default::default(), @@ -439,18 +417,18 @@ mod tests { }; // Fetch succeeds and hands back a held-back cache write. - let fetched = f.fetch_recipe(&recipe).await.expect("first fetch"); + let fetched = r.fetch_recipe(&recipe).await.expect("first fetch"); assert!( - fetched.pending_cache.is_some(), + fetched.deferred, "an unpinned download must defer its cache write" ); // Rendering fails, so the caller never commits. - let ctx = super::super::RecipeContext { + let ctx = RecipeContext { canister_name: "c".to_owned(), }; assert!( - super::super::render_recipe(&fetched.template, &recipe, &ctx).is_err(), + render_recipe(&fetched.template, &recipe, &ctx).is_err(), "fixture template must fail to render" ); @@ -463,7 +441,7 @@ mod tests { // bad bytes from cache. assert!( matches!( - f.fetch_recipe(&recipe).await, + r.fetch_recipe(&recipe).await, Err(RecipeFetchError::HttpStatus { status: 500, .. }) ), "second resolution must refetch, not read the uncommitted template back" @@ -484,26 +462,23 @@ mod tests { let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); let cache_dir = tmp.path().join("pkg"); - let f = fetcher(&cache_dir); + let r = resolver(&cache_dir); let recipe = Recipe { recipe_type: RecipeType::Url(url), configuration: Default::default(), sha256: None, }; - let fetched = f.fetch_recipe(&recipe).await.expect("first fetch"); - let pending = fetched.pending_cache.expect("unpinned defers its write"); - f.write_cache(&pending).await.expect("commit"); + let fetched = r.fetch_recipe(&recipe).await.expect("first fetch"); + assert!(fetched.deferred, "unpinned defers its write"); + r.commit_recipe(&recipe, &fetched).await.expect("commit"); assert!(cache_has_template(&cache_dir)); // Served from cache now, even though the server would answer 500. - let again = f.fetch_recipe(&recipe).await.expect("second fetch"); + let again = r.fetch_recipe(&recipe).await.expect("second fetch"); assert_eq!(again.template, fetched.template); - assert!( - again.pending_cache.is_none(), - "a cache hit has nothing to commit" - ); + assert!(!again.deferred, "a cache hit has nothing to commit"); } /// A checksummed download is cached during the fetch: the checksum already @@ -516,16 +491,16 @@ mod tests { let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); let cache_dir = tmp.path().join("pkg"); - let f = fetcher(&cache_dir); + let r = resolver(&cache_dir); let recipe = Recipe { recipe_type: RecipeType::Url(url), configuration: Default::default(), sha256: Some(hex::encode(Sha256::digest(UNRENDERABLE.as_bytes()))), }; - let fetched = f.fetch_recipe(&recipe).await.expect("first fetch"); + let fetched = r.fetch_recipe(&recipe).await.expect("first fetch"); assert!( - fetched.pending_cache.is_none(), + !fetched.deferred, "a checksummed download is cached during the fetch" ); assert!(cache_has_template(&cache_dir)); diff --git a/crates/icp/src/canister/script.rs b/crates/icp/src/canister/script.rs index 6974a745d..665259599 100644 --- a/crates/icp/src/canister/script.rs +++ b/crates/icp/src/canister/script.rs @@ -65,8 +65,8 @@ pub(super) async fn execute( /// Takes already-resolved commands rather than an [`Adapter`], so the subprocess /// executor needs to know nothing about manifest types. The sync path resolves /// its commands and `ICP_CLI_*` environment into a -/// [`ScriptInvocation`](super::sync::script::ScriptInvocation) before calling -/// this; the build path calls [`execute`] with its adapter. +/// [`ScriptInvocation`](icp_deploy_canister::sync_exec::ScriptInvocation) before +/// calling this; the build path calls [`execute`] with its adapter. pub(super) async fn execute_commands( cmds: &[String], cwd: &Path, diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index 996ec99f2..199d10d71 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -1,109 +1,75 @@ -use std::collections::BTreeMap; -use std::sync::Arc; - use async_trait::async_trait; -use candid::Principal; use ic_agent::Agent; +use icp_deploy_canister::canister::recipe::RemoteResourceResolve; +use icp_deploy_canister::sync_exec::{PluginInvocation, ScriptInvocation}; use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::manifest::canister::SyncStep; -use crate::package::PackageCache; -use crate::prelude::*; - mod plugin; -pub mod script; - -use script::{HostScripts, ScriptInvocation, ScriptRunError, ScriptRunner}; - -pub struct Params { - pub path: PathBuf, - /// The project (workspace root) directory. It bounds what a sync plugin may - /// read: a declared `dirs`/`files` entry may rise out of the canister - /// directory into the rest of the project, but not out of the project. - pub project_dir: PathBuf, - pub cid: Principal, - /// Fully-qualified store key of the canister being synced (e.g. `backend`, - /// or `services/open-crm:backend` for a canister in a subproject). Its namespace - /// prefix identifies which other canisters are in the same subproject. - pub name: String, - /// Name of the environment being synced (e.g. "local", "production"). - /// Passed to sync plugin steps via `SyncExecInput`. - pub environment: String, - /// Name of the network (e.g. "local", "ic"). - pub network: String, - /// IDs of all named canisters in the project for this environment. - pub canister_ids: BTreeMap, - /// Proxy canister to route calls through, if `--proxy` was passed. - pub proxy: Option, -} #[derive(Debug, Snafu)] pub enum SynchronizeError { #[snafu(transparent)] - Script { source: ScriptRunError }, + Script { source: super::script::ScriptError }, #[snafu(transparent)] Plugin { source: plugin::PluginError }, } +/// Host execution of the two sync-step mechanisms that can't run inside a +/// canister: WASI plugins (wasmtime) and subprocess scripts. +/// +/// Step dispatch and *all* input derivation (plugin dirs/files, the `ICP_CLI_*` +/// script environment) live in `icp-deploy-canister`; implementations here +/// receive a fully-resolved [`PluginInvocation`] / [`ScriptInvocation`] and +/// perform only the irreducible host action. This trait is the injection seam +/// the [`crate::context::Context`] carries so tests can stub it out. #[async_trait] pub trait Synchronize: Sync + Send { - async fn sync( + async fn run_plugin( &self, - step: &SyncStep, - params: &Params, + invocation: &PluginInvocation, agent: &Agent, stdio: Option>, - pkg_cache: &PackageCache, + resolver: &dyn RemoteResourceResolve, ) -> Result, SynchronizeError>; -} -/// Dispatches each sync step to the machinery that runs it. Plugin steps run in -/// the wasmtime WASI sandbox, which this drives directly; script steps go through -/// an injected [`ScriptRunner`], since spawning a subprocess is not available -/// everywhere. -pub struct Syncer { - scripts: Arc, + async fn run_script( + &self, + invocation: &ScriptInvocation, + stdio: Option>, + ) -> Result, SynchronizeError>; } -impl Syncer { - /// A syncer that runs script steps as host subprocesses. - pub fn host() -> Self { - Self::new(Arc::new(HostScripts)) - } - - pub fn new(scripts: Arc) -> Self { - Self { scripts } - } -} +pub struct Syncer; #[async_trait] impl Synchronize for Syncer { - async fn sync( + async fn run_plugin( &self, - step: &SyncStep, - params: &Params, + invocation: &PluginInvocation, agent: &Agent, stdio: Option>, - pkg_cache: &PackageCache, + resolver: &dyn RemoteResourceResolve, + ) -> Result, SynchronizeError> { + Ok(plugin::run(invocation, agent, stdio, resolver).await?) + } + + async fn run_script( + &self, + invocation: &ScriptInvocation, + stdio: Option>, ) -> Result, SynchronizeError> { - match step { - SyncStep::Script(adapter) => Ok(self - .scripts - .run_script(ScriptInvocation::new(adapter, params), stdio) - .await?), - SyncStep::Plugin(adapter) => Ok(plugin::sync( - adapter, - params, - agent, - ¶ms.environment, - params.proxy, - stdio, - pkg_cache, - ) - .await?), - } + let env_refs: Vec<(&str, &str)> = invocation + .env + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + super::script::execute_commands(&invocation.commands, &invocation.cwd, &env_refs, stdio) + .await?; + // Persistent stderr is a sync-plugin feature only; script steps don't + // currently retain any output past the rolling step view. + Ok(vec![]) } } @@ -115,102 +81,21 @@ pub struct UnimplementedMockSyncer; #[cfg(test)] #[async_trait] impl Synchronize for UnimplementedMockSyncer { - async fn sync( + async fn run_plugin( &self, - _step: &SyncStep, - _params: &Params, + _invocation: &PluginInvocation, _agent: &Agent, _stdio: Option>, - _pkg_cache: &PackageCache, + _resolver: &dyn RemoteResourceResolve, ) -> Result, SynchronizeError> { - unimplemented!("UnimplementedMockSyncer::sync") - } -} - -#[cfg(test)] -mod tests { - use std::sync::Mutex; - - use crate::manifest::adapter::script::{Adapter, CommandField}; - - use super::*; - - /// A [`ScriptRunner`] that records what it was asked to run instead of - /// running it, so step dispatch can be tested without spawning a shell. - #[derive(Default)] - struct RecordingScripts { - seen: Mutex>, - } - - #[async_trait] - impl ScriptRunner for RecordingScripts { - async fn run_script( - &self, - invocation: ScriptInvocation, - _stdio: Option>, - ) -> Result, ScriptRunError> { - self.seen.lock().unwrap().push(invocation); - Ok(vec![]) - } + unimplemented!("UnimplementedMockSyncer::run_plugin") } - fn dummy_agent() -> Agent { - Agent::builder() - .with_url("http://127.0.0.1:4943") - .build() - .expect("build test agent") - } - - /// A script step reaches the injected runner fully resolved: the commands - /// from the manifest, the canister directory as cwd, and the `ICP_CLI_*` - /// environment assembled from the sync params. Nothing is spawned. - #[tokio::test] - async fn script_steps_are_dispatched_to_the_injected_runner() { - let scripts = Arc::new(RecordingScripts::default()); - let syncer = Syncer::new(scripts.clone()); - - let cid = Principal::from_slice(&[7; 4]); - let params = Params { - path: "/work/backend".into(), - project_dir: "/work".into(), - cid, - name: "backend".to_owned(), - environment: "production".to_owned(), - network: "ic".to_owned(), - canister_ids: BTreeMap::from([( - "my-frontend".to_owned(), - Principal::from_slice(&[8; 4]), - )]), - proxy: None, - }; - let step = SyncStep::Script(Adapter { - command: CommandField::Command("./deploy.sh".to_owned()), - }); - - let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); - let pkg_cache = PackageCache::new(tmp.path().to_owned()).unwrap(); - - let retained = syncer - .sync(&step, ¶ms, &dummy_agent(), None, &pkg_cache) - .await - .expect("script step should dispatch"); - assert!(retained.is_empty()); - - let seen = scripts.seen.lock().unwrap(); - assert_eq!(seen.len(), 1); - assert_eq!(seen[0].commands, vec!["./deploy.sh"]); - assert_eq!(seen[0].cwd, PathBuf::from("/work/backend")); - assert_eq!( - seen[0].env, - vec![ - ("ICP_CLI_ENVIRONMENT".to_owned(), "production".to_owned()), - ("ICP_CLI_NETWORK".to_owned(), "ic".to_owned()), - ("ICP_CLI_CID".to_owned(), cid.to_text()), - ( - "ICP_CLI_CID_MY_FRONTEND".to_owned(), - Principal::from_slice(&[8; 4]).to_text() - ), - ] - ); + async fn run_script( + &self, + _invocation: &ScriptInvocation, + _stdio: Option>, + ) -> Result, SynchronizeError> { + unimplemented!("UnimplementedMockSyncer::run_script") } } diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 54155db73..8eef08af3 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,8 +1,7 @@ -use std::collections::BTreeMap; - use camino::Utf8PathBuf; -use candid::Principal; use ic_agent::Agent; +use icp_deploy_canister::canister::recipe::{RemoteResourceResolve, ResolveError}; +use icp_deploy_canister::sync_exec; use icp_sync_plugin::{ CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, @@ -10,31 +9,12 @@ use icp_sync_plugin::{ use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{ - canister::wasm, - manifest::adapter::plugin::{Adapter, NamedPaths}, - package::PackageCache, -}; - -use super::Params; - -/// Convert a manifest [`NamedPaths`] (or its absence) into the runtime's -/// key-tagged path list. A missing setting yields an empty list. -fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { - paths - .into_iter() - .flat_map(NamedPaths::entries) - .map(|entry| KeyedPath { - key: entry.key.map(str::to_string), - path: entry.path.to_string(), - }) - .collect() -} +use crate::canister::ChannelProgress; #[derive(Debug, Snafu)] pub enum PluginError { - #[snafu(transparent)] - Wasm { source: wasm::WasmError }, + #[snafu(display("failed to resolve plugin wasm"))] + ResolveWasm { source: ResolveError }, #[snafu(display("failed to get identity principal: {err}"))] GetIdentityPrincipal { err: String }, @@ -46,12 +26,6 @@ pub enum PluginError { #[snafu(display("failed to run plugin"))] Run { source: RunPluginError }, - - #[snafu(display( - "sync plugin lists canister '{name}' as callable, but no canister by that name \ - is known in environment '{environment}'" - ))] - UnknownCallableCanister { name: String, environment: String }, } /// Resolve the plugin compute-time limit, honoring the @@ -82,14 +56,29 @@ fn parse_compute_limit(value: &str) -> Result { } } -pub(super) async fn sync( - adapter: &Adapter, - params: &Params, +/// Restate the library's key-tagged paths as the runtime's. The two types are +/// identical by construction and separate only because the runtime crate cannot +/// be depended on from `icp-deploy-canister`. +fn keyed_paths(paths: &[sync_exec::KeyedPath]) -> Vec { + paths + .iter() + .map(|entry| KeyedPath { + key: entry.key.clone(), + path: entry.path.clone(), + }) + .collect() +} + +/// Fetch and run a WASI plugin against a canister for a fully-resolved +/// [`sync_exec::PluginInvocation`]. Dispatch and input derivation — the +/// key-tagged paths, the fields, the exposed canister-id table and the resolved +/// `canisters:` list — happen in `icp-deploy-canister`; this only performs the +/// host-only wasm resolution and wasmtime execution. +pub(super) async fn run( + invocation: &sync_exec::PluginInvocation, agent: &Agent, - environment: &str, - proxy: Option, stdio: Option>, - pkg_cache: &PackageCache, + resolver: &dyn RemoteResourceResolve, ) -> Result, PluginError> { // 0. Resolve the compute-time limit up front so a malformed // ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS fails fast — before downloading the @@ -100,136 +89,49 @@ pub(super) async fn sync( // - Local: sha256 is verified if present, then the original path is returned. // - Remote: downloaded to cache (sha256 required, enforced at parse time) and the // stable cache path is returned — no temp file needed. - let wasm_path = wasm::resolve( - &adapter.source, - ¶ms.path, - adapter.sha256.as_deref(), - stdio.as_ref(), - pkg_cache, - ) - .await?; - - // 2. Collect inputs as manifest strings. `run_plugin` preopens the `dirs` - // and reads the `files` itself — both anchored at `base_dir`, confined to - // `project_dir`, and subject to the runtime's path-safety checks (no - // escaping or symlinked paths). - let base_dir = Utf8PathBuf::from(params.path.as_str()); - let project_dir = Utf8PathBuf::from(params.project_dir.as_str()); - let dirs = keyed_paths(adapter.dirs.as_ref()); - let files = keyed_paths(adapter.files.as_ref()); - let fields: BTreeMap = adapter.fields.clone().unwrap_or_default(); - - // 3. Build the canister ID table exposed to the plugin, then resolve the - // step's `canisters` list against it. - let canister_ids = exposed_canister_ids(params); - let callable = resolve_callable(adapter, &canister_ids, environment)?; - - // 4. Run the plugin (blocking call — signal Tokio that this thread will block). + let progress = ChannelProgress::wrap(stdio.as_ref()); + let wasm_path = resolver + .resolve_wasm( + &invocation.source, + &invocation.base_dir, + invocation.sha256.as_deref(), + ChannelProgress::as_dyn(progress.as_ref()), + ) + .await + .context(ResolveWasmSnafu)?; + + // 2. `run_plugin` preopens the `dirs` and reads the `files` itself — both + // anchored at `base_dir`, confined to `project_dir`, and subject to the + // runtime's path-safety checks (no escaping or symlinked paths). + let base_dir = Utf8PathBuf::from(invocation.base_dir.as_str()); + let project_dir = Utf8PathBuf::from(invocation.project_dir.as_str()); + + // 3. Run the plugin (blocking call — signal Tokio that this thread will block). let identity_principal = agent .get_principal() .map_err(|err| PluginError::GetIdentityPrincipal { err })?; - let agent_clone = agent.clone(); - let environment_owned = environment.to_owned(); - let stdio_clone = stdio.clone(); - - tokio::task::block_in_place(|| { - run_plugin(PluginInvocation { - wasm_path, - base_dir, - project_dir, - dirs, - files, - fields, - host_canister_id: params.cid, - agent: agent_clone, - proxy, - identity_principal, - environment: environment_owned, - compute_limit_secs, - canister_ids, - callable, - stdio: stdio_clone, - }) - }) - .context(RunSnafu) -} - -/// The canister ID table exposed to a sync plugin: every named canister in the -/// project, plus — for every canister in the subproject the synced canister -/// belongs to, or in a subproject nested below it — a duplicate entry under the -/// name that subproject itself uses. A store key is `:` for a -/// canister in a subproject and a bare local name for a canister defined -/// directly in the app root (see the WIT `canister-id-entry` docs), so the -/// syncing canister's namespace is the prefix of its own key. -/// -/// The aliases exist so a subproject's manifest and plugins keep working when it -/// is vendored into a workspace: both the step's `canisters:` list and the name a -/// plugin passes back as a call target are written where the subproject's own -/// names apply, but store keys are relative to the app root, which moves. See -/// [`member_relative_alias`] for the names produced. -/// -/// The aliases take precedence over a canister elsewhere in the workspace whose -/// store key happens to be spelled the same way: a plugin resolving such a name -/// is naming what its own subproject calls it. -fn exposed_canister_ids(params: &Params) -> BTreeMap { - // A canister in the app root is in the subproject the store keys are already - // relative to, so its names need no translation. - let Some((syncing_namespace, _)) = params.name.rsplit_once(':') else { - return params.canister_ids.clone(); + let runtime_invocation = PluginInvocation { + wasm_path, + base_dir, + project_dir, + dirs: keyed_paths(&invocation.dirs), + files: keyed_paths(&invocation.files), + fields: invocation.fields.clone(), + host_canister_id: invocation.canister_id, + agent: agent.clone(), + proxy: invocation.proxy, + identity_principal, + environment: invocation.environment.clone(), + compute_limit_secs, + canister_ids: invocation.canister_ids.clone(), + callable: CallableCanisters { + by_name: invocation.callable.clone(), + }, + stdio, }; - let mut table = params.canister_ids.clone(); - for (key, id) in ¶ms.canister_ids { - if let Some(alias) = member_relative_alias(syncing_namespace, key) { - table.insert(alias.to_owned(), *id); - } - } - table -} - -/// The name a canister has *within* the subproject at `namespace`: its store key -/// with that subproject's prefix removed. A canister of the subproject itself -/// comes back under its bare local name; one belonging to a subproject nested -/// below it comes back under the `:` key it would have if that -/// subproject were the app root. `None` for a canister the subproject has no name -/// of its own for. -/// -/// A local name never contains a colon but a subproject directory may, so a key -/// splits on its *last* colon. -fn member_relative_alias<'a>(namespace: &str, key: &'a str) -> Option<&'a str> { - let (key_namespace, _) = key.rsplit_once(':')?; - let rest = key.strip_prefix(namespace)?; - match key_namespace == namespace { - // The colon separating the subproject from a local name of its own. - true => rest.strip_prefix(':'), - // Otherwise the key's subproject must sit *below* this one. Demanding - // the path separator is what keeps `services/crm-legacy:backend` out of - // `services/crm`, which it merely shares a spelling prefix with. - false => rest.strip_prefix('/'), - } -} - -/// Resolve the step's `canisters` list into a [`CallableCanisters`] enforcement -/// set. Each listed name is looked up in `canister_ids`; a name that does not -/// resolve is a manifest error. -fn resolve_callable( - adapter: &Adapter, - canister_ids: &BTreeMap, - environment: &str, -) -> Result { - let mut by_name = BTreeMap::new(); - for name in adapter.canisters.iter().flatten() { - let principal = canister_ids - .get(name) - .copied() - .context(UnknownCallableCanisterSnafu { - name: name.clone(), - environment: environment.to_owned(), - })?; - by_name.insert(name.clone(), principal); - } - Ok(CallableCanisters { by_name }) + tokio::task::block_in_place(|| run_plugin(runtime_invocation)).context(RunSnafu) } #[cfg(test)] @@ -254,255 +156,4 @@ mod tests { ); } } - - use crate::manifest::adapter::prebuilt::{LocalSource, SourceField}; - - fn principal(byte: u8) -> Principal { - Principal::from_slice(&[byte; 4]) - } - - fn params_named(name: &str, ids: &[(&str, Principal)]) -> Params { - Params { - path: "/work".into(), - project_dir: "/work".into(), - cid: principal(0), - name: name.to_owned(), - environment: "demo".to_owned(), - network: "ic".to_owned(), - canister_ids: ids.iter().map(|(n, p)| ((*n).to_owned(), *p)).collect(), - proxy: None, - } - } - - fn adapter_with(canisters: Option>) -> Adapter { - Adapter { - source: SourceField::Local(LocalSource { - path: "plugin.wasm".into(), - }), - sha256: None, - dirs: None, - files: None, - fields: None, - canisters, - } - } - - /// Canisters sharing the syncing canister's subproject are additionally - /// exposed under their bare local name; canisters in other subprojects are - /// not. - #[test] - fn exposed_ids_add_bare_names_for_same_subproject() { - let backend = principal(1); - let frontend = principal(2); - let foreign = principal(3); - let params = params_named( - "services/open-accounts:backend", - &[ - ("services/open-accounts:backend", backend), - ("services/open-accounts:frontend", frontend), - ("services/open-crm:backend", foreign), - ], - ); - - let table = exposed_canister_ids(¶ms); - - // Same-subproject canisters gain a bare-local duplicate... - assert_eq!(table.get("backend"), Some(&backend)); - assert_eq!(table.get("frontend"), Some(&frontend)); - // ...while the fully-qualified keys are still present for everyone. - assert_eq!( - table.get("services/open-accounts:frontend"), - Some(&frontend) - ); - assert_eq!(table.get("services/open-crm:backend"), Some(&foreign)); - // The other subproject's canister is not reachable by a bare name; the - // bare "backend" belongs to the syncing canister's own subproject. - assert_eq!(table.get("backend"), Some(&backend)); - } - - /// A canister of a subproject nested below the syncing canister's own is - /// exposed under the key that subproject has relative to it — the very key - /// its manifest and plugins use when it is built standalone, so vendoring it - /// into a workspace leaves both spellings working. - #[test] - fn exposed_ids_add_member_relative_names_for_nested_subprojects() { - let ledger = principal(1); - let deep = principal(2); - let params = params_named( - "services/crm:backend", - &[ - ("services/crm:backend", principal(9)), - ("services/crm/vendor/ledger:ledger", ledger), - ("services/crm/vendor/ledger/vendor/util:util", deep), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("vendor/ledger:ledger"), Some(&ledger)); - // Nesting is not limited to one level: the whole subtree below the - // syncing canister's subproject is renamed relative to it. - assert_eq!(table.get("vendor/ledger/vendor/util:util"), Some(&deep)); - // The workspace-absolute keys remain. - assert_eq!( - table.get("services/crm/vendor/ledger:ledger"), - Some(&ledger) - ); - } - - /// A subproject whose path merely starts with the same characters is not - /// nested below the syncing canister's, so it contributes no alias. - #[test] - fn exposed_ids_ignore_a_subproject_sharing_a_spelling_prefix() { - let legacy = principal(1); - let params = params_named( - "services/crm:backend", - &[ - ("services/crm:backend", principal(9)), - ("services/crm-legacy:backend", legacy), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("services/crm-legacy:backend"), Some(&legacy)); - // Its local name belongs to the syncing canister, not to it. - assert_eq!(table.get("backend"), Some(&principal(9))); - assert_eq!(table.get("-legacy:backend"), None); - } - - /// A member-relative alias wins over a workspace canister whose store key is - /// spelled the same way, for the same reason a bare sibling name does: the - /// name is being read where the subproject's own names apply. - #[test] - fn exposed_ids_member_relative_alias_overrides_a_root_dependency_key() { - let root_ledger = principal(1); - let own_ledger = principal(2); - let params = params_named( - "services/crm:backend", - &[ - ("services/crm:backend", principal(9)), - ("services/crm/vendor/ledger:ledger", own_ledger), - ("vendor/ledger:ledger", root_ledger), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("vendor/ledger:ledger"), Some(&own_ledger)); - // The root's own dependency is still reachable, by its store key. - assert_eq!( - table.get("services/crm/vendor/ledger:ledger"), - Some(&own_ledger) - ); - assert!(!table.values().any(|id| *id == root_ledger)); - } - - /// An app-root canister sharing a local name with a sibling of the syncing - /// canister does not keep the bare name: the syncing subproject's own - /// canister is what that name means to the plugin. - #[test] - fn exposed_ids_sibling_alias_overrides_the_app_root_name() { - let root_backend = principal(1); - let sibling_backend = principal(2); - let params = params_named( - "services/open-accounts:frontend", - &[ - ("backend", root_backend), - ("services/open-accounts:backend", sibling_backend), - ("services/open-accounts:frontend", principal(3)), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("backend"), Some(&sibling_backend)); - // The app-root canister's only key was that bare name, so it drops out - // of the table entirely rather than answering to a sibling's name. - assert!(!table.values().any(|id| *id == root_backend)); - } - - /// A subproject directory may itself contain a colon, so keys are split on - /// their last one — the same rule bundling uses. - #[test] - fn exposed_ids_split_subproject_prefix_at_the_last_colon() { - let backend = principal(1); - let frontend = principal(2); - let nested = principal(3); - let params = params_named( - "services/odd:name:backend", - &[ - ("services/odd:name:backend", backend), - ("services/odd:name:frontend", frontend), - ("services/odd:name/vendor/ledger:ledger", nested), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("backend"), Some(&backend)); - assert_eq!(table.get("frontend"), Some(&frontend)); - assert_eq!(table.get("vendor/ledger:ledger"), Some(&nested)); - } - - /// The mirror image: a directory whose *name* contains a colon is not a - /// subproject nested below the part before it. `services/odd:name` holds one - /// directory named `odd:name`, so from `services/odd` it is nothing at all. - #[test] - fn exposed_ids_do_not_read_a_colon_in_a_directory_name_as_nesting() { - let odd = principal(1); - let params = params_named( - "services/odd:backend", - &[ - ("services/odd:backend", principal(9)), - ("services/odd:name:frontend", odd), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("services/odd:name:frontend"), Some(&odd)); - assert_eq!(table.get("name:frontend"), None); - } - - /// A single-project layout keys canisters by bare local name already, so no - /// duplicates are added. - #[test] - fn exposed_ids_unchanged_without_a_subproject() { - let backend = principal(1); - let params = params_named("backend", &[("backend", backend)]); - let table = exposed_canister_ids(¶ms); - assert_eq!(table.len(), 1); - assert_eq!(table.get("backend"), Some(&backend)); - } - - #[test] - fn resolve_callable_resolves_names() { - let dep = principal(1); - let sibling = principal(2); - let table = BTreeMap::from([ - ("backend".to_owned(), sibling), - ("services/open-crm:backend".to_owned(), dep), - ]); - let adapter = adapter_with(Some(vec![ - "backend".to_owned(), - "services/open-crm:backend".to_owned(), - ])); - - let callable = resolve_callable(&adapter, &table, "demo").unwrap(); - - assert_eq!(callable.by_name.get("backend"), Some(&sibling)); - assert_eq!( - callable.by_name.get("services/open-crm:backend"), - Some(&dep) - ); - } - - #[test] - fn resolve_callable_rejects_unknown_name() { - let adapter = adapter_with(Some(vec!["nope".to_owned()])); - let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") - .expect_err("an undeclared name must fail"); - assert!(matches!(err, PluginError::UnknownCallableCanister { .. })); - } } diff --git a/crates/icp/src/canister/sync/script.rs b/crates/icp/src/canister/sync/script.rs deleted file mode 100644 index 982396495..000000000 --- a/crates/icp/src/canister/sync/script.rs +++ /dev/null @@ -1,290 +0,0 @@ -//! Script sync steps, split into resolution and execution. -//! -//! [`ScriptInvocation::new`] resolves a manifest script step against the sync -//! [`Params`] — the command list, the working directory and the `ICP_CLI_*` -//! environment — without running anything. [`ScriptRunner`] then executes a -//! resolved invocation. -//! -//! Execution sits behind a trait because spawning a subprocess is the one part of -//! sync that cannot be done everywhere: plugin steps run inside the wasmtime WASI -//! sandbox, but a script step needs a shell. Keeping the split here means the -//! resolution half is portable and unit-testable, and an environment without -//! subprocesses can substitute a runner that refuses instead of losing the whole -//! sync path. - -use async_trait::async_trait; -use snafu::prelude::*; -use tokio::sync::mpsc::Sender; - -use crate::manifest::adapter::script::Adapter; -use crate::prelude::*; - -use super::Params; - -use super::super::script::execute_commands; - -/// A fully-resolved script sync step: the command(s), the working directory, and -/// the environment variables to set for them (see [`system_env_vars`]). -#[derive(Clone, Debug, PartialEq)] -pub struct ScriptInvocation { - /// Shell command(s) to run in order. - pub commands: Vec, - /// Working directory (the canister directory). - pub cwd: PathBuf, - /// Environment variables a runner **adds to** its execution environment, in - /// insertion order — not the complete environment. [`HostScripts`] overlays - /// them onto the inherited process environment, so the script still sees - /// ambient variables such as `PATH` and `HOME`; entries here win on a name - /// collision. A runner is not expected to clear what it inherits. - pub env: Vec<(String, String)>, -} - -impl ScriptInvocation { - /// Resolve a script step's adapter against the sync context, assembling the - /// `ICP_CLI_*` system environment variables the command runs with. - pub fn new(adapter: &Adapter, params: &Params) -> Self { - Self { - commands: adapter.command.as_vec(), - cwd: params.path.clone(), - env: system_env_vars(params), - } - } -} - -/// The `ICP_CLI_*` system environment variables every script sync step runs -/// with: the environment and network names, the target canister id, and one -/// `ICP_CLI_CID_` per known canister in the environment (name uppercased, -/// non-alphanumerics replaced with `_`). -pub fn system_env_vars(params: &Params) -> Vec<(String, String)> { - let mut envs = vec![ - ("ICP_CLI_ENVIRONMENT".to_owned(), params.environment.clone()), - ("ICP_CLI_NETWORK".to_owned(), params.network.clone()), - ("ICP_CLI_CID".to_owned(), params.cid.to_text()), - ]; - for (name, id) in ¶ms.canister_ids { - let key = format!( - "ICP_CLI_CID_{}", - name.to_uppercase() - .chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .collect::() - ); - envs.push((key, id.to_text())); - } - envs -} - -#[derive(Debug, Snafu)] -#[snafu(display("script sync step failed"))] -pub struct ScriptRunError { - /// Boxed because the error depends on how the runner executes: the host - /// runner fails with a [`ScriptError`](super::super::script::ScriptError), a - /// runner that refuses scripts fails with something else entirely. - pub source: Box, -} - -/// Executes resolved script sync steps. -#[async_trait] -pub trait ScriptRunner: Sync + Send { - /// Run a resolved script step, streaming output to `stdio`, and return any - /// stderr lines to retain past the streamed view. - async fn run_script( - &self, - invocation: ScriptInvocation, - stdio: Option>, - ) -> Result, ScriptRunError>; -} - -/// The [`ScriptRunner`] that spawns each command as a host subprocess. -pub struct HostScripts; - -#[async_trait] -impl ScriptRunner for HostScripts { - async fn run_script( - &self, - invocation: ScriptInvocation, - stdio: Option>, - ) -> Result, ScriptRunError> { - let env_refs: Vec<(&str, &str)> = invocation - .env - .iter() - .map(|(k, v)| (k.as_str(), v.as_str())) - .collect(); - execute_commands(&invocation.commands, &invocation.cwd, &env_refs, stdio) - .await - .map_err(|source| ScriptRunError { - source: Box::new(source), - })?; - // Persistent stderr is a sync-plugin feature only; script steps don't - // currently retain any output past the rolling step view. - Ok(vec![]) - } -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use tokio::sync::Mutex; - - use candid::Principal; - - use super::*; - use crate::manifest::adapter::script::CommandField; - - /// Serializes the tests here that mutate the process environment, since - /// cargo runs tests in parallel threads. Async-aware because the variable - /// has to stay set across the subprocess `await` that reads it. - static ENV_MUTEX: Mutex<()> = Mutex::const_new(()); - - fn principal(byte: u8) -> Principal { - Principal::from_slice(&[byte; 4]) - } - - fn params(canister_ids: &[(&str, Principal)]) -> Params { - Params { - path: "/work/backend".into(), - project_dir: "/work".into(), - cid: principal(1), - name: "backend".to_owned(), - environment: "production".to_owned(), - network: "ic".to_owned(), - canister_ids: canister_ids - .iter() - .map(|(n, p)| ((*n).to_owned(), *p)) - .collect::>(), - proxy: None, - } - } - - /// The environment, network and target canister id are always present. - #[test] - fn base_env_vars_are_always_set() { - let p = params(&[]); - assert_eq!( - system_env_vars(&p), - vec![ - ("ICP_CLI_ENVIRONMENT".to_owned(), "production".to_owned()), - ("ICP_CLI_NETWORK".to_owned(), "ic".to_owned()), - ("ICP_CLI_CID".to_owned(), principal(1).to_text()), - ] - ); - } - - /// Each known canister gets an `ICP_CLI_CID_` variable, with the name - /// uppercased and every non-alphanumeric byte replaced by `_` so the result - /// is a legal shell identifier. - #[test] - fn canister_names_are_normalized_into_env_var_keys() { - let p = params(&[("my-frontend", principal(2)), ("dep:api", principal(3))]); - let keys: Vec = system_env_vars(&p) - .into_iter() - .map(|(k, _)| k) - .filter(|k| k.starts_with("ICP_CLI_CID_")) - .collect(); - // `canister_ids` is a BTreeMap, so ordering follows the canister names. - assert_eq!(keys, vec!["ICP_CLI_CID_DEP_API", "ICP_CLI_CID_MY_FRONTEND"]); - } - - /// Resolution takes the commands and cwd from the step and its canister, and - /// runs nothing. - #[test] - fn invocation_resolves_commands_and_cwd() { - let adapter = Adapter { - command: CommandField::Commands(vec!["first".to_owned(), "second".to_owned()]), - }; - let invocation = ScriptInvocation::new(&adapter, ¶ms(&[])); - - assert_eq!(invocation.commands, vec!["first", "second"]); - assert_eq!(invocation.cwd, PathBuf::from("/work/backend")); - assert_eq!(invocation.env, system_env_vars(¶ms(&[]))); - } - - /// The host runner passes the resolved environment through to the subprocess. - #[tokio::test] - async fn host_runner_applies_the_resolved_environment() { - let out = camino_tempfile::NamedUtf8TempFile::new().unwrap(); - let invocation = ScriptInvocation { - commands: vec![format!("printenv ICP_CLI_NETWORK > '{}'", out.path())], - cwd: "/".into(), - env: system_env_vars(¶ms(&[])), - }; - - HostScripts.run_script(invocation, None).await.unwrap(); - - assert_eq!(std::fs::read_to_string(out.path()).unwrap(), "ic\n"); - } - - /// `env` is an overlay, not the whole environment: the script still sees - /// variables the parent process had. Pins the contract documented on - /// [`ScriptInvocation::env`]. - /// - /// Deliberately avoids two things that are not portable across the shells - /// this runs under. `printenv` accepts only one operand on BSD (macOS) and - /// so silently drops later names, hence one `echo` — a shell builtin - /// everywhere — per variable. And the inherited variable is one this test - /// sets rather than `PATH`, because Git-for-Windows bash rewrites `PATH` - /// into POSIX form, so its value there never equals the `PATH` the Rust side - /// reads. - #[tokio::test] - async fn host_runner_overlays_rather_than_replaces_the_environment() { - const AMBIENT: &str = "ICP_CLI_TEST_AMBIENT_VAR"; - const AMBIENT_VALUE: &str = "inherited-from-parent"; - - let _guard = ENV_MUTEX.lock().await; - // SAFETY: ENV_MUTEX serializes the tests in this module that mutate the - // process environment, and the name is used by this test alone. - unsafe { std::env::set_var(AMBIENT, AMBIENT_VALUE) }; - - let overlaid = camino_tempfile::NamedUtf8TempFile::new().unwrap(); - let inherited = camino_tempfile::NamedUtf8TempFile::new().unwrap(); - let invocation = ScriptInvocation { - commands: vec![ - format!("echo \"$ICP_CLI_NETWORK\" > '{}'", overlaid.path()), - format!("echo \"${AMBIENT}\" > '{}'", inherited.path()), - ], - cwd: "/".into(), - env: system_env_vars(¶ms(&[])), - }; - - let run = HostScripts.run_script(invocation, None).await; - - // SAFETY: as above; the guard is still held. - unsafe { std::env::remove_var(AMBIENT) }; - run.expect("script must run"); - - assert_eq!( - std::fs::read_to_string(overlaid.path()).unwrap().trim(), - "ic", - "the overlaid variable must be set" - ); - assert_eq!( - std::fs::read_to_string(inherited.path()).unwrap().trim(), - AMBIENT_VALUE, - "a variable the parent process had must still reach the script" - ); - } - - /// A command that exits non-zero surfaces as a `ScriptRunError` whose source - /// still names the command and its status, so the `caused by:` line the CLI - /// prints stays specific. - #[tokio::test] - async fn host_runner_reports_a_failing_command() { - let invocation = ScriptInvocation { - commands: vec!["exit 3".to_owned()], - cwd: "/".into(), - env: vec![], - }; - - let err = HostScripts - .run_script(invocation, None) - .await - .expect_err("a non-zero exit must fail the step"); - - assert_eq!(err.to_string(), "script sync step failed"); - assert_eq!( - std::error::Error::source(&err).expect("cause").to_string(), - "command 'exit 3' failed with status code 3" - ); - } -} diff --git a/crates/icp/src/canister/wasm.rs b/crates/icp/src/canister/wasm.rs index 2cf2b219d..3b2ead5e5 100644 --- a/crates/icp/src/canister/wasm.rs +++ b/crates/icp/src/canister/wasm.rs @@ -1,8 +1,8 @@ use camino::{Utf8Path, Utf8PathBuf}; +use icp_deploy_canister::sync_exec::StepProgress; use reqwest::{Client, Method, Request}; use sha2::{Digest, Sha256}; use snafu::prelude::*; -use tokio::sync::mpsc::Sender; use url::Url; use crate::{ @@ -50,21 +50,21 @@ pub async fn resolve( source: &SourceField, base_dir: &Utf8Path, sha256: Option<&str>, - stdio: Option<&Sender>, + progress: Option<&dyn StepProgress>, pkg_cache: &PackageCache, ) -> Result { match source { SourceField::Local(s) => { let path = base_dir.join(&s.path); if let Some(expected) = sha256 { - if let Some(tx) = stdio { - let _ = tx.send(format!("Reading wasm: {}", s.path)).await; + if let Some(p) = progress { + p.line(format!("Reading wasm: {}", s.path)); } let bytes = read(&path).context(ReadLocalSnafu { path: s.path.clone(), })?; - if let Some(tx) = stdio { - let _ = tx.send("Verifying checksum".to_string()).await; + if let Some(p) = progress { + p.line("Verifying checksum".to_string()); } let actual = hex::encode(Sha256::digest(&bytes)); ensure!( @@ -94,16 +94,16 @@ pub async fn resolve( .await .context(LockCacheSnafu)?; if let Some(path) = cached { - if let Some(tx) = stdio { - let _ = tx.send("Using cached file".to_string()).await; + if let Some(p) = progress { + p.line("Using cached file".to_string()); } return Ok(path); } } let url = Url::parse(&s.url).context(ParseUrlSnafu)?; - if let Some(tx) = stdio { - let _ = tx.send(format!("Fetching wasm: {url}")).await; + if let Some(p) = progress { + p.line(format!("Fetching wasm: {url}")); } let resp = Client::new() .execute(Request::new(Method::GET, url)) @@ -118,8 +118,8 @@ pub async fn resolve( // Use provided sha256 as cache key (after verifying), or compute from bytes. let cache_sha = match sha256 { Some(expected) => { - if let Some(tx) = stdio { - let _ = tx.send("Verifying checksum".to_string()).await; + if let Some(p) = progress { + p.line("Verifying checksum".to_string()); } let actual = hex::encode(Sha256::digest(&bytes)); ensure!( diff --git a/crates/icp/src/context/init.rs b/crates/icp/src/context/init.rs index 4af5cda79..1db542d89 100644 --- a/crates/icp/src/context/init.rs +++ b/crates/icp/src/context/init.rs @@ -3,7 +3,7 @@ use std::{env::current_dir, sync::Arc}; use snafu::prelude::*; use crate::canister::build::Builder; -use crate::canister::recipe::fetch::RecipeFetcher; +use crate::canister::recipe::resolver::ResourceResolver; use crate::canister::sync::Syncer; use crate::context::Context; use crate::directories::{Access as _, Directories}; @@ -90,7 +90,7 @@ pub fn initialize( let pkg_cache = dirs.package_cache().context(PackageCacheSnafu)?; // Recipes - let recipe = Arc::new(RecipeFetcher { + let recipe = Arc::new(ResourceResolver { http_client, pkg_cache, }); @@ -99,7 +99,7 @@ pub fn initialize( let builder = Arc::new(Builder); // Canister syncer - let syncer = Arc::new(Syncer::host()); + let syncer = Arc::new(Syncer); // Project loader let pload = ProjectLoadImpl { diff --git a/crates/icp/src/context/mod.rs b/crates/icp/src/context/mod.rs index 3ae5ea72a..8befa0ff5 100644 --- a/crates/icp/src/context/mod.rs +++ b/crates/icp/src/context/mod.rs @@ -111,6 +111,20 @@ pub struct Context { } impl Context { + /// Build a resolver for the project's remote resources — recipe templates + /// and plugin wasms — backed by the package cache and an HTTP client. + pub fn resource_resolver( + &self, + ) -> Result, crate::fs::lock::LockError> + { + Ok(Arc::new( + crate::canister::recipe::resolver::ResourceResolver { + http_client: reqwest::Client::new(), + pkg_cache: self.dirs.package_cache()?, + }, + )) + } + /// Gets an identity based on the provided identity selection. // TODO: refactor the whole codebase to use this method instead of directly accessing `ctx.identity.load()` pub async fn get_identity( diff --git a/crates/icp/src/host_files.rs b/crates/icp/src/host_files.rs new file mode 100644 index 000000000..9129f3e45 --- /dev/null +++ b/crates/icp/src/host_files.rs @@ -0,0 +1,64 @@ +//! Host filesystem implementation of [`icp_deploy_canister::FileAccess`]. +//! +//! Backs project loading/consolidation on the real filesystem. Stateless: +//! operates on the (absolute) paths the model passes in. + +use async_trait::async_trait; +use icp_deploy_canister::files::{FileAccess, FileAccessError}; + +use crate::prelude::*; + +#[derive(Debug, Default, Clone, Copy)] +pub struct HostFileAccess; + +#[async_trait] +impl FileAccess for HostFileAccess { + async fn read_file(&self, path: &Path) -> Result, FileAccessError> { + crate::fs::read(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn read_to_string(&self, path: &Path) -> Result { + crate::fs::read_to_string(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn exists(&self, path: &Path) -> bool { + path.exists() + } + + async fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + async fn is_dir(&self, path: &Path) -> bool { + path.is_dir() + } + + async fn read_dir(&self, path: &Path) -> Result, FileAccessError> { + let rd = std::fs::read_dir(path).map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + let mut out = Vec::new(); + for entry in rd { + let entry = entry.map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + if let Ok(p) = PathBuf::from_path_buf(entry.path()) { + out.push(p); + } + } + Ok(out) + } + + async fn canonicalize(&self, path: &Path) -> Option { + let canon = dunce::canonicalize(path.as_std_path()).ok()?; + PathBuf::try_from(canon).ok() + } +} diff --git a/crates/icp/src/lib.rs b/crates/icp/src/lib.rs index c5433ce82..4e1da41bb 100644 --- a/crates/icp/src/lib.rs +++ b/crates/icp/src/lib.rs @@ -1,34 +1,35 @@ -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, -}; +use std::sync::Arc; use async_trait::async_trait; -use indexmap::IndexMap; -use serde::Serialize; use snafu::prelude::*; use tokio::sync::Mutex; use tracing::debug; -use candid_parser::parse_idl_args; +pub use icp_deploy_canister::{ + Canister, Environment, InitArgs, InitArgsToBytesError, Network, Project, +}; use crate::{ - canister::{Settings, recipe::Resolve}, + canister::recipe::RemoteResourceResolve, manifest::{ - ArgsFormat, LoadManifestFromPathError, PROJECT_MANIFEST, ProjectRootLocate, - ProjectRootLocateError, - canister::{BuildSteps, SyncSteps}, + LoadManifestFromPathError, PROJECT_MANIFEST, ProjectRootLocate, ProjectRootLocateError, load_manifest_from_path, }, - network::Configuration, prelude::*, }; +// Imports used only by the in-crate test mock builders below. +#[cfg(test)] +use std::collections::{BTreeMap, HashMap}; +#[cfg(test)] +use {crate::canister::Settings, indexmap::IndexMap}; + pub mod agent; pub mod canister; pub mod context; pub mod directories; pub mod fs; +pub mod host_files; pub mod identity; pub mod manifest; pub mod network; @@ -47,157 +48,6 @@ const ICP_BASE: &str = ".icp"; const CACHE_DIR: &str = "cache"; const DATA_DIR: &str = "data"; -/// Resolved initialization arguments, with any file references already loaded. -#[derive(Clone, Debug, PartialEq, Serialize)] -pub enum InitArgs { - /// Text content (inline or loaded from file). Format is always known. - Text { content: String, format: ArgsFormat }, - /// Raw binary bytes (from a file with `format: bin`). Used directly. - Binary(Vec), -} - -#[derive(Debug, Snafu)] -pub enum InitArgsToBytesError { - #[snafu(display("failed to decode hex init args"))] - HexDecode { source: hex::FromHexError }, - - #[snafu(display("failed to parse Candid init args"))] - CandidParse { source: candid_parser::Error }, - - #[snafu(display("failed to encode Candid init args to bytes"))] - CandidEncode { source: candid::Error }, -} - -impl InitArgs { - /// Resolve to raw bytes according to the format. - pub fn to_bytes(&self) -> Result, InitArgsToBytesError> { - match self { - InitArgs::Binary(bytes) => Ok(bytes.clone()), - InitArgs::Text { content, format } => match format { - ArgsFormat::Hex => hex::decode(content.trim()).context(HexDecodeSnafu), - ArgsFormat::Candid => { - let args = parse_idl_args(content.trim()).context(CandidParseSnafu)?; - args.to_bytes().context(CandidEncodeSnafu) - } - ArgsFormat::Bin => { - unreachable!("binary format cannot appear in InitArgs::Text") - } - }, - } - } -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Canister { - pub name: String, - - /// Canister settings, such as memory constaints, etc. - pub settings: Settings, - - /// The build configuration specifying how to compile the canister's source - /// code into a WebAssembly module, including the adapter to use. - pub build: BuildSteps, - - /// The configuration specifying how to sync the canister - pub sync: SyncSteps, - - /// Initialization arguments passed to the canister during installation. - /// Resolved from the manifest — file contents are already loaded. - pub init_args: Option, - - /// If the canister was defined via a recipe reference, this holds the - /// original recipe specifier string (e.g. `@dfinity/motoko@v4.0.0`). - /// `None` when the canister uses explicit build/sync instructions. - pub registry_recipe: Option, - - /// Canister-discovery wiring. Maps the name this canister reads in a - /// `PUBLIC_CANISTER_ID:` environment variable to the store key of the - /// referenced canister. Computed during consolidation so each canister sees - /// the view its owning project expects: its own project's canisters under - /// their local names, plus any declared dependencies under their aliases - /// (`:`). For a project with no dependencies this maps every - /// canister's local name to itself, reproducing the flat "every canister sees - /// every sibling" behavior. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub bindings: BTreeMap, - - /// Subdomain prefixes for the canister's friendly URLs, most-specific label - /// first, e.g. `["backend"]` for an own canister or `["backend.openemail"]` - /// for a dependency canister (dot-nested by alias chain). A de-duplicated - /// shared dependency canister carries one entry per alias chain that reaches - /// it. Consumed only at deploy time to build `custom-domains.txt` entries and - /// the printed URLs; a runtime display aid that is always recomputed during - /// consolidation, so it is never serialized. - #[serde(skip)] - pub friendly_names: Vec, - - /// For each environment variable whose value came from a file, the file it - /// was read from. `settings.environment_variables` already holds the - /// contents; the paths are kept so `icp project bundle` can hold a file - /// backing a variable to the same containment rule it applies to every other - /// file a manifest points at. Bookkeeping for that check rather than part of - /// the resolved configuration, so it is never serialized. - #[serde(skip)] - pub environment_variable_files: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Network { - pub name: String, - pub configuration: Configuration, -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Environment { - pub name: String, - pub network: Network, - pub canisters: IndexMap, -} - -impl Environment { - pub fn get_canister_names(&self) -> Vec { - self.canisters.keys().cloned().collect() - } - - pub fn contains_canister(&self, canister_name: &str) -> bool { - self.canisters.contains_key(canister_name) - } - - pub fn get_canister_info(&self, canister: &str) -> Result<(PathBuf, Canister), String> { - self.canisters - .get(canister) - .ok_or_else(|| { - format!( - "canister '{}' not declared in environment '{}'", - canister, self.name - ) - }) - .cloned() - } -} - -/// Consolidated project definition -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Project { - pub dir: PathBuf, - pub canisters: IndexMap, - pub networks: HashMap, - pub environments: HashMap, - - /// Environments the workspace defines that some vendored member does *not* - /// declare, keyed by environment name → the missing members' store-key - /// prefixes. Enforced when the environment is selected (strict rule). - /// Empty for standalone projects and workspaces whose members are complete. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub member_missing_envs: HashMap>, -} - -impl Project { - pub fn get_canister(&self, canister_name: &str) -> Option<&(PathBuf, Canister)> { - self.canisters.get(canister_name) - } -} - #[derive(Debug, Snafu)] pub enum ProjectLoadError { #[snafu(display("failed to locate project directory"))] @@ -228,7 +78,7 @@ pub trait ProjectLoad: Sync + Send { pub struct ProjectLoadImpl { pub project_root_locate: Arc, - pub recipe: Arc, + pub recipe: Arc, } /// Ensures the "operating on a workspace root above your sub-project" notice is @@ -278,10 +128,15 @@ impl ProjectLoad for ProjectLoadImpl { debug!("Loaded project manifest: {m:#?}"); - // Consolidate manifest into project - let p = project::consolidate_manifest(&pdir, self.recipe.as_ref(), &m) - .await - .context(ProjectSnafu)?; + // Consolidate manifest into project, reading files from the host filesystem. + let p = project::consolidate_manifest( + &crate::host_files::HostFileAccess, + &pdir, + self.recipe.as_ref(), + &m, + ) + .await + .context(ProjectSnafu)?; debug!("Rendered project definition: {p:#?}"); @@ -696,9 +551,12 @@ impl ProjectLoad for NoProjectLoader { #[cfg(test)] mod tests { use super::*; - use crate::canister::recipe::{Fetched, Resolve, ResolveError}; - use crate::manifest::{ProjectRootLocate, ProjectRootLocateError, recipe::Recipe}; + use crate::canister::recipe::{FetchedRecipe, RemoteResourceResolve, ResolveError}; + use crate::manifest::{ + ProjectRootLocate, ProjectRootLocateError, adapter::prebuilt::SourceField, recipe::Recipe, + }; use camino_tempfile::Utf8TempDir; + use icp_deploy_canister::sync_exec::StepProgress; use indoc::indoc; struct MockProjectRootLocate { @@ -724,11 +582,10 @@ mod tests { struct MockRecipeResolver; #[async_trait] - impl Resolve for MockRecipeResolver { - /// A minimal template rendering to a single dummy pre-built step. Nothing - /// is fetched, so there is no cache write to hold back. - async fn resolve(&self, _recipe: &Recipe) -> Result { - Ok(Fetched { + impl RemoteResourceResolve for MockRecipeResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + // A minimal recipe template rendering to a single prebuilt build step. + Ok(FetchedRecipe { template: indoc! {r#" build: steps: @@ -736,9 +593,27 @@ mod tests { path: dummy.wasm "#} .to_owned(), - pending_cache: None, + deferred: false, }) } + + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + Ok(()) + } + + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _progress: Option<&dyn StepProgress>, + ) -> Result { + unimplemented!("MockRecipeResolver::resolve_wasm") + } } #[tokio::test] diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index 0a811e358..c1019bf76 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -1,113 +1,20 @@ +//! Host-side manifest facade. +//! +//! The manifest *model* (types, `Item`, constants) lives in +//! `icp_deploy_canister::manifest` and is re-exported here. This module keeps +//! the pieces that walk the real filesystem: locating the project/workspace root +//! and the `std::fs`-based manifest loader. + use std::collections::HashSet; -use std::marker::PhantomData; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use snafu::prelude::*; +pub use icp_deploy_canister::manifest::*; + use crate::fs; use crate::prelude::*; -pub(crate) mod adapter; -pub(crate) mod canister; -pub(crate) mod dependency; -pub(crate) mod environment; -pub(crate) mod network; -pub(crate) mod project; -pub(crate) mod recipe; -pub(crate) mod serde_helpers; - -pub use { - adapter::plugin, - adapter::prebuilt, - canister::{ - ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, - SyncStep, SyncSteps, - }, - dependency::DependencyManifest, - environment::EnvironmentManifest, - network::{ManagedMode, Mode, NetworkManifest}, - project::ProjectManifest, -}; - -pub const PROJECT_MANIFEST: &str = "icp.yaml"; -pub const CANISTER_MANIFEST: &str = "canister.yaml"; - -// A manifest item that can either be a path to another manifest file or the manifest itself. -// -// The valid path specifications are: -// - CanisterManifest: path or glob pattern to the directory containing "canister.yaml" -// - NetworkManifest: path to network manifest -// - EnvironmentManifest: path to environment manifest -#[derive(Clone, Debug, PartialEq, JsonSchema)] -#[serde(untagged)] -pub enum Item { - /// Path to a manifest - Path(String), - - /// The manifest - Manifest(T), -} - -/// Items in path form serialize back to a bare path string, *not* to the contents of the -/// referenced file. Callers that need a self-contained YAML output (e.g. `icp project bundle`) -/// must convert any `Item::Path` to `Item::Manifest` themselves by loading the referenced -/// manifest first. -impl Serialize for Item { - fn serialize(&self, serializer: S) -> Result { - match self { - Item::Path(p) => p.serialize(serializer), - Item::Manifest(m) => m.serialize(serializer), - } - } -} - -impl<'de, T> Deserialize<'de> for Item -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::{MapAccess, Visitor, value::MapAccessDeserializer}; - use std::fmt; - - struct ItemVisitor(PhantomData); - - impl<'de, T: Deserialize<'de>> Visitor<'de> for ItemVisitor { - type Value = Item; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a string path or a manifest object") - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Ok(Item::Path(v.to_owned())) - } - - fn visit_string(self, v: String) -> Result - where - E: serde::de::Error, - { - Ok(Item::Path(v)) - } - - fn visit_map(self, map: M) -> Result - where - M: MapAccess<'de>, - { - T::deserialize(MapAccessDeserializer::new(map)).map(Item::Manifest) - } - } - - deserializer.deserialize_any(ItemVisitor(PhantomData)) - } -} - #[derive(Debug, Snafu)] pub enum ProjectRootLocateError { #[snafu(display("project manifest not found in {path}"))] @@ -290,7 +197,7 @@ pub enum LoadManifestFromPathError { }, } -/// Loads a manifest of type `T` from the specified file path. +/// Loads a manifest of type `T` from the specified file path (host filesystem). pub async fn load_manifest_from_path(path: &Path) -> Result where T: for<'de> Deserialize<'de>, diff --git a/crates/icp/src/network/mod.rs b/crates/icp/src/network/mod.rs index 3d025fb40..e075c49e3 100644 --- a/crates/icp/src/network/mod.rs +++ b/crates/icp/src/network/mod.rs @@ -1,29 +1,28 @@ +//! Host-side network facade. +//! +//! The network *configuration* model lives in `icp_deploy_canister::network` +//! and is re-exported here. Runtime access (launching/stopping managed +//! networks, descriptors, agent bootstrap) stays in this crate. + use std::sync::Arc; use async_trait::async_trait; -use schemars::JsonSchema; -use serde::{Deserialize, Deserializer, Serialize}; use snafu::prelude::*; -pub use crate::manifest::network::RootKeySpec; +pub use icp_deploy_canister::network::*; + pub use access::RootKeySource; pub use directory::{LoadPidError, NetworkDirectory, SavePidError}; pub use managed::run::{RunNetworkError, run_network}; -use strum::EnumString; -use url::Url; use crate::{ CACHE_DIR, ICP_BASE, Network, - manifest::{ - ProjectRootLocate, ProjectRootLocateError, - network::{Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode}, - }, + manifest::{ProjectRootLocate, ProjectRootLocateError}, network::access::{ GetNetworkAccessError, NetworkAccess, get_connected_network_access, get_managed_network_access, }, prelude::*, - project::DEFAULT_LOCAL_NETWORK_PORT, }; pub mod access; @@ -32,309 +31,6 @@ pub mod custom_domains; pub mod directory; pub mod managed; -#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] -pub enum Port { - Fixed(u16), - Random, -} - -impl Default for Port { - fn default() -> Self { - Port::Fixed(8000) - } -} - -impl<'de> Deserialize<'de> for Port { - fn deserialize>(d: D) -> Result { - Ok(match u16::deserialize(d)? { - 0 => Port::Random, - p => Port::Fixed(p), - }) - } -} - -fn default_bind() -> String { - "127.0.0.1".to_string() -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Gateway { - #[serde(default = "default_bind")] - pub bind: String, - - #[serde(default)] - pub port: Port, - - #[serde(default)] - pub domains: Vec, -} - -impl Default for Gateway { - fn default() -> Self { - Self { - bind: default_bind(), - port: Default::default(), - domains: Default::default(), - } - } -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Managed { - #[serde(flatten)] - pub mode: ManagedMode, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -#[serde(untagged)] -pub enum ManagedMode { - Image(Box), - Launcher(Box), -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct ManagedLauncherConfig { - pub gateway: Gateway, - pub artificial_delay_ms: Option, - pub ii: bool, - pub nns: bool, - pub subnets: Option>, - pub bitcoind_addr: Option>, - pub dogecoind_addr: Option>, - pub version: Option, -} - -#[derive( - Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize, EnumString, strum::Display, -)] -#[serde(rename_all = "kebab-case")] -#[strum(serialize_all = "kebab-case")] -pub enum SubnetKind { - Application, - System, - VerifiedApplication, - Bitcoin, - Fiduciary, - Nns, - Sns, -} - -impl Default for ManagedMode { - fn default() -> Self { - Self::default_for_port(DEFAULT_LOCAL_NETWORK_PORT) - } -} - -impl ManagedMode { - pub fn default_for_port(port: u16) -> Self { - ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway: Gateway { - bind: default_bind(), - port: if port == 0 { - Port::Random - } else { - Port::Fixed(port) - }, - domains: vec![], - }, - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: None, - dogecoind_addr: None, - version: None, - })) - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct ManagedImageConfig { - pub image: String, - pub port_mapping: Vec, - pub rm_on_exit: bool, - pub args: Vec, - pub entrypoint: Option>, - pub environment: Vec, - pub volumes: Vec, - pub platform: Option, - pub user: Option, - pub shm_size: Option, - pub status_dir: String, - pub mounts: Vec, - pub extra_hosts: Vec, -} - -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct Connected { - /// The URL this network's API can be reached at. - pub api_url: Url, - - /// The URL this network's HTTP gateway can be reached at. - pub http_gateway_url: Option, - - /// How to obtain the root key used to verify responses from this network. - pub root_key: RootKeySpec, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -#[serde(tag = "mode", rename_all = "lowercase")] -pub enum Configuration { - // Note: we must use struct variants to be able to flatten - // and make schemars generate the proper schema - /// A managed network is one which can be controlled and manipulated. - Managed { - #[serde(flatten)] - managed: Managed, - }, - - /// A connected network is one which can be interacted with - /// but cannot be controlled or manipulated. - Connected { - #[serde(flatten)] - connected: Connected, - }, -} - -impl Default for Configuration { - fn default() -> Self { - Configuration::Managed { - managed: Managed::default(), - } - } -} - -impl From for Gateway { - fn from(value: ManifestGateway) -> Self { - let ManifestGateway { - bind, - domains, - port, - } = value; - let bind = bind.unwrap_or("127.0.0.1".to_string()); - let port = match port { - Some(0) => Port::Random, - Some(p) => Port::Fixed(p), - None => Port::default(), - }; - let mut domains = domains.unwrap_or_default(); - if bind == "127.0.0.1" || bind == "0.0.0.0" || bind == "::1" || bind == "::" { - domains.insert(0, "localhost".to_string()); - } - Gateway { - bind, - port, - domains, - } - } -} - -impl From for Connected { - fn from(value: ManifestConnected) -> Self { - let root_key = value.root_key; - match value.endpoints { - Endpoints::Implicit { url } => Connected { - api_url: url.clone(), - http_gateway_url: Some(url), - root_key, - }, - Endpoints::Explicit { - api_url, - http_gateway_url, - } => Connected { - api_url, - http_gateway_url, - root_key, - }, - } - } -} - -impl From for Configuration { - fn from(value: Mode) -> Self { - match value { - Mode::Managed(managed) => match *managed.mode { - crate::manifest::network::ManagedMode::Launcher { - gateway, - artificial_delay_ms, - ii, - nns, - subnets, - bitcoind_addr, - dogecoind_addr, - version, - } => { - let gateway: Gateway = match gateway { - Some(g) => g.into(), - None => Gateway::default(), - }; - let version = match version { - Some(v) => { - if v.starts_with('v') { - Some(v) - } else { - Some(format!("v{v}")) - } - } - None => None, - }; - Configuration::Managed { - managed: Managed { - mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway, - artificial_delay_ms, - ii: ii.unwrap_or(false), - nns: nns.unwrap_or(false), - subnets, - bitcoind_addr, - dogecoind_addr, - version, - })), - }, - } - } - crate::manifest::network::ManagedMode::Image { - image, - port_mapping, - rm_on_exit, - args, - entrypoint, - environment, - volumes, - platform, - user, - shm_size, - status_dir, - mounts: mount, - extra_hosts, - } => Configuration::Managed { - managed: Managed { - mode: ManagedMode::Image(Box::new(ManagedImageConfig { - image, - port_mapping, - rm_on_exit: rm_on_exit.unwrap_or(false), - args: args.unwrap_or_default(), - entrypoint, - environment: environment.unwrap_or_default(), - volumes: volumes.unwrap_or_default(), - platform, - user, - shm_size, - status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), - mounts: mount.unwrap_or_default(), - extra_hosts: extra_hosts.unwrap_or_default(), - })), - }, - }, - }, - Mode::Connected(connected) => Configuration::Connected { - connected: connected.into(), - }, - } - } -} - #[derive(Debug, Snafu)] pub enum AccessError { #[snafu(display("failed to find project root"))] @@ -444,51 +140,3 @@ impl Access for MockNetworkAccessor { }) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::manifest::network::{ - Gateway as ManifestGateway, Managed as ManifestManaged, ManagedMode as ManifestManagedMode, - Mode, - }; - - #[test] - fn from_mode_launcher_with_bitcoind_addr() { - let mode = Mode::Managed(ManifestManaged { - mode: Box::new(ManifestManagedMode::Launcher { - gateway: Some(ManifestGateway { - bind: None, - port: Some(8000), - domains: None, - }), - artificial_delay_ms: None, - ii: None, - nns: None, - subnets: None, - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: None, - version: None, - }), - }); - - let config: Configuration = mode.into(); - match config { - Configuration::Managed { - managed: - Managed { - mode: ManagedMode::Launcher(launcher_config), - }, - } => { - assert_eq!( - launcher_config.bitcoind_addr, - Some(vec!["127.0.0.1:18444".to_string()]) - ); - assert_eq!(launcher_config.dogecoind_addr, None); - assert!(!launcher_config.ii); - assert!(!launcher_config.nns); - } - _ => panic!("expected ManagedMode::Launcher"), - } - } -} diff --git a/crates/icp/src/parsers.rs b/crates/icp/src/parsers.rs index b0a74730f..44cdff84f 100644 --- a/crates/icp/src/parsers.rs +++ b/crates/icp/src/parsers.rs @@ -1,643 +1,5 @@ -//! Parsing of token, cycle, memory, and duration amounts with support for suffixes and underscores. +//! Parsing of token, cycle, memory, and duration amounts. +//! +//! Defined in `icp_deploy_canister::parsers` and re-exported here. -use bigdecimal::{BigDecimal, Signed}; -use num_bigint::BigUint; -use num_integer::Integer; -use num_traits::{ToPrimitive, Zero}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::fmt; -use std::str::FromStr; - -/// Parse a token amount with support for suffixes (k, m, b, t) and underscores. -/// -/// Examples: -/// - `1` -> 1 -/// - `1_000` -> 1000 -/// - `1k` or `1K` -> 1000 -/// - `1t` or `1T` -> 1000000000000 -/// - `0.5` -> 0.5 -/// - `0.5k` -> 500 -pub fn parse_token_amount(input: &str) -> Result { - let input = input.trim(); - - if input.is_empty() { - return Err("Token amount cannot be empty".to_string()); - } - - let (number_part, multiplier) = if let Some(last_char) = input.chars().last() { - match last_char { - 'k' | 'K' => (&input[..input.len() - 1], 1_000u128), - 'm' | 'M' => (&input[..input.len() - 1], 1_000_000u128), - 'b' | 'B' => (&input[..input.len() - 1], 1_000_000_000u128), - 't' | 'T' => (&input[..input.len() - 1], 1_000_000_000_000u128), - _ => (input, 1u128), - } - } else { - (input, 1u128) - }; - - let cleaned = number_part.replace('_', ""); - let base = - BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid token amount: '{}'", input))?; - - if base.is_negative() { - return Err(format!("Token amount cannot be negative: '{}'", input)); - } - - let multiplier_decimal = BigDecimal::from(multiplier); - Ok(base * multiplier_decimal) -} - -/// Convert a token amount to the smallest unit by multiplying by 10^token_decimals. -/// E.g. 1.5 with 8 decimals -> 150000000. Fails if the result would be fractional. -pub fn to_token_unit_amount( - token_amount: BigDecimal, - token_decimals: u8, -) -> Result { - use num_bigint::BigInt; - use num_traits::pow::Pow; - - let (mantissa, exponent) = token_amount.into_bigint_and_exponent(); - let scale_adjustment = token_decimals as i64 - exponent; - let ten = BigInt::from(10); - - let result = if scale_adjustment >= 0 { - let multiplier = ten.pow(scale_adjustment as u32); - mantissa * multiplier - } else { - let divisor = ten.pow((-scale_adjustment) as u32); - let (quotient, remainder) = mantissa.div_rem(&divisor); - if !remainder.is_zero() { - return Err(format!( - "Token amount cannot be represented with {} decimals (would result in fractional units)", - token_decimals - )); - } - quotient - }; - - result - .try_into() - .map_err(|_| "Token amount cannot be negative".to_string()) -} - -fn parse_cycles_str(s: &str) -> Result { - let token_amount = parse_token_amount(s)?; - let unit_amount = to_token_unit_amount(token_amount, 0)?; - unit_amount - .to_u128() - .ok_or_else(|| format!("Cycles amount too large: '{}'", s)) -} - -/// An amount of cycles. -/// -/// Deserializes from a number or a string with suffixes (k, m, b, t) and optional underscore separators. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum CyclesAmount { - Number(u64), // yaml only supports up to u64 - Str(String), -} - -impl CyclesAmount { - pub fn get(&self) -> u128 { - match self { - CyclesAmount::Number(n) => *n as u128, - CyclesAmount::Str(s) => parse_cycles_str(s) - .unwrap_or_else(|e| panic!("invalid cycles amount '{}': {}", s, e)), - } - } -} - -impl<'de> Deserialize<'de> for CyclesAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - // Identical enum to CyclesAmount. Needed to avoid a circular dependency. - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom("cycles amount must be a number or a string with optional suffix (k, m, b, t), e.g. 1000 or \"4t\"") - })?; - let c = match v { - Raw::Number(n) => CyclesAmount::Number(n), - Raw::Str(ref s) => { - parse_cycles_str(s).map_err(serde::de::Error::custom)?; // validate the string is a valid cycles amount - CyclesAmount::Str(s.clone()) - } - }; - Ok(c) - } -} - -impl Serialize for CyclesAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - CyclesAmount::Number(n) => serializer.serialize_u64(*n), - CyclesAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for CyclesAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_cycles_str(s)?; // validate the string is a valid cycles amount - Ok(CyclesAmount::Str(s.to_string())) - } -} - -impl fmt::Display for CyclesAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u128 { - fn from(c: CyclesAmount) -> Self { - c.get() - } -} - -impl From for CyclesAmount { - fn from(n: u128) -> Self { - if let Ok(n64) = u64::try_from(n) { - CyclesAmount::Number(n64) - } else { - CyclesAmount::Str(n.to_string()) - } - } -} - -const KB: u64 = 1000; -const KIB: u64 = 1024; -const MB: u64 = 1_000_000; -const MIB: u64 = 1024 * 1024; -const GB: u64 = 1_000_000_000; -const GIB: u64 = 1024 * 1024 * 1024; - -fn parse_memory_str(s: &str) -> Result { - let s = s.trim(); - if s.is_empty() { - return Err("Memory amount cannot be empty".to_string()); - } - let lower = s.to_lowercase(); - let (number_part, factor) = if lower.ends_with("gib") { - (&s[..s.len() - 3], GIB) - } else if lower.ends_with("gb") { - (&s[..s.len() - 2], GB) - } else if lower.ends_with("mib") { - (&s[..s.len() - 3], MIB) - } else if lower.ends_with("mb") { - (&s[..s.len() - 2], MB) - } else if lower.ends_with("kib") { - (&s[..s.len() - 3], KIB) - } else if lower.ends_with("kb") { - (&s[..s.len() - 2], KB) - } else { - (s, 1u64) - }; - let cleaned = number_part.trim().replace('_', ""); - let amount = - BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid memory amount: '{}'", s))?; - if amount.is_negative() { - return Err(format!("Memory amount cannot be negative: '{}'", s)); - } - let product = amount * BigDecimal::from(factor); - if !product.is_integer() { - return Err( - "Memory amount must be a whole number of bytes (fractional bytes not allowed)" - .to_string(), - ); - } - product - .to_u64() - .ok_or_else(|| format!("Memory amount too large: '{}'", s)) -} - -/// An amount of memory in bytes. -/// -/// Deserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib), -/// optional decimals, and optional underscore separators. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum MemoryAmount { - Number(u64), - Str(String), -} - -impl MemoryAmount { - pub fn get(&self) -> u64 { - match self { - MemoryAmount::Number(n) => *n, - MemoryAmount::Str(s) => parse_memory_str(s) - .unwrap_or_else(|e| panic!("invalid memory amount '{}': {}", s, e)), - } - } -} - -impl<'de> Deserialize<'de> for MemoryAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom( - "memory amount must be a number or a string with optional suffix (kb, kib, mb, mib, gb, gib), e.g. 1024 or \"2.5gib\"", - ) - })?; - let m = match v { - Raw::Number(n) => MemoryAmount::Number(n), - Raw::Str(ref s) => { - parse_memory_str(s).map_err(serde::de::Error::custom)?; - MemoryAmount::Str(s.clone()) - } - }; - Ok(m) - } -} - -impl Serialize for MemoryAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - MemoryAmount::Number(n) => serializer.serialize_u64(*n), - MemoryAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for MemoryAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_memory_str(s)?; - Ok(MemoryAmount::Str(s.to_string())) - } -} - -impl fmt::Display for MemoryAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u64 { - fn from(m: MemoryAmount) -> Self { - m.get() - } -} - -impl From for MemoryAmount { - fn from(n: u64) -> Self { - MemoryAmount::Number(n) - } -} - -const SECONDS_PER_MINUTE: u64 = 60; -const SECONDS_PER_HOUR: u64 = 3600; -const SECONDS_PER_DAY: u64 = 86400; -const SECONDS_PER_WEEK: u64 = 604800; - -fn parse_duration_str(s: &str) -> Result { - let s = s.trim(); - if s.is_empty() { - return Err("Duration cannot be empty".to_string()); - } - let lower = s.to_lowercase(); - let (number_part, factor) = if lower.ends_with('w') { - (&s[..s.len() - 1], SECONDS_PER_WEEK) - } else if lower.ends_with('d') { - (&s[..s.len() - 1], SECONDS_PER_DAY) - } else if lower.ends_with('h') { - (&s[..s.len() - 1], SECONDS_PER_HOUR) - } else if lower.ends_with('m') { - (&s[..s.len() - 1], SECONDS_PER_MINUTE) - } else if lower.ends_with('s') { - (&s[..s.len() - 1], 1u64) - } else { - (s, 1u64) - }; - let cleaned = number_part.trim().replace('_', ""); - if cleaned.is_empty() { - return Err(format!("Invalid duration: '{s}'")); - } - let value: u64 = cleaned - .parse() - .map_err(|_| format!("Invalid duration: '{s}'"))?; - value - .checked_mul(factor) - .ok_or_else(|| format!("Duration too large: '{s}'")) -} - -/// A duration in seconds. -/// -/// Deserializes from a number (seconds) or a string with duration suffix (s, m, h, d, w) -/// and optional underscore separators. -/// -/// Suffixes (case-insensitive): -/// - `s` — seconds -/// - `m` — minutes (×60) -/// - `h` — hours (×3600) -/// - `d` — days (×86400) -/// - `w` — weeks (×604800) -/// -/// A bare number without suffix is treated as seconds. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum DurationAmount { - Number(u64), - Str(String), -} - -impl DurationAmount { - pub fn get(&self) -> u64 { - match self { - DurationAmount::Number(n) => *n, - DurationAmount::Str(s) => { - parse_duration_str(s).unwrap_or_else(|e| panic!("invalid duration '{}': {}", s, e)) - } - } - } -} - -impl<'de> Deserialize<'de> for DurationAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom( - "duration must be a number (seconds) or a string with optional suffix (s, m, h, d, w), e.g. 2592000 or \"30d\"", - ) - })?; - let c = match v { - Raw::Number(n) => DurationAmount::Number(n), - Raw::Str(ref s) => { - parse_duration_str(s).map_err(serde::de::Error::custom)?; - DurationAmount::Str(s.clone()) - } - }; - Ok(c) - } -} - -impl Serialize for DurationAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - DurationAmount::Number(n) => serializer.serialize_u64(*n), - DurationAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for DurationAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_duration_str(s)?; - Ok(DurationAmount::Str(s.to_string())) - } -} - -impl fmt::Display for DurationAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u64 { - fn from(d: DurationAmount) -> Self { - d.get() - } -} - -impl From for DurationAmount { - fn from(n: u64) -> Self { - DurationAmount::Number(n) - } -} - -impl PartialEq for DurationAmount { - fn eq(&self, other: &u64) -> bool { - self.get() == *other - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cycles_amount_from_str_plain() { - assert_eq!("1".parse::().unwrap().get(), 1); - assert_eq!("1000".parse::().unwrap().get(), 1000); - } - - #[test] - fn cycles_amount_from_str_suffixes() { - assert_eq!("1k".parse::().unwrap().get(), 1000); - assert_eq!( - "1t".parse::().unwrap().get(), - 1_000_000_000_000 - ); - assert_eq!( - "4t".parse::().unwrap().get(), - 4_000_000_000_000 - ); - assert_eq!( - "0.5t".parse::().unwrap().get(), - 500_000_000_000 - ); - } - - #[test] - fn cycles_amount_from_str_underscores() { - assert_eq!("1_000".parse::().unwrap().get(), 1000); - } - - #[test] - fn cycles_amount_from_str_fractional_rejected() { - assert!("1.5".parse::().is_err()); - } - - #[test] - fn cycles_amount_deserialize() { - let yaml = "4t"; - let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(c.get(), 4_000_000_000_000); - - let yaml = "5000000000000"; - let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(c.get(), 5_000_000_000_000); - } - - #[test] - fn parse_token_amount_plain_and_suffixes() { - use std::str::FromStr; - assert_eq!( - parse_token_amount("1").unwrap(), - BigDecimal::from_str("1").unwrap() - ); - assert_eq!( - parse_token_amount("1k").unwrap(), - BigDecimal::from_str("1000").unwrap() - ); - assert_eq!( - parse_token_amount("0.5t").unwrap(), - BigDecimal::from_str("500000000000").unwrap() - ); - } - - #[test] - fn memory_amount_from_str_plain() { - assert_eq!("1".parse::().unwrap().get(), 1); - assert_eq!("1024".parse::().unwrap().get(), 1024); - } - - #[test] - fn memory_amount_from_str_suffixes() { - assert_eq!("1kb".parse::().unwrap().get(), 1000); - assert_eq!("1kib".parse::().unwrap().get(), 1024); - assert_eq!("1mb".parse::().unwrap().get(), 1_000_000); - assert_eq!("1mib".parse::().unwrap().get(), 1024 * 1024); - assert_eq!("1gb".parse::().unwrap().get(), 1_000_000_000); - assert_eq!( - "1gib".parse::().unwrap().get(), - 1024 * 1024 * 1024 - ); - assert_eq!( - "2 GiB".parse::().unwrap().get(), - 2 * 1024 * 1024 * 1024 - ); - } - - #[test] - fn memory_amount_from_str_decimals() { - assert_eq!("0.5kib".parse::().unwrap().get(), 512); - assert_eq!("1.5gib".parse::().unwrap().get(), 1610612736); - } - - #[test] - fn memory_amount_fractional_bytes_rejected() { - assert!("1.5".parse::().is_err()); // 1.5 bytes - assert!("0.3kib".parse::().is_err()); // 307.2 bytes - } - - #[test] - fn memory_amount_from_str_underscores() { - assert_eq!("1_024".parse::().unwrap().get(), 1024); - } - - #[test] - fn memory_amount_deserialize() { - let yaml = "2gib"; - let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(m.get(), 2 * 1024 * 1024 * 1024); - - let yaml = "4294967296"; - let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(m.get(), 4294967296); - } - - #[test] - fn duration_amount_from_str_plain() { - assert_eq!("60".parse::().unwrap().get(), 60); - assert_eq!("2592000".parse::().unwrap().get(), 2592000); - } - - #[test] - fn duration_amount_from_str_underscores() { - assert_eq!( - "2_592_000".parse::().unwrap().get(), - 2592000 - ); - } - - #[test] - fn duration_amount_from_str_suffixes() { - assert_eq!("60s".parse::().unwrap().get(), 60); - assert_eq!("90m".parse::().unwrap().get(), 5400); - assert_eq!("24h".parse::().unwrap().get(), 86400); - assert_eq!("30d".parse::().unwrap().get(), 2592000); - assert_eq!("4w".parse::().unwrap().get(), 2419200); - } - - #[test] - fn duration_amount_from_str_case_insensitive() { - assert_eq!("30D".parse::().unwrap().get(), 2592000); - assert_eq!("1W".parse::().unwrap().get(), 604800); - assert_eq!("24H".parse::().unwrap().get(), 86400); - assert_eq!("60S".parse::().unwrap().get(), 60); - assert_eq!("90M".parse::().unwrap().get(), 5400); - } - - #[test] - fn duration_amount_from_str_underscores_with_suffix() { - assert_eq!( - "2_592_000s".parse::().unwrap().get(), - 2592000 - ); - } - - #[test] - fn duration_amount_from_str_errors() { - assert!("abc".parse::().is_err()); - assert!("".parse::().is_err()); - assert!("1x".parse::().is_err()); - assert!("1.5d".parse::().is_err()); - assert!("-1d".parse::().is_err()); - } - - #[test] - fn duration_amount_deserialize() { - let yaml = "30d"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - - let yaml = "2592000"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - - let yaml = "2_592_000"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - } - - #[test] - fn duration_amount_partial_eq_u64() { - let d = DurationAmount::Number(2592000); - assert!(d == 2592000); - assert!(d != 0); - - let d = DurationAmount::Str("30d".to_string()); - assert!(d == 2592000); - } -} +pub use icp_deploy_canister::parsers::*; diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index 1f5520def..4dda9dc90 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -1,520 +1,32 @@ -use std::collections::{BTreeMap, HashMap, HashSet, hash_map::Entry}; +//! Host-side project facade. +//! +//! Consolidation of manifests into a [`Project`] lives in +//! `icp_deploy_canister::project` (over an injected `FileAccess`) and is +//! re-exported here. Walking a workspace's dependency edges on disk and +//! member-scoping both resolve real filesystem paths against the current working +//! directory, so they stay here. -use indexmap::{IndexMap, map::Entry as IndexEntry}; +use std::collections::HashSet; use snafu::prelude::*; +pub use icp_deploy_canister::project::{ + ConsolidateManifestError, EnvironmentError, LoadProjectError, VerifySandboxError, + consolidate_manifest, load_project, relative_prefix, verify_sandbox, +}; + use crate::{ - Canister, Environment, InitArgs, Network, Project, - canister::{ControllerRef, ManifestEnvVar, ManifestSettings, Settings, recipe}, - fs, + Environment, manifest::{ - ArgsFormat, CANISTER_MANIFEST, CanisterManifest, DependencyManifest, EnvironmentManifest, - Item, LoadManifestFromPathError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, - ProjectManifest, ProjectRootLocateError, - canister::{Instructions, SyncSteps}, - environment::CanisterSelection, - load_manifest_from_path, - network::RootKeySpec, - recipe::RecipeType, - }, - network::{ - Configuration, Connected, Gateway, Managed, ManagedLauncherConfig, ManagedMode, Port, + LoadManifestFromPathError, PROJECT_MANIFEST, ProjectManifest, load_manifest_from_path, }, prelude::*, }; -pub const DEFAULT_LOCAL_NETWORK_BIND: &str = "127.0.0.1"; -pub const DEFAULT_LOCAL_NETWORK_PORT: u16 = 8000; - -#[derive(Debug, Snafu)] -pub enum EnvironmentError { - #[snafu(display("environment '{environment}' points to invalid network '{network}'"))] - InvalidNetwork { - environment: String, - network: String, - }, - - #[snafu(display("environment '{environment}' points to invalid canister '{canister}'"))] - InvalidCanister { - environment: String, - canister: String, - }, -} - -#[derive(Debug, Snafu)] -pub enum ConsolidateManifestError { - #[snafu(display("failed to locate project directory"))] - Locate { source: ProjectRootLocateError }, - - #[snafu(display("failed to perform glob parsing"))] - GlobParse { source: glob::PatternError }, - - #[snafu(display("failed to get glob iter"))] - GlobIter { source: glob::GlobError }, - - #[snafu(display("failed to convert path to UTF-8"))] - Utf8Path { source: FromPathBufError }, - - #[snafu(display("failed to load canister manifest"))] - LoadCanister { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load network manifest"))] - LoadNetwork { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load environment manifest"))] - LoadEnvironment { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load {kind} manifest at: {path}"))] - Failed { kind: String, path: String }, - - #[snafu(display("failed to fetch canister recipe: {recipe_type:?}"))] - FetchRecipe { - #[snafu(source(from(recipe::ResolveError, Box::new)))] - source: Box, - recipe_type: RecipeType, - }, - - #[snafu(display("failed to render canister recipe: {recipe_type:?}"))] - RenderRecipe { - #[snafu(source(from(recipe::RenderRecipeError, Box::new)))] - source: Box, - recipe_type: RecipeType, - }, - - #[snafu(display("failed to cache canister recipe: {recipe_type:?}"))] - CacheRecipe { - #[snafu(source(from(recipe::ResolveError, Box::new)))] - source: Box, - recipe_type: RecipeType, - }, - - #[snafu(display("project contains two similarly named {kind}s: '{name}'"))] - Duplicate { kind: String, name: String }, - - #[snafu(display("`{name}` is a reserved {kind} name."))] - Reserved { kind: String, name: String }, - - #[snafu(display("could not locate a {kind} manifest at: '{path}'"))] - NotFound { kind: String, path: String }, - - #[snafu(display("failed to read init_args file for canister '{canister}'"))] - ReadInitArgs { - source: fs::IoError, - canister: String, - }, - - #[snafu(display( - "failed to read the file backing environment variable '{variable}' of canister '{canister}'" - ))] - ReadEnvironmentVariable { - source: fs::IoError, - canister: String, - variable: String, - }, - - #[snafu(display( - "init_args for canister '{canister}' uses format 'bin' with inline content; \ - binary format requires a file path" - ))] - BinFormatInlineContent { canister: String }, - - #[snafu(display( - "canister '{canister}' lists controller '{controller}', but no canister with that \ - name is declared in the project" - ))] - UnknownControllerCanister { - canister: String, - controller: String, - }, - - #[snafu(display( - "canister name '{name}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ - (':' is reserved as the dependency namespace separator)" - ))] - InvalidCanisterName { name: String }, - - #[snafu(display( - "dependency alias '{alias}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ - (':' is reserved as the dependency namespace separator)" - ))] - InvalidDependencyAlias { alias: String }, - - #[snafu(display("project declares two dependencies with the same alias '{alias}'"))] - DuplicateDependencyAlias { alias: String }, - - #[snafu(display( - "dependency alias '{alias}' collides with a canister of the same name in the same project" - ))] - DependencyAliasCollision { alias: String }, - - #[snafu(display("could not find a project manifest for dependency '{alias}' at: '{path}'"))] - DependencyNotFound { alias: String, path: String }, - - #[snafu(display("failed to canonicalize path for dependency '{alias}' at: '{path}'"))] - DependencyCanonicalize { alias: String, path: String }, - - #[snafu(display("failed to load project manifest for dependency '{alias}'"))] - LoadDependencyManifest { - source: LoadManifestFromPathError, - alias: String, - }, - - #[snafu(display( - "dependency '{alias}' selects canister '{canister}', which the dependency does not declare" - ))] - UnknownDependencyCanister { alias: String, canister: String }, - - #[snafu(display("dependency cycle detected: {chain}"))] - CircularDependency { chain: String }, - - #[snafu(transparent)] - Environment { source: EnvironmentError }, -} - -/// Resolve a [`ManifestInitArgs`] into a canonical [`InitArgs`] by reading -/// any file references relative to `base_path`. -fn resolve_manifest_init_args( - manifest_init_args: &ManifestInitArgs, - base_path: &Path, - canister: &str, -) -> Result { - match manifest_init_args { - ManifestInitArgs::String(content) => Ok(InitArgs::Text { - content: content.trim().to_owned(), - format: ArgsFormat::Candid, - }), - ManifestInitArgs::Path { path, format } => { - let file_path = base_path.join(path); - match format { - ArgsFormat::Bin => { - let bytes = fs::read(&file_path).context(ReadInitArgsSnafu { canister })?; - Ok(InitArgs::Binary(bytes)) - } - fmt => { - let content = - fs::read_to_string(&file_path).context(ReadInitArgsSnafu { canister })?; - Ok(InitArgs::Text { - content: content.trim().to_owned(), - format: fmt.clone(), - }) - } - } - } - ManifestInitArgs::Value { value, format } => match format { - ArgsFormat::Bin => BinFormatInlineContentSnafu { canister }.fail(), - fmt => Ok(InitArgs::Text { - content: value.trim().to_owned(), - format: fmt.clone(), - }), - }, - } -} - -/// Resolve a manifest's [`ManifestSettings`] into the model's [`Settings`] by -/// reading any file-backed environment variable values relative to `base_path`. -/// Also returns the file each such value came from, for -/// [`Canister::environment_variable_files`]. -fn resolve_manifest_settings( - manifest_settings: &ManifestSettings, - base_path: &Path, - canister: &str, -) -> Result<(Settings, BTreeMap), ConsolidateManifestError> { - let ManifestSettings { - log_visibility, - compute_allocation, - memory_allocation, - freezing_threshold, - reserved_cycles_limit, - wasm_memory_limit, - wasm_memory_threshold, - log_memory_limit, - environment_variables, - controllers, - } = manifest_settings; - - let mut files = BTreeMap::new(); - let environment_variables = environment_variables - .as_ref() - .map(|vars| { - vars.iter() - .map(|(name, var)| { - let value = match var { - ManifestEnvVar::Value(value) => value.to_owned(), - ManifestEnvVar::Path { path } => { - let file = base_path.join(path); - let contents = fs::read_to_string(&file).context( - ReadEnvironmentVariableSnafu { - canister, - variable: name, - }, - )?; - files.insert(name.to_owned(), file); - contents.trim().to_owned() - } - }; - Ok((name.to_owned(), value)) - }) - .collect::, ConsolidateManifestError>>() - }) - .transpose()?; - - let settings = Settings { - log_visibility: log_visibility.clone(), - compute_allocation: *compute_allocation, - memory_allocation: memory_allocation.clone(), - freezing_threshold: freezing_threshold.clone(), - reserved_cycles_limit: reserved_cycles_limit.clone(), - wasm_memory_limit: wasm_memory_limit.clone(), - wasm_memory_threshold: wasm_memory_threshold.clone(), - log_memory_limit: log_memory_limit.clone(), - environment_variables, - controllers: controllers.clone(), - }; - Ok((settings, files)) -} - -fn is_glob(s: &str) -> bool { - s.contains('*') || s.contains('?') || s.contains('[') || s.contains('{') -} - -/// Whether `name` is a valid canister name or dependency alias: non-empty and -/// containing only ASCII letters, digits, `_`, or `-`. -/// -/// A single strict rule keeps names safe for every purpose they are reused for — -/// store-key segments, `PUBLIC_CANISTER_ID:` env vars, DNS subdomains, and -/// archive paths — so no per-site sanitizing is needed. In particular `:` is the -/// dependency namespace separator, and `.` / `/` would be ambiguous in -/// subdomains and paths. -fn is_valid_name(name: &str) -> bool { - !name.is_empty() - && name - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') -} - -/// Builds the canonical canisters declared directly in one project manifest, -/// resolving glob/path/inline entries, recipes, and init-args relative to -/// `pdir`. Returns `(local name, canister dir, canister)` with empty bindings; -/// callers assign store keys and bindings. Does not check for duplicate names -/// across projects — that is the caller's responsibility (via the global map). -async fn build_manifest_canisters( - pdir: &Path, - manifest_canisters: &[Item], - recipe_resolver: &dyn recipe::Resolve, -) -> Result, ConsolidateManifestError> { - let mut result: Vec<(String, PathBuf, Canister)> = Vec::new(); - - for i in manifest_canisters { - let ms = match i { - Item::Path(pattern) => { - let is_glob_pattern = is_glob(pattern); - let paths = match is_glob_pattern { - // Explicit path - false => vec![pdir.join(pattern)], - - // Glob pattern - true => { - let paths = - glob::glob(pdir.join(pattern).as_str()).context(GlobParseSnafu)?; - - let mut v = vec![]; - for p in paths { - let path = p.context(GlobIterSnafu)?; - let utf8_path = PathBuf::try_from(path).context(Utf8PathSnafu)?; - v.push(utf8_path); - } - v - } - }; - - let paths = if is_glob_pattern { - // For glob patterns, filter out non-directories and non-canister directories - paths - .into_iter() - .filter(|p| p.is_dir()) - .filter(|p| p.join(CANISTER_MANIFEST).exists()) - .collect::>() - } else { - // For explicit paths, validate that they exist and contain canister.yaml - let mut validated_paths = vec![]; - for p in paths { - if !p.join(CANISTER_MANIFEST).is_file() { - return NotFoundSnafu { - kind: "canister".to_string(), - path: pattern.to_string(), - } - .fail(); - } - validated_paths.push(p); - } - validated_paths - }; - - let mut ms = vec![]; - for p in paths { - ms.push(( - p.to_owned(), - load_manifest_from_path::(&p.join(CANISTER_MANIFEST)) - .await - .context(LoadCanisterSnafu)?, - )); - } - ms - } - - Item::Manifest(m) => vec![(pdir.to_owned(), m.to_owned())], - }; - - for (cdir, m) in ms { - if !is_valid_name(&m.name) { - return InvalidCanisterNameSnafu { - name: m.name.clone(), - } - .fail(); - } - - let registry_recipe = match &m.instructions { - Instructions::BuildSync { .. } => None, - Instructions::Recipe { recipe } => match &recipe.recipe_type { - RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), - _ => None, - }, - }; - - let (build, sync) = match &m.instructions { - // Build/Sync - Instructions::BuildSync { build, sync } => ( - build.to_owned(), - match sync { - Some(sync) => sync.to_owned(), - None => SyncSteps::default(), - }, - ), - - // Recipe - Instructions::Recipe { recipe } => { - let fetched = - recipe_resolver - .resolve(recipe) - .await - .context(FetchRecipeSnafu { - recipe_type: recipe.recipe_type.clone(), - })?; - let ctx = recipe::RecipeContext { - canister_name: m.name.clone(), - }; - let steps = recipe::render_recipe(&fetched.template, recipe, &ctx).context( - RenderRecipeSnafu { - recipe_type: recipe.recipe_type.clone(), - }, - )?; - - // The template rendered, so an unpinned download is now known - // good and safe to cache. Committing only here is what keeps a - // bad remote response from becoming sticky. - if let Some(pending) = fetched.pending_cache { - recipe_resolver - .commit(pending) - .await - .context(CacheRecipeSnafu { - recipe_type: recipe.recipe_type.clone(), - })?; - } - - steps - } - }; - - let (settings, environment_variable_files) = - resolve_manifest_settings(&m.settings, &cdir, &m.name)?; - - let init_args = m - .init_args - .as_ref() - .map(|mia| resolve_manifest_init_args(mia, &cdir, &m.name)) - .transpose()?; - - result.push(( - m.name.clone(), - cdir, - Canister { - name: m.name.clone(), - settings, - build, - sync, - init_args, - registry_recipe, - bindings: BTreeMap::new(), - // Default to the bare local name; overwritten with the - // dot-nested alias form when the canister is imported as a - // dependency (see `import_dependency`). - friendly_names: vec![m.name.clone()], - environment_variable_files, - }, - )); - } - } - - Ok(result) -} - -/// A dependency instance imported into the workspace. Returned by -/// [`import_dependency`] and cached per canonical path so diamond dependencies -/// reuse the same instance. -#[derive(Clone)] -struct ImportedInstance { - /// This instance's own canisters, as `(local name, full store key)` — the - /// set exposable to the parent via `canisters:` selection. - own: Vec<(String, String)>, - /// Every canister in this instance's subtree (its own canisters plus all - /// transitively imported ones), as `(store key, local name, alias chain - /// from this instance down to the canister's owning project)`. Used to - /// register a friendly URL per alias chain when the instance is reached - /// again via de-duplication (a diamond), including for its descendants. - subtree: Vec<(String, String, Vec)>, -} - -/// A member environment's per-canister config, to be folded into the root's -/// same-named environment beneath any root overrides. -#[derive(Default, Clone)] -struct MemberCanisterOverride { - settings: Option, - init_args: Option, -} - -/// Per-environment member overrides: env name → store key → override. -type MemberEnvOverrides = HashMap>; - -/// A member's identity (store-key prefix) and the environment names it defines, -/// used to enforce that a member declares every environment the root targets -/// (strict rule). -struct MemberEnvInfo { - prefix: String, - defined: HashSet, -} - -/// Canonicalize a dependency root (resolving symlinks and `..`) for use as a -/// de-dup / cycle-detection identity. -fn canonicalize_dep(alias: &str, dep_root: &Path) -> Result { - let build_err = || { - DependencyCanonicalizeSnafu { - alias: alias.to_owned(), - path: dep_root.to_string(), - } - .build() - }; - let canon = dunce::canonicalize(dep_root.as_std_path()).map_err(|_| build_err())?; - PathBuf::try_from(canon).map_err(|_| build_err()) -} - -/// Store-key prefix for a dependency instance: its canonical directory relative -/// to the canonical app root, forward-slash separated so keys are stable across -/// platforms and independent of how each edge spells the path. -fn relative_prefix(app_root_canonical: &Path, dep_canonical: &Path) -> String { - let rel = pathdiff::diff_utf8_paths(dep_canonical, app_root_canonical) - .unwrap_or_else(|| dep_canonical.to_owned()); - rel.as_str().replace('\\', "/") +/// Canonicalize into a UTF-8 path, or `None` if it does not exist / is not UTF-8. +fn canonicalize_or(dir: &Path) -> Option { + let canon = dunce::canonicalize(dir.as_std_path()).ok()?; + PathBuf::try_from(canon).ok() } /// One project in a workspace: the root project, or a dependency instance @@ -617,7 +129,7 @@ fn resolve_edges( /// every dependency instance reachable from it, in depth-first declaration /// order. /// -/// This retraces the same edges [`import_dependency`] follows — de-duplicating +/// This retraces the same edges `import_dependency` follows — de-duplicating /// instances by canonical directory, so a diamond yields one entry whose /// `prefix` matches the store-key prefix its canisters were assigned — but keeps /// each instance's *raw* manifest instead of folding its canisters into the @@ -685,348 +197,6 @@ pub async fn workspace_instances( Ok(out) } -/// Build a dependency canister's friendly-URL subdomain prefix: the canister's -/// local name as the most-specific label, followed by its alias chain reversed -/// (root-most alias last). E.g. local `backend` reached via `[service-a, -/// openemail]` → `backend.openemail.service-a`. Dot-nested so it stays a valid, -/// collision-free multi-label host; see DESIGN §17.2. -fn friendly_name_for(local: &str, alias_chain: &[String]) -> String { - let mut labels = Vec::with_capacity(alias_chain.len() + 1); - labels.push(local.to_string()); - labels.extend(alias_chain.iter().rev().cloned()); - labels.join(".") -} - -/// Rewrite `CanisterName` controller references from a dependency's local -/// canister names to their store keys, so global controller validation and -/// deploy-time id lookup operate uniformly on store keys. -fn translate_controllers(canister: &mut Canister, local_to_key: &BTreeMap) { - translate_settings_controllers(&mut canister.settings, local_to_key); -} - -/// Rewrite `CanisterName` controller references in a `Settings` from a -/// dependency's local canister names to their store keys. -fn translate_settings_controllers( - settings: &mut Settings, - local_to_key: &BTreeMap, -) { - if let Some(controllers) = &mut settings.controllers { - for cref in controllers.iter_mut() { - if let ControllerRef::CanisterName(name) = cref - && let Some(key) = local_to_key.get(name) - { - *name = key.clone(); - } - } - } -} - -/// Compute the `PUBLIC_CANISTER_ID` env-var wiring for canisters in one project -/// scope: its own canisters by local name, plus each dependency's exposed -/// canisters under `:`. -fn compute_bindings( - own: &[(String, String)], - edges: &[(String, Vec<(String, String)>)], -) -> BTreeMap { - let mut bindings = BTreeMap::new(); - for (local, key) in own { - bindings.insert(local.clone(), key.clone()); - } - for (alias, exposed) in edges { - for (dep_local, key) in exposed { - bindings.insert(format!("{alias}:{dep_local}"), key.clone()); - } - } - bindings -} - -/// Select which of a dependency instance's own canisters are exposed to the -/// parent, per the dependency's `canisters` selection. -fn select_exposed( - own: &[(String, String)], - selection: &CanisterSelection, - alias: &str, -) -> Result, ConsolidateManifestError> { - match selection { - CanisterSelection::Everything => Ok(own.to_vec()), - CanisterSelection::None => Ok(vec![]), - CanisterSelection::Named(names) => { - let mut out = Vec::new(); - for name in names { - match own.iter().find(|(local, _)| local == name) { - Some(pair) => out.push(pair.clone()), - None => { - return UnknownDependencyCanisterSnafu { - alias: alias.to_owned(), - canister: name.clone(), - } - .fail(); - } - } - } - Ok(out) - } - } -} - -/// Validate the dependency aliases declared in one project scope: no `:`, no -/// collision with a local canister name, and no duplicate alias. -fn validate_dependency_aliases( - deps: &[DependencyManifest], - own_canister_names: &HashSet, -) -> Result<(), ConsolidateManifestError> { - let mut seen: HashSet<&str> = HashSet::new(); - for d in deps { - if !is_valid_name(&d.name) { - return InvalidDependencyAliasSnafu { - alias: d.name.clone(), - } - .fail(); - } - if own_canister_names.contains(&d.name) { - return DependencyAliasCollisionSnafu { - alias: d.name.clone(), - } - .fail(); - } - if !seen.insert(&d.name) { - return DuplicateDependencyAliasSnafu { - alias: d.name.clone(), - } - .fail(); - } - } - Ok(()) -} - -/// Recursively import a dependency's canisters into `canisters`, keyed by their -/// app-root-relative store keys. De-duplicates instances by canonical path -/// (diamond dependencies deploy once) and detects cycles. Returns the imported -/// instance's prefix and its own canisters. -#[allow(clippy::too_many_arguments)] -async fn import_dependency( - app_root_canonical: &Path, - parent_dir: &Path, - dep: &DependencyManifest, - recipe_resolver: &dyn recipe::Resolve, - canisters: &mut IndexMap, - registry: &mut HashMap, - stack: &mut Vec, - member_env_overrides: &mut MemberEnvOverrides, - members: &mut Vec, - // Alias chain from the workspace root to and including this dependency, - // used to build friendly-URL subdomains (§17.2). - alias_chain: &[String], -) -> Result { - let dep_root = parent_dir.join(&dep.path); - let manifest_path = dep_root.join(PROJECT_MANIFEST); - if !manifest_path.is_file() { - return DependencyNotFoundSnafu { - alias: dep.name.clone(), - path: dep_root.to_string(), - } - .fail(); - } - - let canonical = canonicalize_dep(&dep.name, &dep_root)?; - - // Cycle detection. - if stack.contains(&canonical) { - let mut chain: Vec = stack.iter().map(|p| p.to_string()).collect(); - chain.push(canonical.to_string()); - return CircularDependencySnafu { - chain: chain.join(" -> "), - } - .fail(); - } - - // Diamond de-dup: same resolved directory means the same instance, deployed - // once. It is still reachable via this new alias chain, so register an - // additional friendly URL per chain (§17.3) rather than picking one — for - // the whole subtree (its own canisters *and* its transitive dependencies), - // each named by this chain extended with the canister's alias path below the - // instance. - if let Some(inst) = registry.get(&canonical) { - let inst = inst.clone(); - for (key, local, rel_chain) in &inst.subtree { - let mut chain = alias_chain.to_vec(); - chain.extend(rel_chain.iter().cloned()); - let fname = friendly_name_for(local, &chain); - if let Some((_, canister)) = canisters.get_mut(key) - && !canister.friendly_names.contains(&fname) - { - canister.friendly_names.push(fname); - } - } - return Ok(inst); - } - - stack.push(canonical.clone()); - - let prefix = relative_prefix(app_root_canonical, &canonical); - - let dep_manifest: ProjectManifest = - load_manifest_from_path(&manifest_path) - .await - .context(LoadDependencyManifestSnafu { - alias: dep.name.clone(), - })?; - - // Build the dependency's own canisters and key them under the prefix. All of - // them are imported (deploy-all); the `canisters` exposure subset is applied - // by the caller when wiring env vars. - let built = - build_manifest_canisters(&dep_root, &dep_manifest.canisters, recipe_resolver).await?; - - let mut own: Vec<(String, String)> = Vec::new(); - let mut local_to_key: BTreeMap = BTreeMap::new(); - for (local, cdir, mut canister) in built { - let store_key = format!("{prefix}:{local}"); - canister.name = store_key.clone(); - // Friendly URL from the alias chain, not the path-based store key. - canister.friendly_names = vec![friendly_name_for(&local, alias_chain)]; - own.push((local.clone(), store_key.clone())); - local_to_key.insert(local.clone(), store_key.clone()); - match canisters.entry(store_key.clone()) { - IndexEntry::Occupied(_) => { - return DuplicateSnafu { - kind: "canister".to_string(), - name: store_key, - } - .fail(); - } - IndexEntry::Vacant(e) => { - e.insert((cdir, canister)); - } - } - } - - // Now that every sibling's store key is known, translate the dependency's - // controller references (local sibling name -> store key). - for (_, key) in &own { - if let Some((_, canister)) = canisters.get_mut(key) { - translate_controllers(canister, &local_to_key); - } - } - - // Capture the member's own environments so the parent can honor its - // per-canister settings/init_args for the same-named environment - // (standalone-equivalence). The network binding and canister selection are - // ignored; only overrides on the member's *own* canisters are - // folded in — keys naming its dependencies are left to those dependencies. - let mut defined_envs: HashSet = HashSet::new(); - for env_item in &dep_manifest.environments { - let em: EnvironmentManifest = match env_item { - Item::Manifest(m) => m.clone(), - Item::Path(path) => { - let p = dep_root.join(path); - if !p.is_file() { - return NotFoundSnafu { - kind: "environment".to_string(), - path: p.to_string(), - } - .fail(); - } - load_manifest_from_path::(&p) - .await - .context(LoadEnvironmentSnafu)? - } - }; - defined_envs.insert(em.name.clone()); - if let Some(settings) = &em.settings { - for (local, s) in settings { - if let Some(key) = local_to_key.get(local) { - // Translate the override's own controller references from the - // member's local names to store keys, so name-based controllers - // resolve against the workspace id map just like base settings. - let mut s = s.clone(); - translate_settings_controllers(&mut s, &local_to_key); - member_env_overrides - .entry(em.name.clone()) - .or_default() - .entry(key.clone()) - .or_default() - .settings = Some(s); - } - } - } - if let Some(init_args) = &em.init_args { - for (local, ia) in init_args { - if let Some(key) = local_to_key.get(local) { - member_env_overrides - .entry(em.name.clone()) - .or_default() - .entry(key.clone()) - .or_default() - .init_args = Some(ia.clone()); - } - } - } - } - members.push(MemberEnvInfo { - prefix: prefix.clone(), - defined: defined_envs, - }); - - // Recurse into the dependency's own dependencies. - let own_names: HashSet = own.iter().map(|(l, _)| l.clone()).collect(); - validate_dependency_aliases(&dep_manifest.dependencies, &own_names)?; - - // The instance's subtree, for diamond-hit friendly-URL propagation: its own - // canisters sit at the instance root (empty relative alias chain); each - // nested dependency contributes its subtree prefixed with the nested alias. - let mut subtree: Vec<(String, String, Vec)> = own - .iter() - .map(|(local, key)| (key.clone(), local.clone(), Vec::new())) - .collect(); - - let mut edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for nested in &dep_manifest.dependencies { - let mut nested_chain = alias_chain.to_vec(); - nested_chain.push(nested.name.clone()); - let inst = Box::pin(import_dependency( - app_root_canonical, - &dep_root, - nested, - recipe_resolver, - canisters, - registry, - stack, - member_env_overrides, - members, - &nested_chain, - )) - .await?; - for (key, local, rel) in &inst.subtree { - let mut r = Vec::with_capacity(rel.len() + 1); - r.push(nested.name.clone()); - r.extend(rel.iter().cloned()); - subtree.push((key.clone(), local.clone(), r)); - } - let exposed = select_exposed(&inst.own, &nested.canisters, &nested.name)?; - edges.push((nested.name.clone(), exposed)); - } - - // Assign env-var bindings for this instance's own canisters. - let bindings = compute_bindings(&own, &edges); - for (_, key) in &own { - if let Some((_, canister)) = canisters.get_mut(key) { - canister.bindings = bindings.clone(); - } - } - - stack.pop(); - let instance = ImportedInstance { own, subtree }; - registry.insert(canonical, instance.clone()); - Ok(instance) -} - -/// Canonicalize into a UTF-8 path, or `None` if it does not exist / is not UTF-8. -fn canonicalize_or(dir: &Path) -> Option { - let canon = dunce::canonicalize(dir.as_std_path()).ok()?; - PathBuf::try_from(canon).ok() -} - /// The default set of target canisters when the user names none, honoring /// member-scoping. /// @@ -1063,587 +233,98 @@ pub fn member_scoped_canisters( Some(names) } -/// Build one environment's canister map: select from `canisters`, then apply the -/// member overrides for this environment (standalone-equivalence), then -/// the root's own overrides (highest precedence). Precedence is therefore -/// root-explicit > member-env > canister-base. -fn build_environment_canisters( - canisters: &IndexMap, - env_name: &str, - selection: &CanisterSelection, - member_overrides: Option<&HashMap>, - root_settings: Option<&HashMap>, - root_init_args: Option<&HashMap>, -) -> Result, ConsolidateManifestError> { - let mut cs = match selection { - CanisterSelection::None => IndexMap::new(), - CanisterSelection::Everything => canisters.clone(), - CanisterSelection::Named(names) => { - let mut cs: IndexMap = IndexMap::new(); - for name in names { - let v = canisters.get(name).ok_or( - InvalidCanisterSnafu { - environment: env_name.to_owned(), - canister: name.to_owned(), - } - .build(), - )?; - cs.insert(name.to_owned(), v.to_owned()); - } - cs - } - }; +#[cfg(test)] +mod tests { + use super::*; + use crate::canister::recipe::{FetchedRecipe, RemoteResourceResolve, ResolveError}; + use crate::host_files::HostFileAccess; + use crate::manifest::adapter::prebuilt::SourceField; + use crate::manifest::recipe::Recipe; + use crate::manifest::{PROJECT_MANIFEST, ProjectManifest, load_manifest_from_path}; + use crate::prelude::LOCAL; + use camino_tempfile::Utf8TempDir; + use icp_deploy_canister::sync_exec::StepProgress; - // Member overrides first (lower precedence than the root's own overrides). - if let Some(overrides) = member_overrides { - for (key, ov) in overrides { - if let Some((cpath, canister)) = cs.get_mut(key) { - if let Some(s) = &ov.settings { - (canister.settings, canister.environment_variable_files) = - resolve_manifest_settings(s, cpath, key)?; - } - if let Some(ia) = &ov.init_args { - canister.init_args = Some(resolve_manifest_init_args(ia, cpath, key)?); - } - } - } - } + /// Recipes and plugins are never used in this test; every canister is pre-built. + struct PanicResolver; - // Root overrides last (highest precedence). - if let Some(settings) = root_settings { - for (name, s) in settings { - if let Some((cpath, canister)) = cs.get_mut(name) { - (canister.settings, canister.environment_variable_files) = - resolve_manifest_settings(s, cpath, name)?; - } - } - } - if let Some(init_args) = root_init_args { - for (name, ia) in init_args { - if let Some((cpath, canister)) = cs.get_mut(name) { - canister.init_args = Some(resolve_manifest_init_args(ia, cpath, name)?); - } + #[async_trait::async_trait] + impl RemoteResourceResolve for PanicResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + panic!("recipe resolver should not be called in this test"); } - } - - Ok(cs) -} - -/// Turns the ProjectManifest into a Project struct -/// - Adds the default Networks -/// - Adds the default Environment -/// - Imports any dependency projects' canisters -/// - Validates the manifest to make sure that: -/// - There are no duplicates -/// - All the environments have networks -/// - All the referenced canisters exist -/// - All the recipes have been resolved -pub async fn consolidate_manifest( - pdir: &Path, - recipe_resolver: &dyn recipe::Resolve, - m: &ProjectManifest, -) -> Result { - // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved - // through to consumers like `icp project bundle`, which needs reproducible output. - let mut canisters: IndexMap = IndexMap::new(); - - // Canonical app root, used to derive stable, order-independent store-key - // prefixes for imported dependency canisters. - let app_root_canonical = - canonicalize_dep("", pdir).unwrap_or_else(|_| pdir.to_owned()); - // This project's own canisters, keyed by their bare local names. - let app_built = build_manifest_canisters(pdir, &m.canisters, recipe_resolver).await?; - let mut app_own: Vec<(String, String)> = Vec::new(); - for (local, cdir, canister) in app_built { - app_own.push((local.clone(), local.clone())); - match canisters.entry(local.clone()) { - IndexEntry::Occupied(e) => { - return DuplicateSnafu { - kind: "canister".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - IndexEntry::Vacant(e) => { - e.insert((cdir, canister)); - } + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + panic!("recipe resolver should not be called in this test"); } - } - - // Import dependency projects. Each dependency is deployed in full and keyed - // under its app-root-relative path; diamonds (the same directory reached via - // multiple edges) resolve to a single instance. - let mut registry: HashMap = HashMap::new(); - let mut stack: Vec = Vec::new(); - // Member environment config folded into the root's same-named environments, - // and the per-member set of declared environment names for the strict rule. - let mut member_env_overrides: MemberEnvOverrides = HashMap::new(); - let mut members: Vec = Vec::new(); - let app_own_names: HashSet = app_own.iter().map(|(l, _)| l.clone()).collect(); - validate_dependency_aliases(&m.dependencies, &app_own_names)?; - let mut app_edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for dep in &m.dependencies { - let inst = import_dependency( - &app_root_canonical, - pdir, - dep, - recipe_resolver, - &mut canisters, - &mut registry, - &mut stack, - &mut member_env_overrides, - &mut members, - std::slice::from_ref(&dep.name), - ) - .await?; - let exposed = select_exposed(&inst.own, &dep.canisters, &dep.name)?; - app_edges.push((dep.name.clone(), exposed)); - } - - // Assign env-var bindings for this project's own canisters (own canisters by - // local name plus each dependency's exposed canisters under `:`). - let app_bindings = compute_bindings(&app_own, &app_edges); - for (_, key) in &app_own { - if let Some((_, canister)) = canisters.get_mut(key) { - canister.bindings = app_bindings.clone(); + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _progress: Option<&dyn StepProgress>, + ) -> Result { + panic!("wasm resolver should not be called in this test"); } } - // Friendly URLs need no de-collision pass: the strict name rule (no '.') makes - // own canisters single-label and dependency canisters multi-label (dot-nested - // by alias chain), so their hostnames are disjoint by construction (§17.2). + fn write(dir: &Path, rel: &str, contents: &str) { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, contents).unwrap(); + } - // Validate that every canister-name controller reference points to a declared canister. - // Catching typos here turns "perpetual warning" into a clear load-time error. - for (canister_name, (_, canister)) in &canisters { - let Some(crefs) = &canister.settings.controllers else { - continue; - }; - for cref in crefs { - if let Some(ref_name) = cref.canister_name() - && !canisters.contains_key(ref_name) - { - return UnknownControllerCanisterSnafu { - canister: canister_name.to_owned(), - controller: ref_name.to_owned(), - } - .fail(); + fn manifest(canisters: &[&str], deps: &str) -> String { + let mut s = String::new(); + if canisters.is_empty() { + s.push_str("canisters: []\n"); + } else { + s.push_str("canisters:\n"); + for c in canisters { + s.push_str(&format!( + " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" + )); } } + s.push_str(deps); + s } - // Networks - let mut networks: HashMap = HashMap::new(); + #[tokio::test] + async fn member_scope_targets_only_the_members_canisters() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); - // Add IC network first - this is always protected and non-overridable - networks.insert( - IC.to_string(), - Network { - name: IC.to_string(), - configuration: Configuration::Connected { - connected: Connected { - api_url: IC_MAINNET_NETWORK_API_URL.parse().unwrap(), - http_gateway_url: Some(IC_MAINNET_NETWORK_GATEWAY_URL.parse().unwrap()), - root_key: RootKeySpec::Mainnet, - }, - }, - }, - ); + let m: ProjectManifest = load_manifest_from_path(&tmp.path().join(PROJECT_MANIFEST)) + .await + .unwrap(); + let p = consolidate_manifest(&HostFileAccess, tmp.path(), &PanicResolver, &m) + .await + .unwrap(); + let env = p.environments.get(LOCAL).expect("local environment"); - // Track which network names are protected (only IC network) - let protected_network_names: HashSet = [IC.to_string()].into_iter().collect(); + // At the workspace root (member == root): no scoping. + assert_eq!(member_scoped_canisters(&p.dir, Some(&p.dir), env), None); - // Resolve NetworkManifests and add them (including user-defined "local" if provided) - for i in &m.networks { - let m = match i { - Item::Path(path) => { - let path = pdir.join(path); - if !path.exists() || !path.is_file() { - return NotFoundSnafu { - kind: "network".to_string(), - path: path.to_string(), - } - .fail(); - } - load_manifest_from_path::(&path) - .await - .context(LoadNetworkSnafu)? - } - Item::Manifest(ms) => ms.clone(), - }; - - match networks.entry(m.name.to_owned()) { - // Duplicate - Entry::Occupied(e) => { - // Only error if trying to override a protected network - if protected_network_names.contains(&m.name) { - return ReservedSnafu { - kind: "network".to_string(), - name: m.name.to_string(), - } - .fail(); - } - - // For non-protected duplicates, this is a user error (defining same network twice) - return DuplicateSnafu { - kind: "network".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - - // Ok - Entry::Vacant(e) => { - e.insert(Network { - name: m.name.to_owned(), - configuration: m.configuration.into(), // Convert manifest to config struct - }); - } - } - } - - // After processing user networks, add default "local" if not already defined - // This provides backward compatibility for projects that don't define their own "local" network - if !networks.contains_key(LOCAL) { - networks.insert( - LOCAL.to_string(), - Network { - name: LOCAL.to_string(), - configuration: Configuration::Managed { - managed: Managed { - mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway: Gateway { - bind: DEFAULT_LOCAL_NETWORK_BIND.to_string(), - port: Port::Fixed(DEFAULT_LOCAL_NETWORK_PORT), - domains: vec![], - }, - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: None, - dogecoind_addr: None, - version: None, - })), - }, - }, - }, - ); - } - - // Environments - let mut environments: HashMap = HashMap::new(); - - for i in &m.environments { - let m = match i { - Item::Path(path) => { - let path = pdir.join(path); - if !path.exists() || !path.is_file() { - return NotFoundSnafu { - kind: "environment".to_string(), - path: path.to_string(), - } - .fail(); - } - load_manifest_from_path::(&path) - .await - .context(LoadEnvironmentSnafu)? - } - Item::Manifest(ms) => ms.clone(), - }; - - match environments.entry(m.name.to_owned()) { - // Duplicate - Entry::Occupied(e) => { - return DuplicateSnafu { - kind: "environment".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - - // Ok - Entry::Vacant(e) => { - e.insert(Environment { - name: m.name.to_owned(), - - // Embed network in environment - network: { - let v = networks.get(&m.network).ok_or( - InvalidNetworkSnafu { - environment: m.name.to_owned(), - network: m.network.to_owned(), - } - .build(), - )?; - - v.to_owned() - }, - - // Embed canisters in environment, folding member overrides - // beneath the root's own settings/init_args overrides. - canisters: build_environment_canisters( - &canisters, - &m.name, - &m.canisters, - member_env_overrides.get(&m.name), - m.settings.as_ref(), - m.init_args.as_ref(), - )?, - }); - } - } - } - - // We're done adding all the user environments - // Now we add the implicit `local` and `ic` environment if the user hasn't overriden it - if let Entry::Vacant(vacant_entry) = environments.entry(LOCAL.to_string()) { - let network = networks - .get(LOCAL) - .ok_or( - InvalidNetworkSnafu { - environment: LOCAL.to_owned(), - network: LOCAL.to_owned(), - } - .build(), - )? - .to_owned(); - vacant_entry.insert(Environment { - name: LOCAL.to_string(), - network, - canisters: build_environment_canisters( - &canisters, - LOCAL, - &CanisterSelection::Everything, - member_env_overrides.get(LOCAL), - None, - None, - )?, - }); - } - if let Entry::Vacant(vacant_entry) = environments.entry(IC.to_string()) { - let network = networks - .get(IC) - .ok_or( - InvalidNetworkSnafu { - environment: IC.to_owned(), - network: IC.to_owned(), - } - .build(), - )? - .to_owned(); - vacant_entry.insert(Environment { - name: IC.to_string(), - network, - canisters: build_environment_canisters( - &canisters, - IC, - &CanisterSelection::Everything, - member_env_overrides.get(IC), - None, - None, - )?, - }); - } - - // Strict rule: every member must declare each environment the root targets. - // `local`/`ic` are implicit for every project, so they never count - // as missing; other environments must be declared explicitly by the member. - // Recorded per-environment and enforced lazily when that environment is - // selected (so a missing `staging` never blocks `deploy -e local`). - let mut member_missing_envs: HashMap> = HashMap::new(); - for env_name in environments.keys() { - if env_name == LOCAL || env_name == IC { - continue; - } - for member in &members { - if !member.defined.contains(env_name) { - member_missing_envs - .entry(env_name.clone()) - .or_default() - .push(member.prefix.clone()); - } - } - } - - Ok(Project { - dir: pdir.into(), - canisters, - networks, - environments, - member_missing_envs, - }) -} - -#[cfg(test)] -mod dependency_tests { - use super::*; - use crate::canister::recipe::{Fetched, Resolve, ResolveError}; - use crate::manifest::recipe::Recipe; - use camino_tempfile::Utf8TempDir; - - /// Recipes are never used in these tests; every canister is pre-built. - struct PanicResolver; - - #[async_trait::async_trait] - impl Resolve for PanicResolver { - async fn resolve(&self, _recipe: &Recipe) -> Result { - panic!("recipe resolver should not be called in dependency tests"); - } - } - - fn write(dir: &Path, rel: &str, contents: &str) { - let p = dir.join(rel); - std::fs::create_dir_all(p.parent().unwrap()).unwrap(); - std::fs::write(p, contents).unwrap(); - } - - /// A minimal `icp.yaml` body declaring the given pre-built canisters, - /// followed by a raw `dependencies:` block (may be empty). - fn manifest(canisters: &[&str], deps: &str) -> String { - let mut s = String::new(); - if canisters.is_empty() { - s.push_str("canisters: []\n"); - } else { - s.push_str("canisters:\n"); - for c in canisters { - s.push_str(&format!( - " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" - )); - } - } - s.push_str(deps); - s - } - - async fn consolidate(pdir: &Path) -> Result { - let m: ProjectManifest = load_manifest_from_path(&pdir.join(PROJECT_MANIFEST)) - .await - .expect("failed to parse project manifest"); - consolidate_manifest(pdir, &PanicResolver, &m).await - } - - fn bindings_of<'a>(p: &'a Project, key: &str) -> &'a BTreeMap { - &p.canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - p.canisters.keys().collect::>() - ) - }) - .1 - .bindings - } - - fn friendly_names_of<'a>(p: &'a Project, key: &str) -> &'a [String] { - &p.canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - p.canisters.keys().collect::>() - ) - }) - .1 - .friendly_names - } - - #[tokio::test] - async fn single_project_bindings_are_self_and_siblings() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Flat behavior preserved: every canister maps every sibling (incl. self) - // to itself. - let expected = BTreeMap::from([ - ("backend".to_string(), "backend".to_string()), - ("frontend".to_string(), "frontend".to_string()), - ]); - assert_eq!(bindings_of(&p, "backend"), &expected); - assert_eq!(bindings_of(&p, "frontend"), &expected); - } - - #[tokio::test] - async fn dependency_import_and_exposure_subset() { - let tmp = Utf8TempDir::new().unwrap(); - // Dependency nested inside the app (mirrors a submodule under the app). - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [backend]\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // The whole dependency is deployed (both canisters imported), keyed by path. - assert!(p.canisters.contains_key("backend")); - assert!(p.canisters.contains_key("openemail:backend")); - assert!(p.canisters.contains_key("openemail:frontend")); - - // App's own canister sees itself and only the *exposed* dependency canister. - assert_eq!( - bindings_of(&p, "backend"), - &BTreeMap::from([ - ("backend".to_string(), "backend".to_string()), - ( - "openemail:backend".to_string(), - "openemail:backend".to_string() - ), - ]) - ); - - // The dependency's own canisters keep their standalone view (bare names). - assert_eq!( - bindings_of(&p, "openemail:backend"), - &BTreeMap::from([ - ("backend".to_string(), "openemail:backend".to_string()), - ("frontend".to_string(), "openemail:frontend".to_string()), - ]) - ); - } - - #[tokio::test] - async fn member_scope_targets_only_the_members_canisters() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let env = p.environments.get(LOCAL).expect("local environment"); - - // At the workspace root (member == root): no scoping. - assert_eq!(member_scoped_canisters(&p.dir, Some(&p.dir), env), None); - - // Unknown member dir: no scoping. - assert_eq!(member_scoped_canisters(&p.dir, None, env), None); + // Unknown member dir: no scoping. + assert_eq!(member_scoped_canisters(&p.dir, None, env), None); // Inside the member: only the member's own canisters, not the app's. let member = tmp.path().join("openemail"); @@ -1658,777 +339,4 @@ mod dependency_tests { ] ); } - - #[tokio::test] - async fn member_env_config_folds_in_with_root_override_winning() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail defines `staging` with per-canister settings for its own - // canisters. - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - name: backend - build: - steps: - - type: pre-built - path: backend.wasm - - name: frontend - build: - steps: - - type: pre-built - path: frontend.wasm -environments: - - name: staging - settings: - backend: - compute_allocation: 5 - frontend: - compute_allocation: 7 -"#, - ); - // The app declares openemail and also defines `staging`, overriding the - // imported backend's settings (the root override must win). - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging - settings: - "openemail:backend": - compute_allocation: 99 -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - - // Root override wins over the member's config. - assert_eq!( - staging - .canisters - .get("openemail:backend") - .unwrap() - .1 - .settings - .compute_allocation, - Some(99), - ); - // No root override → the member's own config applies (standalone-equivalence). - assert_eq!( - staging - .canisters - .get("openemail:frontend") - .unwrap() - .1 - .settings - .compute_allocation, - Some(7), - ); - // Both projects declared staging, so nothing is recorded as missing. - assert!(p.member_missing_envs.is_empty()); - } - - #[tokio::test] - async fn missing_member_environment_is_recorded() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - // openemail does not declare `staging`, so it is recorded as missing. - assert_eq!( - p.member_missing_envs.get("staging"), - Some(&vec!["openemail".to_string()]), - ); - // Implicit environments are never recorded as missing. - assert!(!p.member_missing_envs.contains_key("local")); - assert!(!p.member_missing_envs.contains_key("ic")); - } - - #[tokio::test] - async fn diamond_dedups_to_single_instance() { - let tmp = Utf8TempDir::new().unwrap(); - // umbrella layout: service-a and service-b both depend on ../openemail. - write( - tmp.path(), - "umbrella/openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "umbrella/service-a/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "umbrella/service-b/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // openemail is imported exactly once despite two edges reaching it. - let openemail_keys: Vec<_> = p - .canisters - .keys() - .filter(|k| k.contains("openemail")) - .collect(); - assert_eq!( - openemail_keys, - vec![&"umbrella/openemail:backend".to_string()], - "expected a single shared openemail instance" - ); - - // Both services' code reads `openemail:backend`, resolving to the one instance. - assert_eq!( - bindings_of(&p, "umbrella/service-a:backend").get("openemail:backend"), - Some(&"umbrella/openemail:backend".to_string()) - ); - assert_eq!( - bindings_of(&p, "umbrella/service-b:backend").get("openemail:backend"), - Some(&"umbrella/openemail:backend".to_string()) - ); - - // The single shared instance is reachable at one friendly URL per alias - // chain (§17.3) — the store-key path (`umbrella/`) never appears. - assert_eq!( - friendly_names_of(&p, "umbrella/openemail:backend"), - &["backend.openemail.service-a", "backend.openemail.service-b"] - ); - // Each service's own canister is named by its own alias chain. - assert_eq!( - friendly_names_of(&p, "umbrella/service-a:backend"), - &["backend.service-a"] - ); - assert_eq!( - friendly_names_of(&p, "umbrella/service-b:backend"), - &["backend.service-b"] - ); - } - - #[tokio::test] - async fn diamond_transitive_dependency_gets_url_per_chain() { - let tmp = Utf8TempDir::new().unwrap(); - // The shared openemail itself depends on libfoo, and is reached via both - // service-a and service-b. - write( - tmp.path(), - "umbrella/openemail/libfoo/icp.yaml", - &manifest(&["bar"], ""), - ); - write( - tmp.path(), - "umbrella/openemail/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: libfoo\n path: ./libfoo\n", - ), - ); - write( - tmp.path(), - "umbrella/service-a/icp.yaml", - &manifest( - &["service"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "umbrella/service-b/icp.yaml", - &manifest( - &["service"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // The shared instance's own canister gets one URL per chain... - assert_eq!( - friendly_names_of(&p, "umbrella/openemail:backend"), - &["backend.openemail.service-a", "backend.openemail.service-b"] - ); - // ...and so does its *transitive* dependency (the subtree is revisited on - // the diamond hit, not just the instance's own canisters). - assert_eq!( - friendly_names_of(&p, "umbrella/openemail/libfoo:bar"), - &[ - "bar.libfoo.openemail.service-a", - "bar.libfoo.openemail.service-b" - ] - ); - } - - #[tokio::test] - async fn dot_in_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - // '.' is banned: it would be ambiguous in a dot-nested friendly subdomain - // (an own canister named `frontend.openemail` could collide with dependency - // `openemail`'s `frontend`). The strict name rule rejects it up front. - write( - tmp.path(), - "icp.yaml", - &manifest(&["frontend.openemail"], ""), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn invalid_dependency_alias_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["app"], - "dependencies:\n - name: open.email\n path: ./openemail\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidDependencyAlias { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn member_override_controllers_are_translated_to_store_keys() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail's `staging` override names a controller by its local name. - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - name: backend - build: - steps: - - type: pre-built - path: backend.wasm - - name: frontend - build: - steps: - - type: pre-built - path: frontend.wasm -environments: - - name: staging - settings: - backend: - controllers: ["frontend"] -"#, - ); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - let controllers = staging - .canisters - .get("openemail:backend") - .unwrap() - .1 - .settings - .controllers - .clone() - .expect("controllers set by the member override"); - - // The member-local `frontend` must be translated to its store key, so it - // resolves against the workspace id map at deploy time. - assert_eq!( - controllers, - vec![ControllerRef::CanisterName( - "openemail:frontend".to_string() - )] - ); - } - - fn env_vars_of<'a>( - canisters: &'a IndexMap, - key: &str, - ) -> &'a HashMap { - canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - canisters.keys().collect::>() - ) - }) - .1 - .settings - .environment_variables - .as_ref() - .expect("environment variables set") - } - - /// A canister manifest's file-backed environment variable resolves against - /// the canister's own directory, and the file's trailing newline is not part - /// of the value. - #[tokio::test] - async fn env_var_file_resolves_against_canister_dir() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "canisters/backend/canister.yaml", - r#" -name: backend -settings: - environment_variables: - API_KEY: - path: secrets/api-key -build: - steps: - - type: pre-built - path: backend.wasm -"#, - ); - write(tmp.path(), "canisters/backend/secrets/api-key", "s3cret\n"); - write( - tmp.path(), - "icp.yaml", - "canisters:\n - ./canisters/backend\n", - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - assert_eq!( - env_vars_of(&p.canisters, "backend"), - &HashMap::from([("API_KEY".to_string(), "s3cret".to_string())]), - ); - } - - /// A canister declared in its own directory, for the override tests below: - /// its directory is neither the project's nor an environment manifest's, so - /// the base a path resolves against is unambiguous. - fn write_backend_canister(dir: &Path, at: &str) { - write( - dir, - &format!("{at}/canister.yaml"), - r#" -name: backend -build: - steps: - - type: pre-built - path: backend.wasm -"#, - ); - } - - /// An environment override resolves a path against the *canister's* directory - /// — the same base an `init_args` override uses — not against the manifest - /// declaring the override, even when that is an environment manifest of its - /// own. - #[tokio::test] - async fn env_var_file_in_environment_override_resolves_against_canister_dir() { - let tmp = Utf8TempDir::new().unwrap(); - write_backend_canister(tmp.path(), "canisters/backend"); - write( - tmp.path(), - "icp.yaml", - "canisters:\n - ./canisters/backend\nenvironments:\n - ./environments/staging.yaml\n", - ); - write( - tmp.path(), - "environments/staging.yaml", - r#" -name: staging -settings: - backend: - environment_variables: - API_KEY: - path: secrets/api-key -"#, - ); - write( - tmp.path(), - "canisters/backend/secrets/api-key", - "staging-key\n", - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - - assert_eq!( - env_vars_of(&staging.canisters, "backend"), - &HashMap::from([("API_KEY".to_string(), "staging-key".to_string())]), - ); - // The override applies to the environment only; the canister's own - // settings are untouched. - assert_eq!( - p.canisters - .get("backend") - .unwrap() - .1 - .settings - .environment_variables, - None, - ); - } - - /// A member's own environment override resolves against the member's - /// canister, not against the member's or the root's project directory. - #[tokio::test] - async fn env_var_file_in_member_environment_resolves_against_member_canister_dir() { - let tmp = Utf8TempDir::new().unwrap(); - write_backend_canister(tmp.path(), "openemail/canisters/backend"); - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - ./canisters/backend -environments: - - name: staging - settings: - backend: - environment_variables: - API_KEY: - path: secrets/api-key -"#, - ); - write( - tmp.path(), - "openemail/canisters/backend/secrets/api-key", - "member-key\n", - ); - write( - tmp.path(), - "icp.yaml", - &format!( - "{}environments:\n - name: staging\n", - manifest( - &["app"], - "dependencies:\n - name: openemail\n path: ./openemail\n" - ) - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - - assert_eq!( - env_vars_of(&staging.canisters, "openemail:backend"), - &HashMap::from([("API_KEY".to_string(), "member-key".to_string())]), - ); - } - - #[tokio::test] - async fn missing_env_var_file_is_reported_with_the_variable_and_canister() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: backend - settings: - environment_variables: - API_KEY: - path: secrets/api-key - build: - steps: - - type: pre-built - path: backend.wasm -"#, - ); - - let err = consolidate(tmp.path()) - .await - .expect_err("the environment variable's file does not exist"); - - assert!( - matches!( - &err, - ConsolidateManifestError::ReadEnvironmentVariable { canister, variable, .. } - if canister == "backend" && variable == "API_KEY" - ), - "unexpected error: {err}" - ); - } - - #[tokio::test] - async fn friendly_names_are_bare_for_own_and_dotted_for_dependencies() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail (with a transitive dep libfoo) vendored under the app. - write( - tmp.path(), - "openemail/libfoo/icp.yaml", - &manifest(&["bar"], ""), - ); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest( - &["backend", "frontend"], - "dependencies:\n - name: libfoo\n path: ./libfoo\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Own canister: bare name (unchanged from single-project behavior). - assert_eq!(friendly_names_of(&p, "backend"), &["backend"]); - // Direct dependency: dot-nested by alias (no `vendor/` path noise). - assert_eq!( - friendly_names_of(&p, "openemail:backend"), - &["backend.openemail"] - ); - assert_eq!( - friendly_names_of(&p, "openemail:frontend"), - &["frontend.openemail"] - ); - // Transitive dependency: full alias chain, canister-most-specific first. - assert_eq!( - friendly_names_of(&p, "openemail/libfoo:bar"), - &["bar.libfoo.openemail"] - ); - } - - #[tokio::test] - async fn cycle_is_detected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest(&[], "dependencies:\n - name: a\n path: ./a\n"), - ); - write( - tmp.path(), - "a/icp.yaml", - &manifest(&["x"], "dependencies:\n - name: b\n path: ../b\n"), - ); - write( - tmp.path(), - "b/icp.yaml", - &manifest(&["y"], "dependencies:\n - name: a\n path: ../a\n"), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::CircularDependency { .. }), - "expected CircularDependency, got {err:?}" - ); - } - - #[tokio::test] - async fn alias_colliding_with_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["openemail"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::DependencyAliasCollision { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn duplicate_alias_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write(tmp.path(), "one/icp.yaml", &manifest(&["backend"], "")); - write(tmp.path(), "two/icp.yaml", &manifest(&["backend"], "")); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: dup\n path: ./one\n - name: dup\n path: ./two\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::DuplicateDependencyAlias { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn colon_in_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write(tmp.path(), "icp.yaml", &manifest(&["foo:bar"], "")); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn unknown_exposed_canister_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [nope]\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::UnknownDependencyCanister { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn missing_dependency_path_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: openemail\n path: ./does-not-exist\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::DependencyNotFound { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn imported_canisters_appear_in_implicit_environments() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Deploy-all: the implicit `local` environment includes the dependency. - let local = p.environments.get("local").unwrap(); - assert!(local.canisters.contains_key("backend")); - assert!(local.canisters.contains_key("openemail:backend")); - } } diff --git a/docs/concepts/recipes.md b/docs/concepts/recipes.md index b8b31b89d..494db939a 100644 --- a/docs/concepts/recipes.md +++ b/docs/concepts/recipes.md @@ -39,6 +39,13 @@ canisters: - cp target/wasm32-unknown-unknown/release/my_backend.wasm "$ICP_WASM_OUTPUT_PATH" ``` +### Extra Sync Steps + +A recipe defines the canister's build, so `recipe` and `build` are mutually +exclusive. `sync` is not: a canister using a recipe may declare its own `sync` +steps, which run after the recipe's. See +[Using Recipes](../guides/using-recipes.md#adding-your-own-sync-steps). + ## Recipe Sources Recipes can come from three sources: diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 86646e197..7317870d8 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls, read canister metadata, and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls, read canister metadata, set canister environment variables, and read declared files — nothing more. By default it can reach only the canister being synced; it may reach other canisters it lists in the sync step's `canisters:` list. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -15,7 +15,7 @@ Sync plugins fill that gap. A plugin is: - **Portable** — written in any language that compiles to `wasm32-wasip2`, distributed as one `.wasm` file (local path or remote URL + `sha256`). - **Sandboxed** — it cannot open network sockets, spawn subprocesses, or touch the filesystem outside the directories you explicitly grant it. -- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister listed in the manifest's `canisters:` list. A call to a canister that was not listed is rejected by the host. +- **Scoped by declaration** — it can call update and query methods on, read metadata from, and set environment variables on the canister being synced, plus any canister listed in the manifest's `canisters:` list. A request aimed at a canister that was not listed is rejected by the host. The most common way to get a sync plugin is through a [recipe](recipes.md). For example, the `@dfinity/asset-canister` recipe emits a `plugin` sync step (starting with `v2.2.1`) that uploads your built static files to the asset canister — so for everyday frontend deployment you never write a plugin yourself. @@ -38,15 +38,16 @@ icp sync │ canister-ids = │ dirs/files/fields = what you declared in the manifest │ - └─ plugin makes canister-call({ target, ... }) (× N) - and canister-metadata-section({ target, name }) + └─ plugin makes canister-call({ target, ... }) (× N), + canister-metadata-section({ target, name }), and + canister-set-environment-variable({ target, name, value }) target = host (the canister being synced), or a canister from `canisters:` by name ``` ## The Plugin Interface -The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `canister-metadata-section`); the plugin provides one export (`exec`): +The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides three imports (`canister-call`, `canister-metadata-section`, and `canister-set-environment-variable`); the plugin provides one export (`exec`): ```wit world sync-plugin { @@ -56,6 +57,9 @@ world sync-plugin { // Host import: read a metadata section from one of those same canisters. import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; + // Host import: set an environment variable on one of those same canisters. + import canister-set-environment-variable: func(req: set-environment-variable-request) -> result<_, string>; + // Plugin export: run the sync step. export exec: func(input: sync-exec-input) -> result<_, string>; } @@ -118,6 +122,24 @@ The two routes differ in who the target sees asking, which decides whether a **p With no proxy configured, both settings read directly. +### Setting an environment variable — `canister-set-environment-variable` + +The plugin writes one of a canister's [environment variables](../reference/environment-variables.md#canister-runtime-environment-variables) — configuration the canister's own code reads at runtime — through the `canister-set-environment-variable` import: + +| Request field | Meaning | +|---------------|---------| +| `target` | Which canister to set the variable on: `host`, or a canister declared in `canisters:` addressed by `name` — the same targets, and the same enforcement, as `canister-call` | +| `name` | Name of the environment variable, spelled as the canister reads it | +| `value` | Value to set it to, replacing whatever value the target has under that name | +| `direct` | When `false` (default), the update is made by the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, it is signed by the sync identity | + +The target's other environment variables, and the rest of its settings, are left as they are. Setting a variable is **controller-gated**: whoever makes the update — the sync identity or the proxy, per `direct` — must control the target, the same requirement as `icp canister settings update`. + +Two things follow from how the management canister models environment variables, which it can only replace as a whole list: + +- **The update is a read-then-write, not an atomic one.** The host reads the target's current variables and writes them back with yours added. Another party's settings update that lands between the two is overwritten. +- **A later `icp deploy` drops the variable.** Deploy rewrites each canister's variables from the manifest plus the automatic `PUBLIC_CANISTER_ID:*` bindings, without preserving what a plugin added. In the ordinary case this is invisible: deploy runs the sync phase afterwards, so a plugin that sets the variable on every sync sets it again. To have a variable survive independently of the plugin, declare it in the manifest's [`environment_variables`](../reference/canister-settings.md#environment_variables) setting instead. + ### Logging — stdout and stderr The plugin's stdout and stderr are captured by the host (no logging import is needed — use ordinary `println!` / `eprintln!`): @@ -149,7 +171,8 @@ The plugin runs with a deliberately narrow capability surface. | `process::exit` / panics | yes | abort the guest cleanly; the host surfaces the error | | Canister calls | yes | to the canister being synced, and to canisters declared in `canisters:` | | Canister metadata reads | yes | the same set of canisters as calls | -| Environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | +| Canister environment-variable writes | yes | the same set of canisters as calls; the caller must control the target | +| The plugin's own environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | | Network sockets / DNS | blocked | treat the network as unavailable | | Filesystem writes | blocked | no writable preopens | | Spawning subprocesses | blocked | no process interface is linked | @@ -163,7 +186,7 @@ The plugin runs with a deliberately narrow capability surface. | Linear memory | wasm32 address space (≤ 4 GiB) | | stdout / stderr per stream | 1 MiB | -The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `canister-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. +The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `canister-metadata-section`, `canister-set-environment-variable`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. ## Next Steps diff --git a/docs/guides/using-recipes.md b/docs/guides/using-recipes.md index 9201dcce5..55388bec2 100644 --- a/docs/guides/using-recipes.md +++ b/docs/guides/using-recipes.md @@ -163,6 +163,27 @@ canisters: API_KEY: "secret" ``` +## Adding Your Own Sync Steps + +A recipe canister can also declare a `sync` section of its own, for +post-deployment work the recipe does not cover. Those steps run after the +recipe's own sync steps: + +```yaml +canisters: + - name: backend + recipe: + type: "@dfinity/rust@v3.0.0" + configuration: + package: backend + sync: + steps: + - type: script + command: ./scripts/seed-data.sh +``` + +`build` remains exclusive with `recipe`: the recipe is what defines the build. + ## Next Steps - [Recipes](../concepts/recipes.md) — Understand how recipes work diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index fde34ad84..0224ae347 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -37,7 +37,7 @@ wit-bindgen = { version = "0.56", features = ["realloc"] } ## Generate Bindings and Implement `exec` -`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `canister_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. +`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `canister_metadata_section`, `canister_set_environment_variable`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. ```rust // src/lib.rs @@ -109,6 +109,23 @@ match interface { `direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--canister-metadata-section) for the full semantics. +## Set a Canister Environment Variable + +`canister_set_environment_variable` writes one of a canister's [environment variables](../reference/environment-variables.md#canister-runtime-environment-variables) — configuration the canister reads at runtime — leaving its other variables and settings alone: + +```rust +canister_set_environment_variable(&SetEnvironmentVariableRequest { + target: CallTarget::Host, // same targets, same rules, as canister_call + name: "SEEDED_BY".to_string(), + value: input.environment.clone(), + direct: false, // let the proxy make the update if one is configured +})?; +``` + +The update is controller-gated, so whoever makes it must control the target: with `direct: true` that is the sync identity, with `direct: false` and a proxy configured it is the proxy canister. + +Set the variable on every sync rather than once. The management canister can only replace a canister's variables as a whole list, so the host reads them and writes them back with yours added — and a later `icp deploy` rewrites that list from the manifest plus the automatic `PUBLIC_CANISTER_ID:*` bindings, dropping anything a plugin added. Deploy runs the sync phase afterwards, so a plugin that always sets it always restores it. For a variable that should not depend on the plugin running, declare it in the manifest's [`environment_variables`](../reference/canister-settings.md#environment_variables) setting instead. + ## Read Declared Files and Directories A plugin can't see the filesystem freely — only what you grant it in the manifest's `dirs:` and `files:`. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index c8ccf0714..d9668055b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -60,7 +60,7 @@ canisters: | `sync` | object | No | Post-deployment sync configuration | | `settings` | object | No | Canister settings | | `init_args` | string or object | No | Initialization arguments (see [Init Args](#init-args)) | -| `recipe` | object | No | Recipe reference (alternative to build) | +| `recipe` | object | No | Recipe reference (alternative to build; may be combined with `sync`) | ## Build Steps @@ -218,6 +218,26 @@ canisters: | `sha256` | string | Conditional | Required for remote URLs | | `configuration` | object | No | Parameters passed to recipe template | +### Adding Sync Steps to a Recipe + +A canister that uses a recipe may declare a `sync` section of its own. Its steps +run after the ones the recipe renders, in the order written: + +```yaml +canisters: + - name: frontend + recipe: + type: "@dfinity/asset-canister@v2.2.1" + configuration: + dir: dist + sync: + steps: + - type: script + command: ./scripts/warm-cache.sh +``` + +A `recipe` still cannot be combined with `build` — the recipe defines the build. + ### Recipe Type Formats ```yaml diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 0c550ed58..9bf62818c 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -34,29 +34,6 @@ "type": "object" }, "Adapter2": { - "anyOf": [ - { - "$ref": "#/$defs/LocalSource", - "description": "Local path on-disk to read a WASM file from" - }, - { - "$ref": "#/$defs/RemoteSource", - "description": "Remote url to fetch a WASM file from" - } - ], - "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", - "properties": { - "sha256": { - "description": "Optional sha256 checksum of the WASM", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "Adapter3": { "anyOf": [ { "$ref": "#/$defs/LocalSource", @@ -70,7 +47,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call` or `canister-metadata-section` request; a target\nnot listed here is rejected by the host.", + "description": "Canisters this plugin may call, read metadata from, or set environment\nvariables on, in addition to the canister being synced. Each entry is a\ncanister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call`, `canister-metadata-section`, or\n`canister-set-environment-variable` request; a target not listed here is\nrejected by the host.", "items": { "type": "string" }, @@ -125,6 +102,29 @@ }, "type": "object" }, + "Adapter3": { + "anyOf": [ + { + "$ref": "#/$defs/LocalSource", + "description": "Local path on-disk to read a WASM file from" + }, + { + "$ref": "#/$defs/RemoteSource", + "description": "Remote url to fetch a WASM file from" + } + ], + "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", + "properties": { + "sha256": { + "description": "Optional sha256 checksum of the WASM", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "ArgsFormat": { "description": "Format specifier for canister call/install args content.", "oneOf": [ @@ -163,7 +163,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter2", + "$ref": "#/$defs/Adapter3", "description": "Represents a pre-built canister.\nThis variant allows for retrieving a canister WASM from various sources.", "properties": { "type": { @@ -542,7 +542,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter3", + "$ref": "#/$defs/Adapter2", "description": "Represents a sync step executed by a WebAssembly plugin running inside\na wasmtime WASI sandbox. The plugin can call canister methods on exactly\nthe canister being synced and read files from the declared `dirs`.", "properties": { "type": { @@ -580,6 +580,17 @@ "properties": { "recipe": { "$ref": "#/$defs/Recipe" + }, + "sync": { + "anyOf": [ + { + "$ref": "#/$defs/SyncSteps" + }, + { + "type": "null" + } + ], + "description": "Additional sync steps, run after the ones the recipe renders." } }, "required": [ diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 32fece2fd..7f160518c 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -34,29 +34,6 @@ "type": "object" }, "Adapter2": { - "anyOf": [ - { - "$ref": "#/$defs/LocalSource", - "description": "Local path on-disk to read a WASM file from" - }, - { - "$ref": "#/$defs/RemoteSource", - "description": "Remote url to fetch a WASM file from" - } - ], - "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", - "properties": { - "sha256": { - "description": "Optional sha256 checksum of the WASM", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "Adapter3": { "anyOf": [ { "$ref": "#/$defs/LocalSource", @@ -70,7 +47,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call` or `canister-metadata-section` request; a target\nnot listed here is rejected by the host.", + "description": "Canisters this plugin may call, read metadata from, or set environment\nvariables on, in addition to the canister being synced. Each entry is a\ncanister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call`, `canister-metadata-section`, or\n`canister-set-environment-variable` request; a target not listed here is\nrejected by the host.", "items": { "type": "string" }, @@ -125,6 +102,29 @@ }, "type": "object" }, + "Adapter3": { + "anyOf": [ + { + "$ref": "#/$defs/LocalSource", + "description": "Local path on-disk to read a WASM file from" + }, + { + "$ref": "#/$defs/RemoteSource", + "description": "Remote url to fetch a WASM file from" + } + ], + "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", + "properties": { + "sha256": { + "description": "Optional sha256 checksum of the WASM", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "ArgsFormat": { "description": "Format specifier for canister call/install args content.", "oneOf": [ @@ -163,7 +163,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter2", + "$ref": "#/$defs/Adapter3", "description": "Represents a pre-built canister.\nThis variant allows for retrieving a canister WASM from various sources.", "properties": { "type": { @@ -199,6 +199,17 @@ "properties": { "recipe": { "$ref": "#/$defs/Recipe" + }, + "sync": { + "anyOf": [ + { + "$ref": "#/$defs/SyncSteps" + }, + { + "type": "null" + } + ], + "description": "Additional sync steps, run after the ones the recipe renders." } }, "required": [ @@ -1065,7 +1076,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter3", + "$ref": "#/$defs/Adapter2", "description": "Represents a sync step executed by a WebAssembly plugin running inside\na wasmtime WASI sandbox. The plugin can call canister methods on exactly\nthe canister being synced and read files from the declared `dirs`.", "properties": { "type": { diff --git a/examples/icp-sync-plugin/README.md b/examples/icp-sync-plugin/README.md index 8f2c8ee2e..f8f7b42ae 100644 --- a/examples/icp-sync-plugin/README.md +++ b/examples/icp-sync-plugin/README.md @@ -27,14 +27,15 @@ A simple Rust canister with three methods: A Rust Wasm component that implements the `sync-plugin` world defined in `crates/icp-sync-plugin/sync-plugin.wit`. The host runtime calls its `exec` -export and provides the `canister-call` and `canister-metadata-section` imports the -plugin uses to reach the canister. +export and provides the `canister-call`, `canister-metadata-section`, and +`canister-set-environment-variable` imports the plugin uses to reach the +canister. ## How the plugin system is exercised This example is designed to demonstrate both routing modes of the `canister-call` import — the `direct` flag — in a single sync run, plus a -metadata read that follows the same routing. +metadata read and an environment-variable write that follow the same routing. ### Read — `candid:service` via proxy (`direct: false`) @@ -60,6 +61,19 @@ canister is listed as a controller of the target, the controller guard passes. This models a pattern where privileged, one-time setup calls must come from a known controller — not directly from an end-user identity. +### Write — `SEEDED_BY` environment variable via proxy (`direct: false`) + +The plugin then records which environment seeded the canister as a canister +environment variable, so the canister's own code can read it back at runtime. +Setting settings is controller-gated exactly like `set_uploader`, so it takes +the same route: the proxy makes both the settings read and the settings write, +and it is the proxy's control over the canister that they are checked against. + +The management canister can only replace a canister's environment variables as a +whole list, so the host reads the current ones and writes them back with +`SEEDED_BY` added — the `PUBLIC_CANISTER_ID:*` variables `icp deploy` writes +itself are still there afterwards. + ### Call 2 — `register` directly (`direct: true`) For each file under `seed-data/`, the plugin calls `register` with @@ -85,6 +99,10 @@ icp sync ├─ canister-call set_uploader() direct=false → proxy → canister │ canister stores uploader = │ + ├─ canister-set-environment-variable SEEDED_BY= + │ direct=false → proxy → mgmt canister + │ read current settings, write them back with SEEDED_BY added + │ └─ canister-call register(name, content) direct=true → canister (× N files) canister checks caller == uploader ✓ ``` diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index 09595a47d..723c75ed4 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -46,7 +46,19 @@ impl Guest for Plugin { })?; println!("set_uploader ({}): ok", input.identity_principal); - // 3. Register every file found by traversing the preopened dirs. + // 3. Record which environment seeded the canister as an environment + // variable, so the canister's own code can read it back. Setting + // settings is controller-gated like set_uploader, so it takes the + // same route (direct: false). + canister_set_environment_variable(&SetEnvironmentVariableRequest { + target: CallTarget::Host, + name: "SEEDED_BY".to_string(), + value: input.environment.clone(), + direct: false, + })?; + eprintln!("SEEDED_BY={}", input.environment); + + // 4. Register every file found by traversing the preopened dirs. // Direct calls (direct: true) because register is gated on the // uploader principal, which is the current identity — not the proxy. let mut registered = 0u32;