From 63923ff41f4000e8ff5cba7bdbcf25f9387f1075 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Wed, 19 Aug 2026 09:06:11 -0700 Subject: [PATCH 01/51] Add parallel plugin interface version support --- Cargo.lock | 1 + crates/icp-sync-plugin/Cargo.toml | 1 + crates/icp-sync-plugin/DESIGN.md | 34 +- crates/icp-sync-plugin/build.rs | 20 +- crates/icp-sync-plugin/src/runtime.rs | 298 ++++++++++++--- crates/icp-sync-plugin/sync-plugin-v1.wit | 99 +++++ crates/icp-sync-plugin/sync-plugin.wit | 2 +- .../tests/fixtures/test-plugin-v1/Cargo.lock | 349 ++++++++++++++++++ .../tests/fixtures/test-plugin-v1/Cargo.toml | 13 + .../tests/fixtures/test-plugin-v1/build.rs | 3 + .../tests/fixtures/test-plugin-v1/src/lib.rs | 24 ++ docs/concepts/sync-plugins.md | 2 + 12 files changed, 781 insertions(+), 65 deletions(-) create mode 100644 crates/icp-sync-plugin/sync-plugin-v1.wit create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 67ca726a3..330024485 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3785,6 +3785,7 @@ dependencies = [ "hex", "ic-agent", "icp-canister-interfaces", + "semver", "snafu", "tokio", "wasmtime", diff --git a/crates/icp-sync-plugin/Cargo.toml b/crates/icp-sync-plugin/Cargo.toml index 216e9c761..74feb682e 100644 --- a/crates/icp-sync-plugin/Cargo.toml +++ b/crates/icp-sync-plugin/Cargo.toml @@ -15,6 +15,7 @@ console.workspace = true hex.workspace = true ic-agent.workspace = true icp-canister-interfaces.workspace = true +semver.workspace = true snafu.workspace = true tokio.workspace = true wasmtime.workspace = true diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 099a14714..411376f93 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -103,11 +103,12 @@ the WASI sandbox — cap-std — at runtime.) ### `HostState` and bindgen +Both interface versions are bound, each `bindgen!` in its own module so their +generated types don't collide: + ```rust -wasmtime::component::bindgen!({ - world: "sync-plugin", - path: "sync-plugin.wit", -}); +mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugin.wit" }); } +mod v1 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugin-v1.wit" }); } struct HostState { target_canister_id: Principal, @@ -118,17 +119,30 @@ struct HostState { epoch_extension: Arc, } -impl SyncPluginImports for HostState { - fn canister_call(&mut self, req: CanisterCallRequest) -> Result, String> { ... } -} +// Implemented for both v1::SyncPluginImports and v2::SyncPluginImports; both +// delegate to one shared `do_canister_call(...)`. ``` `HostState` implements `WasiView` so wasmtime_wasi can access the WASI context. `canister_call` uses `tokio::runtime::Handle::current().block_on(...)` because the caller already wraps the synchronous `run_plugin` in -`tokio::task::block_in_place`. When a proxy is configured and the call is a -non-`direct` update, it is encoded as `ProxyArgs` and routed through the proxy's -`proxy` method; otherwise it goes straight to the target via `ic-agent`. +`tokio::task::block_in_place`. Both interface versions call the canister being +synced. When a proxy is configured and the call is a non-`direct` update, it is +encoded as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise +it goes straight to the target via `ic-agent`. + +### Interface versioning (parallel v0.1.0 / v0.2.0 support) + +A component built with wit-bindgen imports the interface it `use`s as a +versioned instance — `icp:sync-plugin/types@0.1.0` or `@0.2.0`. `run_plugin` +reads that name off `Component::component_type().imports(...)` and matches the +version with semver caret requirements (`^0.1`, `^0.2`) to pick the ABI, then +instantiates the matching `bindgen!` world and builds the matching +`sync-exec-input`. Reading the plugin's declared metadata is preferred over +trial instantiation: it is unambiguous and needs no throwaway `Store`. A +component with no recognized `icp:sync-plugin/types@` import, or an +unsupported version, is rejected with `UnsupportedInterface`. Both `.wit` files +are checked in; `sync-plugin-v1.wit` is the frozen v0.1.0 contract. ### Compute budget (epoch interruption) diff --git a/crates/icp-sync-plugin/build.rs b/crates/icp-sync-plugin/build.rs index cbe6461f3..8e08a442e 100644 --- a/crates/icp-sync-plugin/build.rs +++ b/crates/icp-sync-plugin/build.rs @@ -3,11 +3,17 @@ use std::process::Command; fn main() { println!("cargo:rerun-if-changed=sync-plugin.wit"); + println!("cargo:rerun-if-changed=sync-plugin-v1.wit"); println!("cargo:rerun-if-changed=tests/fixtures/test-plugin/src/lib.rs"); println!("cargo:rerun-if-changed=tests/fixtures/test-plugin/Cargo.toml"); + println!("cargo:rerun-if-changed=tests/fixtures/test-plugin-v1/src/lib.rs"); + println!("cargo:rerun-if-changed=tests/fixtures/test-plugin-v1/Cargo.toml"); if wasm32_wasip2_is_installed() { - build_test_fixture(); + // Current-interface fixture, and a legacy-interface one so tests can + // exercise both load paths. + build_test_fixture("test-plugin", "test_plugin", "TEST_PLUGIN_WASM"); + build_test_fixture("test-plugin-v1", "test_plugin_v1", "TEST_PLUGIN_V1_WASM"); } } @@ -24,11 +30,11 @@ fn wasm32_wasip2_is_installed() -> bool { .exists() } -fn build_test_fixture() { +fn build_test_fixture(crate_dir: &str, wasm_stem: &str, env_var: &str) { let manifest_dir = Utf8PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); let out_dir = Utf8PathBuf::from(std::env::var("OUT_DIR").unwrap()); - let fixture_manifest = manifest_dir.join("tests/fixtures/test-plugin/Cargo.toml"); - let fixture_target_dir = out_dir.join("fixture-target"); + let fixture_manifest = manifest_dir.join(format!("tests/fixtures/{crate_dir}/Cargo.toml")); + let fixture_target_dir = out_dir.join(format!("fixture-target/{crate_dir}")); let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); let status = Command::new(&cargo) @@ -47,8 +53,8 @@ fn build_test_fixture() { .expect("failed to spawn cargo build for test fixture"); assert!( status.success(), - "cargo build --target wasm32-wasip2 failed for test fixture" + "cargo build --target wasm32-wasip2 failed for test fixture {crate_dir}" ); - let wasm = fixture_target_dir.join("wasm32-wasip2/release/test_plugin.wasm"); - println!("cargo:rustc-env=TEST_PLUGIN_WASM={wasm}"); + let wasm = fixture_target_dir.join(format!("wasm32-wasip2/release/{wasm_stem}.wasm")); + println!("cargo:rustc-env={env_var}={wasm}"); } diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index fb284fb7f..27091fc34 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -24,19 +24,38 @@ use bytes::Bytes; use camino::Utf8PathBuf; use candid::{Encode, Principal}; use ic_agent::Agent; +use semver::{Version, VersionReq}; use snafu::prelude::*; use tokio::io::{self, AsyncWrite}; use tokio::sync::mpsc::Sender; +use wasmtime::component::{Component, HasSelf, Linker}; +use wasmtime::{Config, Engine, Store}; use wasmtime_wasi::cli::{IsTerminal, StdoutStream}; use wasmtime_wasi::p2::{OutputStream, Pollable, StreamError}; use wasmtime_wasi::{DirPerms, FilePerms}; -wasmtime::component::bindgen!({ - world: "sync-plugin", - path: "sync-plugin.wit", -}); +// Both the current and the legacy plugin interfaces are bound, each in its own +// module so their generated type names don't collide. `run_plugin` reads the +// interface version from the component's own metadata (see `detect_plugin_abi`) +// and drives it through the matching module, so plugins built against either +// interface load. The two interfaces are currently structurally identical; the +// split exists so later breaking changes to the current interface can land +// without dropping support for already-built plugins. +mod v2 { + wasmtime::component::bindgen!({ + world: "sync-plugin", + path: "sync-plugin.wit", + }); +} + +mod v1 { + wasmtime::component::bindgen!({ + world: "sync-plugin", + path: "sync-plugin-v1.wit", + }); +} -use icp::sync_plugin::types::CallType; +use v2::icp::sync_plugin::types::CallType; // HostState holds everything the plugin's import functions need. struct HostState { @@ -64,31 +83,35 @@ impl wasmtime_wasi::WasiView for HostState { } } -// `types::Host` is an empty marker trait generated for the `types` interface. -impl icp::sync_plugin::types::Host for HostState {} - -impl SyncPluginImports for HostState { - fn canister_call(&mut self, req: CanisterCallRequest) -> Result, String> { +impl HostState { + /// Perform a canister call to the canister being synced. Shared by both + /// interface versions. + fn do_canister_call( + &mut self, + method: String, + arg_bytes: Vec, + call_type: CallType, + direct: bool, + cycles: u64, + ) -> Result, String> { use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; - let arg_bytes = req.arg; let cid = self.target_canister_id; - let method = req.method.clone(); let agent = Arc::clone(&self.agent); - let proxy = if req.direct { None } else { self.proxy }; + let proxy = if direct { None } else { self.proxy }; // We are already inside tokio::task::block_in_place (see sync/plugin.rs), // so blocking the thread here is safe. let start = Instant::now(); let result = tokio::runtime::Handle::current().block_on(async move { - match req.call_type { + match call_type { CallType::Update => { if let Some(proxy_cid) = proxy { let proxy_args = ProxyArgs { canister_id: cid, method: method.clone(), args: arg_bytes, - cycles: candid::Nat::from(req.cycles), + cycles: candid::Nat::from(cycles), }; let encoded = Encode!(&proxy_args) .map_err(|e| format!("proxy encode failed: {e}"))?; @@ -128,6 +151,38 @@ impl SyncPluginImports for HostState { } } +// -- v0.2.0 interface. --------------------------------------------------------- + +// `types::Host` is an empty marker trait generated for the `types` interface. +impl v2::icp::sync_plugin::types::Host for HostState {} + +impl v2::SyncPluginImports for HostState { + fn canister_call( + &mut self, + req: v2::icp::sync_plugin::types::CanisterCallRequest, + ) -> Result, String> { + self.do_canister_call(req.method, req.arg, req.call_type, req.direct, req.cycles) + } +} + +// -- v0.1.0 interface. --------------------------------------------------------- + +impl v1::icp::sync_plugin::types::Host for HostState {} + +impl v1::SyncPluginImports for HostState { + fn canister_call( + &mut self, + req: v1::icp::sync_plugin::types::CanisterCallRequest, + ) -> Result, String> { + // v1's `call-type` is a distinct generated enum; map it to the shared one. + let call_type = match req.call_type { + v1::icp::sync_plugin::types::CallType::Update => CallType::Update, + v1::icp::sync_plugin::types::CallType::Query => CallType::Query, + }; + self.do_canister_call(req.method, req.arg, call_type, req.direct, req.cycles) + } +} + // Used as the error payload inside the epoch_deadline_callback closure, which // must return wasmtime::Error (= anyhow::Error). Snafu derives std::error::Error // so .into() converts it via anyhow's blanket From. @@ -191,6 +246,12 @@ pub enum RunPluginError { path: Utf8PathBuf, }, + #[snafu(display( + "wasm component at {path} does not implement a supported sync-plugin interface ({detail}). \ + Supported: icp:sync-plugin@0.1 and icp:sync-plugin@0.2." + ))] + UnsupportedInterface { path: Utf8PathBuf, detail: String }, + #[snafu(display("failed to call exec() on plugin at {path}"))] CallExec { source: wasmtime::Error, @@ -201,6 +262,72 @@ pub enum RunPluginError { PluginFailed { message: String }, } +/// Which version of the sync-plugin interface a component was built against. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PluginAbi { + /// Current interface (`icp:sync-plugin@0.2.x`). + V2, + /// Legacy interface (`icp:sync-plugin@0.1.x`). + V1, +} + +/// The interface package a `use`-ing plugin component imports, whose version we +/// read to pick the ABI. wit-bindgen emits this import for any world that pulls +/// types from the interface, so it is present on every real plugin. +const TYPES_INTERFACE_PREFIX: &str = "icp:sync-plugin/types@"; + +/// Determine which interface a component implements by reading the version off +/// its imported `icp:sync-plugin/types@` instance — the plugin's own +/// declared metadata — rather than probing with a trial instantiation. The +/// version is matched with semver caret requirements, so each supported minor +/// (the breaking unit for 0.x) accepts any patch release within it. +fn detect_plugin_abi( + engine: &Engine, + component: &Component, + wasm_path: &Utf8PathBuf, +) -> Result { + let raw = component + .component_type() + .imports(engine) + .find_map(|(name, _)| name.strip_prefix(TYPES_INTERFACE_PREFIX).map(str::to_owned)); + + let Some(raw) = raw else { + return UnsupportedInterfaceSnafu { + path: wasm_path.clone(), + detail: format!("no {TYPES_INTERFACE_PREFIX} import found"), + } + .fail(); + }; + + let version = Version::parse(&raw).map_err(|source| { + UnsupportedInterfaceSnafu { + path: wasm_path.clone(), + detail: format!("interface version '{raw}' is not valid semver: {source}"), + } + .build() + })?; + + // `^0.1`/`^0.2` follow semver's 0.x rule: they match within the minor and + // exclude the next one (>=0.1.0, <0.2.0 and >=0.2.0, <0.3.0 respectively). + if VersionReq::parse("^0.2") + .expect("valid req") + .matches(&version) + { + Ok(PluginAbi::V2) + } else if VersionReq::parse("^0.1") + .expect("valid req") + .matches(&version) + { + Ok(PluginAbi::V1) + } else { + UnsupportedInterfaceSnafu { + path: wasm_path.clone(), + detail: format!("unsupported interface version {version}"), + } + .fail() + } +} + #[allow(clippy::too_many_arguments)] pub fn run_plugin( wasm_path: Utf8PathBuf, @@ -215,9 +342,6 @@ pub fn run_plugin( compute_limit_secs: u64, stdio: Option>, ) -> Result, RunPluginError> { - use wasmtime::component::{Component, Linker}; - use wasmtime::{Config, Engine, Store}; - let mut config = Config::new(); config.wasm_component_model(true); config.max_wasm_stack(MAX_WASM_STACK); @@ -283,7 +407,9 @@ pub fn run_plugin( // Read each declared file on the host and pass its content inline. The same // path-safety checks as `dirs` apply: reject escaping or symlinked paths so // a read cannot leave `base_dir`. - let mut file_inputs: Vec = Vec::with_capacity(files.len()); + // Held as plain (name, content) pairs so they can be converted to whichever + // interface version's `file-input` record the plugin turns out to use. + let mut file_contents: Vec<(String, String)> = Vec::with_capacity(files.len()); for name in &files { ensure!(!crate::path::escapes_base(name), UnsafeFileSnafu { name }); if let Some(link) = crate::path::first_symlink_component(&base_dir, name) { @@ -292,10 +418,7 @@ pub fn run_plugin( let path = base_dir.join(name); let content = std::fs::read_to_string(path.as_std_path()).context(ReadFileSnafu { path })?; - file_inputs.push(FileInput { - name: name.clone(), - content, - }); + file_contents.push((name.clone(), content)); } let persistent_stderr: Arc>> = Arc::default(); @@ -315,16 +438,6 @@ pub fn run_plugin( epoch_extension: epoch_extension.clone(), }; - let mut linker: Linker = Linker::new(&engine); - wasmtime_wasi::p2::add_to_linker_sync(&mut linker).context(InstantiateSnafu { - path: wasm_path.clone(), - })?; - SyncPlugin::add_to_linker::<_, wasmtime::component::HasSelf<_>>(&mut linker, |s| s).context( - InstantiateSnafu { - path: wasm_path.clone(), - }, - )?; - let mut store = Store::new(&engine, host_state); store.set_epoch_deadline(compute_limit_secs); store.epoch_deadline_callback(move |_| { @@ -339,22 +452,72 @@ pub fn run_plugin( } }); - let plugin = - SyncPlugin::instantiate(&mut store, &component, &linker).context(InstantiateSnafu { - path: wasm_path.clone(), - })?; - - let input = SyncExecInput { - canister_id: target_canister_id.to_text(), - environment, - dirs, - files: file_inputs, - identity_principal: identity_principal.to_text(), - proxy_canister_id: proxy.map(|p| p.to_text()), + let canister_id_text = target_canister_id.to_text(); + let identity_text = identity_principal.to_text(); + let proxy_text = proxy.map(|p| p.to_text()); + + // Which interface the plugin was built against is read from the component's + // own declared metadata (see `detect_plugin_abi`) rather than probed by + // trial instantiation, then driven through the matching bindgen world. + let call_result = match detect_plugin_abi(&engine, &component, &wasm_path)? { + PluginAbi::V2 => { + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_sync(&mut linker).context(InstantiateSnafu { + path: wasm_path.clone(), + })?; + v2::SyncPlugin::add_to_linker::<_, HasSelf<_>>(&mut linker, |s| s).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let plugin = v2::SyncPlugin::instantiate(&mut store, &component, &linker).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let input = v2::SyncExecInput { + canister_id: canister_id_text, + environment, + dirs, + files: file_contents + .into_iter() + .map(|(name, content)| v2::FileInput { name, content }) + .collect(), + identity_principal: identity_text, + proxy_canister_id: proxy_text, + }; + plugin.call_exec(&mut store, &input) + } + PluginAbi::V1 => { + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_sync(&mut linker).context(InstantiateSnafu { + path: wasm_path.clone(), + })?; + v1::SyncPlugin::add_to_linker::<_, HasSelf<_>>(&mut linker, |s| s).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let plugin = v1::SyncPlugin::instantiate(&mut store, &component, &linker).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let input = v1::SyncExecInput { + canister_id: canister_id_text, + environment, + dirs, + files: file_contents + .into_iter() + .map(|(name, content)| v1::FileInput { name, content }) + .collect(), + identity_principal: identity_text, + proxy_canister_id: proxy_text, + }; + plugin.call_exec(&mut store, &input) + } }; - let call_result = plugin.call_exec(&mut store, &input); - // Flush any partial line and emit the truncation note (if any) before // we hand control back, so the last line of plugin output isn't lost. stdout_capture.finalize(); @@ -782,6 +945,47 @@ mod tests { assert!(msg.contains("stdout from plugin"), "got: {msg}"); } + #[test] + fn legacy_v1_plugin_is_detected_and_driven() { + // A plugin built against the v0.1.0 interface must still load: the host + // reads its declared interface version and drives it through the v1 path. + let Some(wasm_path) = option_env!("TEST_PLUGIN_V1_WASM") else { + return; + }; + let result = run_plugin( + wasm_path.into(), + ".".into(), + vec![], + vec![], + anon(), + dummy_agent(), + None, + anon(), + "ok".to_string(), + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, + None, + ); + assert!(result.is_ok()); + // Its error surface flows through the same machinery as v0.2.0 plugins. + let result = run_plugin( + wasm_path.into(), + ".".into(), + vec![], + vec![], + anon(), + dummy_agent(), + None, + anon(), + "error".to_string(), + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, + None, + ); + assert!(matches!( + result, + Err(RunPluginError::PluginFailed { ref message }) if message == "deliberate v1 failure" + )); + } + #[tokio::test(flavor = "multi_thread")] async fn plugin_stderr_lines_returned_as_persistent_output() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { diff --git a/crates/icp-sync-plugin/sync-plugin-v1.wit b/crates/icp-sync-plugin/sync-plugin-v1.wit new file mode 100644 index 000000000..65c0283a9 --- /dev/null +++ b/crates/icp-sync-plugin/sync-plugin-v1.wit @@ -0,0 +1,99 @@ +// Version 0.1.0 of the sync-plugin interface, preserved verbatim so the host +// can still load plugins built against it. New plugins should target the +// current interface in `sync-plugin.wit`; see that file for the up-to-date +// contract. The host tries the current interface first and falls back to this +// one, so both APIs are supported in parallel. +package icp:sync-plugin@0.1.0; + +/// Types shared between the host runtime and sync plugins. +interface types { + /// Whether a canister call is an update or a query. + enum call-type { update, query } + + /// A file the host read on behalf of the plugin. + record file-input { + /// Path of the file as declared in the manifest (relative to + /// the canister directory). + name: string, + /// UTF-8 contents of the file. + content: string, + } + + /// Input passed by the runtime to the plugin's exec() export. + record sync-exec-input { + /// Textual principal of the canister being synced. + canister-id: string, + /// Name of the environment being synced (e.g. "production", "local"). + environment: string, + /// Directories declared in the manifest step's `dirs` setting. + /// The host preopens each entry via WASI; the plugin can traverse + /// them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). + dirs: list, + /// Files declared in the manifest step's `files` setting, read by + /// the host and passed inline. The plugin decides how to use them. + files: list, + /// Textual principal of the signing identity used for canister calls. + identity-principal: string, + /// Textual principal of the proxy canister, if one was configured via + /// `--proxy`. None when no proxy is in use. + proxy-canister-id: option, + } + + /// A request to call a method on the target canister. + record canister-call-request { + /// The canister method to call. + method: string, + /// Candid-encoded argument bytes. The plugin is responsible for + /// encoding; the host forwards these bytes unchanged. + arg: list, + /// Whether to perform an `update` or `query` call. + call-type: call-type, + /// When true, the call bypasses any proxy canister configured via + /// `--proxy`, going directly to the target canister. When false + /// (the default), update calls are routed through the proxy if one + /// is configured; query calls always go directly to the target + /// canister regardless of this flag. + direct: bool, + /// Cycles to attach to a proxied update call. Only meaningful when + /// `direct` is `false`, a proxy canister is configured, and + /// `call-type` is `update`; silently ignored for direct calls and + /// for query calls. + cycles: u64, + } +} + +/// The complete interface of a sync plugin. +world sync-plugin { + use types.{sync-exec-input, canister-call-request, file-input}; + + // ------------------------------------------------------------------------- + // Host functions (imports) — provided by icp-cli, called by the plugin + // ------------------------------------------------------------------------- + + /// Make an update or query call to the canister being synced. + /// The host always calls the canister from sync-exec-input.canister-id; + /// the plugin does not choose the target. + /// Returns the raw Candid-encoded response bytes on success or an error + /// message on failure. The plugin is responsible for decoding. + import canister-call: func(req: canister-call-request) -> result, string>; + + // The plugin's stdout is captured and shown as transient progress in + // the rolling step view of icp-cli; it is discarded when the step ends. + // + // The plugin's stderr is captured and shown in the rolling step view AND + // printed persistently after the step completes successfully. (On + // failure, the error message and the rolling-view dump already surface + // stderr, so it is not reprinted.) + // + // Use stdout for in-flight progress chatter the user doesn't need to see + // once the step is done. Use stderr for messages the user must still see + // after the step completes — warnings, summaries, deprecation notices. + + // ------------------------------------------------------------------------- + // Plugin exports — implemented by the plugin, called by the host + // ------------------------------------------------------------------------- + + /// Execute the sync plugin for the canister being synced. Returns an + /// error message on failure. + export exec: func(input: sync-exec-input) -> result<_, string>; +} diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 64fc8d11d..9e7ffb297 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -1,4 +1,4 @@ -package icp:sync-plugin@0.1.0; +package icp:sync-plugin@0.2.0; /// Types shared between the host runtime and sync plugins. interface types { diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock new file mode 100644 index 000000000..edb77b53d --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock @@ -0,0 +1,349 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "macro-string" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test-plugin-v1" +version = "0.1.0" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "wasm-encoder" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61fb705ce81adde29d2a8e99d87995e39a6e927358c91398f374474746070ef7" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e4c2aa916c425dcca61a6887d3e135acdee2c6d0ed51fd61c08d41ddaf62b1" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71cde4757396defafd25417cfb36aa3161027d06d865b0c24baaae229aac005d" +dependencies = [ + "bitflags", + "hashbrown 0.16.1", + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7607d30e7e5e8fd5a0695f7cb8b2128829e0bf9dca7a1fe8c4d6ed3ca1058fce" +dependencies = [ + "bitflags", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda3a4ce47c08d27f575d451a60102bab5251776abd0a7a323d1f038eb6339ab" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920a1c8c0f89397431db4900a7bf7c511b78e1b7068289fe812dc76e993f1491" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn 2.0.119", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "857a143d2373abfcd31ad946393efe775ed8c90a2a365ce73c61bf38f36a1000" +dependencies = [ + "anyhow", + "macro-string", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1936c26cb24b93dc36bf78fb5dc35c55cd37f66ecdc2d2663a717d9fb3ee951e" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd979042b5ff288607ccf3b314145435453f20fc67173195f91062d2289b204d" +dependencies = [ + "anyhow", + "hashbrown 0.16.1", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml new file mode 100644 index 000000000..25d44def3 --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] + +[package] +name = "test-plugin-v1" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = { version = "0.56", features = ["realloc"] } diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs new file mode 100644 index 000000000..b23d985e0 --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=../../../sync-plugin-v1.wit"); +} diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs new file mode 100644 index 000000000..24ebea1bf --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs @@ -0,0 +1,24 @@ +// A plugin built against the *legacy* (v0.1.0) interface, used to prove the host +// still loads and drives v0.1.0 plugins alongside v0.2.0 ones. It cannot choose +// a call target and never sees the canister ID table — that is the whole point. +wit_bindgen::generate!({ + world: "sync-plugin", + path: "../../../sync-plugin-v1.wit", +}); + +struct TestPluginV1; + +impl Guest for TestPluginV1 { + fn exec(input: SyncExecInput) -> Result<(), String> { + match input.environment.as_str() { + "error" => Err("deliberate v1 failure".to_string()), + "hello" => { + eprintln!("hello from v1"); + Ok(()) + } + _ => Ok(()), + } + } +} + +export!(TestPluginV1); diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index b586ac0d5..f8310deea 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -54,6 +54,8 @@ world sync-plugin { } ``` +The interface is versioned (currently `icp:sync-plugin@0.2.0`). icp-cli reads the version a plugin was built against from the component itself and drives it accordingly, so plugins built against the earlier `@0.1.0` interface — which could only call the canister being synced — continue to load unchanged. + The authoritative interface, including all record fields, lives in [`sync-plugin.wit`](https://github.com/dfinity/icp-cli/blob/main/crates/icp-sync-plugin/sync-plugin.wit) in the icp-cli repository. ### What the plugin receives — `sync-exec-input` From 5961dfa0db7f19444eb1970d31f449d25c179c29 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 08:36:05 -0700 Subject: [PATCH 02/51] fix comment --- crates/icp-sync-plugin/sync-plugin-v1.wit | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/icp-sync-plugin/sync-plugin-v1.wit b/crates/icp-sync-plugin/sync-plugin-v1.wit index 65c0283a9..5cb8ee2f6 100644 --- a/crates/icp-sync-plugin/sync-plugin-v1.wit +++ b/crates/icp-sync-plugin/sync-plugin-v1.wit @@ -1,8 +1,6 @@ -// Version 0.1.0 of the sync-plugin interface, preserved verbatim so the host -// can still load plugins built against it. New plugins should target the +// Version 0.1.0 of the sync-plugin interface. New plugins should target the // current interface in `sync-plugin.wit`; see that file for the up-to-date -// contract. The host tries the current interface first and falls back to this -// one, so both APIs are supported in parallel. +// contract. The host will load this version if the plugin imports it. package icp:sync-plugin@0.1.0; /// Types shared between the host runtime and sync plugins. From 62695ba9917e90ace0f35c745b36c8e4f7e38709 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 09:28:53 -0700 Subject: [PATCH 03/51] fix clippy (???) --- .../icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs index 24ebea1bf..6d38c71ea 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs @@ -1,6 +1,9 @@ +#![allow(clippy::too_many_arguments)] + // A plugin built against the *legacy* (v0.1.0) interface, used to prove the host // still loads and drives v0.1.0 plugins alongside v0.2.0 ones. It cannot choose // a call target and never sees the canister ID table — that is the whole point. + wit_bindgen::generate!({ world: "sync-plugin", path: "../../../sync-plugin-v1.wit", From db63f13ce53030a020aa638901b1eded41169c08 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 09:36:49 -0700 Subject: [PATCH 04/51] copilot --- crates/icp-sync-plugin/DESIGN.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 411376f93..dd54eb68f 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -12,7 +12,8 @@ invokes its `exec()` export during `icp sync` for a single canister. > - [Sync Plugins](../../docs/concepts/sync-plugins.md) — concept, WIT interface, sandbox, resource limits > - [Writing a Sync Plugin](../../docs/guides/writing-sync-plugins.md) — authoring guide (Rust) > - [Plugin Sync (Configuration Reference)](../../docs/reference/configuration.md) — `type: plugin` manifest fields -> - [`sync-plugin.wit`](sync-plugin.wit) — the interface, and the sole source of truth +> - [`sync-plugin.wit`](sync-plugin.wit) — the current interface (v0.2.0), and its source of truth +> - [`sync-plugin-v1.wit`](sync-plugin-v1.wit) — the frozen v0.1.0 interface, still loadable --- @@ -36,8 +37,8 @@ docs; the *reasons* behind those choices are recorded here. - **Logging via stdio, not a host import** — stdout/stderr are captured by the host and forwarded to the CLI. Plugins use normal print facilities. - **No generated bindings checked in** — `wasmtime::component::bindgen!` (host) - and `wit_bindgen::generate!` (guest) both run at build time from the WIT file, - which stays the single source of truth. + and `wit_bindgen::generate!` (guest) both run at build time from the WIT files, + which stay the source of truth for the interface they define. --- @@ -50,10 +51,12 @@ Host-side Component Model runtime for sync plugins. ``` crates/icp-sync-plugin/ src/ - lib.rs — public API: run_plugin(), RunPluginError - runtime.rs — wasmtime component setup, HostState, bindgen!, exec() call - sync-plugin.wit — WIT interface (source of truth) - Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, candid, camino, snafu, tokio + lib.rs — public API: run_plugin(), RunPluginError + runtime.rs — wasmtime component setup, HostState, bindgen!, exec() call + path.rs — declared-path safety checks (escapes_base, symlinks) + sync-plugin.wit — current WIT interface, v0.2.0 + sync-plugin-v1.wit — frozen WIT interface, v0.1.0 + Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, candid, camino, snafu, tokio, semver ``` Public function: From 67207f6392e347b6d3ac10c4a43438e03a69a7fb Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 04:51:42 -0700 Subject: [PATCH 05/51] Expose the project canister ID table to sync plugins --- crates/icp-cli/src/operations/sync.rs | 1 + crates/icp-sync-plugin/DESIGN.md | 33 ++- crates/icp-sync-plugin/src/lib.rs | 3 +- crates/icp-sync-plugin/src/runtime.rs | 297 ++++++++++--------------- crates/icp-sync-plugin/sync-plugin.wit | 26 ++- crates/icp/src/canister/sync/mod.rs | 5 + crates/icp/src/canister/sync/plugin.rs | 103 ++++++++- crates/icp/src/canister/sync/script.rs | 1 + docs/concepts/sync-plugins.md | 6 +- 9 files changed, 267 insertions(+), 208 deletions(-) diff --git a/crates/icp-cli/src/operations/sync.rs b/crates/icp-cli/src/operations/sync.rs index 18c77efcc..77ea04174 100644 --- a/crates/icp-cli/src/operations/sync.rs +++ b/crates/icp-cli/src/operations/sync.rs @@ -59,6 +59,7 @@ async fn sync_canister( &Params { path: canister_path.clone(), cid: canister_id, + name: canister_info.name.clone(), environment: environment.to_owned(), network: network.to_owned(), canister_ids: canister_ids.clone(), diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index dd54eb68f..0f636dd0b 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -31,6 +31,10 @@ docs; the *reasons* behind those choices are recorded here. from `sync-exec-input.canister-id`. There is deliberately no field for a different target, so the single-canister restriction is *structural* rather than a policy the plugin could bypass. +- **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes + the project's name→principal map for the environment, so a plugin can resolve + canister names it knows about. It is informational only; calling still + requires a declaration. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. @@ -62,21 +66,14 @@ crates/icp-sync-plugin/ Public function: ```rust -pub fn run_plugin( - wasm_path: Utf8PathBuf, - base_dir: Utf8PathBuf, - dirs: Vec, - files: Vec, - target_canister_id: Principal, - agent: Agent, - proxy: Option, - identity_principal: Principal, - environment: String, - compute_limit_secs: u64, - stdio: Option>, -) -> Result, RunPluginError> +pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPluginError> ``` +`PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `dirs`, `files`, +`target_canister_id` (the canister being synced), `agent`, `proxy`, +`identity_principal`, `environment`, `compute_limit_secs`, and the exposed +`canister_ids` table, plus `stdio`. + `dirs` and `files` are the manifest-relative path strings, straight from the adapter. The runtime owns *all* filesystem access anchored at `base_dir`: it preopens each `dir` from `base_dir.join(dir)` and reads each `file` from @@ -188,7 +185,9 @@ pub struct Adapter { ### `crates/icp/src/canister/sync/plugin.rs` Resolves the wasm (local read or remote HTTP fetch into the package cache), -verifies sha256, then calls `icp_sync_plugin::run_plugin(...)`, forwarding the -manifest's `dirs`/`files` strings unchanged. The runtime — not the CLI — opens -those paths and enforces the path-safety checks, so the CLI no longer touches -the plugin's input files itself. +verifies sha256, builds the exposed canister ID table, then calls +`icp_sync_plugin::run_plugin(...)` with a `PluginInvocation`. The runtime — not +the CLI — opens the declared paths and enforces the path-safety checks, so the +CLI no longer touches the plugin's input files itself. `exposed_canister_ids` +adds a bare-local-name duplicate for every canister in the same subproject as +the one being synced. diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index dfffddc4c..84c5f4e41 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -2,5 +2,6 @@ mod path; mod runtime; pub use runtime::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, RunPluginError, run_plugin, + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, + run_plugin, }; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 27091fc34..2cf39ad3b 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1,4 +1,5 @@ // Host-side Component Model runtime for sync plugins. +use std::collections::BTreeMap; use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex as StdMutex; @@ -55,7 +56,7 @@ mod v1 { }); } -use v2::icp::sync_plugin::types::CallType; +use v2::icp::sync_plugin::types::{CallType, CanisterIdEntry}; // HostState holds everything the plugin's import functions need. struct HostState { @@ -328,20 +329,53 @@ fn detect_plugin_abi( } } -#[allow(clippy::too_many_arguments)] -pub fn run_plugin( - wasm_path: Utf8PathBuf, - base_dir: Utf8PathBuf, - dirs: Vec, - files: Vec, - target_canister_id: Principal, - agent: Agent, - proxy: Option, - identity_principal: Principal, - environment: String, - compute_limit_secs: u64, - stdio: Option>, -) -> Result, RunPluginError> { +/// Everything [`run_plugin`] needs to load and drive one sync plugin. +#[derive(Debug)] +pub struct PluginInvocation { + /// On-disk path to the plugin's wasm component. + pub wasm_path: Utf8PathBuf, + /// Directory the declared `dirs`/`files` are anchored at (the canister dir). + pub base_dir: Utf8PathBuf, + /// Manifest-relative directories to preopen read-only. + pub dirs: Vec, + /// Manifest-relative files to read and pass inline. + pub files: Vec, + /// The canister being synced. + pub target_canister_id: Principal, + /// Agent used for canister calls. + pub agent: Agent, + /// Proxy canister to route update calls through, if configured. + pub proxy: Option, + /// Signing identity principal, surfaced to the plugin. + pub identity_principal: Principal, + /// Name of the environment being synced. + pub environment: String, + /// Pure-wasm compute-time budget in seconds. + pub compute_limit_secs: u64, + /// The project's canister ID table for this environment, as exposed to the + /// plugin. Same-project canisters appear both under their fully-qualified + /// key and their bare local name (see the WIT `canister-id-entry` docs). + pub canister_ids: BTreeMap, + /// Channel for live rolling-view output, if any. + pub stdio: Option>, +} + +pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPluginError> { + let PluginInvocation { + wasm_path, + base_dir, + dirs, + files, + target_canister_id, + agent, + proxy, + identity_principal, + environment, + compute_limit_secs, + canister_ids, + stdio, + } = invocation; + let mut config = Config::new(); config.wasm_component_model(true); config.max_wasm_stack(MAX_WASM_STACK); @@ -485,6 +519,13 @@ pub fn run_plugin( .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, + canister_ids: canister_ids + .into_iter() + .map(|(name, id)| CanisterIdEntry { + name, + id: id.to_text(), + }) + .collect(), }; plugin.call_exec(&mut store, &input) } @@ -702,25 +743,34 @@ mod tests { Principal::anonymous() } + /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister + /// and identity, no proxy, an empty canister ID table, the default compute + /// limit, and the current directory as the base. Individual tests override + /// the few fields they care about. + fn invocation(wasm_path: &str, environment: &str) -> PluginInvocation { + PluginInvocation { + wasm_path: wasm_path.into(), + base_dir: ".".into(), + dirs: vec![], + files: vec![], + target_canister_id: anon(), + agent: dummy_agent(), + proxy: None, + identity_principal: anon(), + environment: environment.to_string(), + compute_limit_secs: DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, + canister_ids: BTreeMap::new(), + stdio: None, + } + } + // ------------------------------------------------------------------------- // Error-path tests — no fixture WASM needed // ------------------------------------------------------------------------- #[test] fn load_component_error_on_missing_file() { - let result = run_plugin( - "nonexistent.wasm".into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); + let result = run_plugin(invocation("nonexistent.wasm", "test")); assert!(matches!(result, Err(RunPluginError::LoadComponent { .. }))); } @@ -746,20 +796,12 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec!["nonexistent_dir".to_string()], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::PreopenDir { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.dirs = vec!["nonexistent_dir".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::PreopenDir { .. }) + )); } #[cfg(unix)] @@ -774,20 +816,13 @@ mod tests { std::fs::create_dir_all(base.join("real")).expect("create real dir"); symlink(base.join("real"), base.join("link")).expect("create symlink"); - let result = run_plugin( - wasm_path.into(), - base.to_path_buf(), - vec!["link".to_string()], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::SymlinkDir { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.base_dir = base.to_path_buf(); + inv.dirs = vec!["link".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::SymlinkDir { .. }) + )); } #[test] @@ -795,20 +830,12 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec!["nonexistent_file.txt".to_string()], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::ReadFile { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.files = vec!["nonexistent_file.txt".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::ReadFile { .. }) + )); } #[cfg(unix)] @@ -823,20 +850,13 @@ mod tests { std::fs::write(base.join("real.txt"), b"data").expect("write real file"); symlink(base.join("real.txt"), base.join("link.txt")).expect("create symlink"); - let result = run_plugin( - wasm_path.into(), - base.to_path_buf(), - vec![], - vec!["link.txt".to_string()], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::SymlinkFile { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.base_dir = base.to_path_buf(); + inv.files = vec!["link.txt".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::SymlinkFile { .. }) + )); } #[test] @@ -844,20 +864,7 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "ok".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(result.is_ok()); + assert!(run_plugin(invocation(wasm_path, "ok")).is_ok()); } #[test] @@ -865,21 +872,8 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "error".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); assert!(matches!( - result, + run_plugin(invocation(wasm_path, "error")), Err(RunPluginError::PluginFailed { ref message }) if message == "deliberate failure" )); } @@ -891,20 +885,9 @@ mod tests { }; // The "spin" fixture busy-loops forever; a 1-second limit keeps the // test fast while still exercising the epoch-interruption trap. - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "spin".to_string(), - 1, - None, - ); - let err = result.expect_err("spinning plugin should hit the compute limit"); + let mut inv = invocation(wasm_path, "spin"); + inv.compute_limit_secs = 1; + let err = run_plugin(inv).expect_err("spinning plugin should hit the compute limit"); // The trap surfaces through the CallExec source chain, so walk it and // assert the message names both the limit and the override env var. let mut chain = err.to_string(); @@ -926,19 +909,9 @@ mod tests { }; let (tx, mut rx) = tokio::sync::mpsc::channel::(16); let result = tokio::task::block_in_place(|| { - run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "print".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - Some(tx), - ) + let mut inv = invocation(wasm_path, "print"); + inv.stdio = Some(tx); + run_plugin(inv) }); assert!(result.is_ok()); let msg = rx.try_recv().expect("expected stdout message on channel"); @@ -952,36 +925,10 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_V1_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "ok".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(result.is_ok()); + assert!(run_plugin(invocation(wasm_path, "ok")).is_ok()); // Its error surface flows through the same machinery as v0.2.0 plugins. - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "error".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); assert!(matches!( - result, + run_plugin(invocation(wasm_path, "error")), Err(RunPluginError::PluginFailed { ref message }) if message == "deliberate v1 failure" )); } @@ -993,19 +940,9 @@ mod tests { }; let (tx, mut rx) = tokio::sync::mpsc::channel::(16); let result = tokio::task::block_in_place(|| { - run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "hello".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - Some(tx), - ) + let mut inv = invocation(wasm_path, "hello"); + inv.stdio = Some(tx); + run_plugin(inv) }); let lines = result.expect("plugin should succeed"); assert_eq!(lines, vec!["hello".to_string()]); diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 9e7ffb297..8212e5090 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -14,6 +14,24 @@ interface types { content: string, } + /// An entry in the project's canister ID mapping table: a canister name + /// and the textual principal it resolves to in the environment being synced. + record canister-id-entry { + /// The canister's fully-qualified project key: the subproject it belongs + /// to (a path relative to the app root) joined to its local name by a + /// single colon, e.g. "services/open-accounts:backend". A canister + /// defined directly in the project being synced has no subproject prefix + /// and appears as its bare local name, e.g. "backend". + /// + /// Every canister in the same subproject as the canister being synced is + /// additionally listed under its bare local name (a duplicate entry with + /// the same `id`), so a plugin can address a sibling by the same local + /// name the manifest uses. + name: string, + /// Textual principal the name resolves to for this environment. + id: string, + } + /// Input passed by the runtime to the plugin's exec() export. record sync-exec-input { /// Textual principal of the canister being synced. @@ -32,6 +50,12 @@ interface types { /// Textual principal of the proxy canister, if one was configured via /// `--proxy`. None when no proxy is in use. proxy-canister-id: option, + /// Name→principal mapping for every named canister in the project for + /// the environment being synced, sorted by name. Informational: the + /// plugin may use it to resolve canister names it knows about. Being + /// listed here does not grant permission to call a canister — that + /// still requires declaring it as a dependency (see `call-target`). + canister-ids: list, } /// A request to call a method on the target canister. @@ -59,7 +83,7 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, file-input}; + use types.{sync-exec-input, canister-call-request, canister-id-entry, file-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index 519a75b71..93dfdd0bd 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -19,6 +19,10 @@ use script::{HostScripts, ScriptInvocation, ScriptRunError, ScriptRunner}; pub struct Params { pub path: PathBuf, pub cid: Principal, + /// Fully-qualified store key of the canister being synced (e.g. `backend`, + /// or `services/open-crm:backend` for a dependency canister). Its namespace + /// prefix identifies which other canisters are in the same subproject. + pub name: String, /// Name of the environment being synced (e.g. "local", "production"). /// Passed to sync plugin steps via `SyncExecInput`. pub environment: String, @@ -165,6 +169,7 @@ mod tests { let params = Params { path: "/work/backend".into(), cid, + name: "backend".to_owned(), environment: "production".to_owned(), network: "ic".to_owned(), canister_ids: BTreeMap::from([( diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 97056d64d..49c636c35 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,8 +1,11 @@ +use std::collections::BTreeMap; + use camino::Utf8PathBuf; use candid::Principal; use ic_agent::Agent; use icp_sync_plugin::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, RunPluginError, run_plugin, + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, + run_plugin, }; use snafu::prelude::*; use tokio::sync::mpsc::Sender; @@ -91,7 +94,10 @@ pub(super) async fn sync( let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); - // 3. Run the plugin (blocking call — signal Tokio that this thread will block). + // 3. Build the canister ID table exposed to the plugin. + let canister_ids = exposed_canister_ids(params); + + // 4. Run the plugin (blocking call — signal Tokio that this thread will block). let identity_principal = agent .get_principal() .map_err(|err| PluginError::GetIdentityPrincipal { err })?; @@ -101,23 +107,44 @@ pub(super) async fn sync( let stdio_clone = stdio.clone(); tokio::task::block_in_place(|| { - run_plugin( + run_plugin(PluginInvocation { wasm_path, base_dir, dirs, files, - params.cid, - agent_clone, + target_canister_id: params.cid, + agent: agent_clone, proxy, identity_principal, - environment_owned, + environment: environment_owned, compute_limit_secs, - stdio_clone, - ) + canister_ids, + stdio: stdio_clone, + }) }) .context(RunSnafu) } +/// The canister ID table exposed to a sync plugin: every named canister in the +/// project, plus — for canisters in the same subproject as the one being synced +/// — a duplicate entry under the bare local name. A store key is +/// `:` for a dependency canister and a bare local name for a +/// canister defined directly in the project (see the WIT `canister-id-entry` +/// docs), so the syncing canister's namespace is the prefix of its own key. +fn exposed_canister_ids(params: &Params) -> BTreeMap { + let syncing_namespace = params.name.split_once(':').map(|(namespace, _)| namespace); + + let mut table = params.canister_ids.clone(); + for (key, id) in ¶ms.canister_ids { + if let Some((namespace, local)) = key.split_once(':') + && Some(namespace) == syncing_namespace + { + table.entry(local.to_owned()).or_insert(*id); + } + } + table +} + #[cfg(test)] mod tests { use super::*; @@ -140,4 +167,64 @@ mod tests { ); } } + + fn principal(byte: u8) -> Principal { + Principal::from_slice(&[byte; 4]) + } + + fn params_named(name: &str, ids: &[(&str, Principal)]) -> Params { + Params { + path: "/work".into(), + cid: principal(0), + name: name.to_owned(), + environment: "demo".to_owned(), + network: "ic".to_owned(), + canister_ids: ids.iter().map(|(n, p)| ((*n).to_owned(), *p)).collect(), + proxy: None, + } + } + + /// Canisters sharing the syncing canister's subproject are additionally + /// exposed under their bare local name; canisters in other subprojects are + /// not. + #[test] + fn exposed_ids_add_bare_names_for_same_subproject() { + let backend = principal(1); + let frontend = principal(2); + let foreign = principal(3); + let params = params_named( + "services/open-accounts:backend", + &[ + ("services/open-accounts:backend", backend), + ("services/open-accounts:frontend", frontend), + ("services/open-crm:backend", foreign), + ], + ); + + let table = exposed_canister_ids(¶ms); + + // Same-subproject canisters gain a bare-local duplicate... + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + // ...while the fully-qualified keys are still present for everyone. + assert_eq!( + table.get("services/open-accounts:frontend"), + Some(&frontend) + ); + assert_eq!(table.get("services/open-crm:backend"), Some(&foreign)); + // The other subproject's canister is not reachable by a bare name; the + // bare "backend" belongs to the syncing canister's own subproject. + assert_eq!(table.get("backend"), Some(&backend)); + } + + /// A single-project layout keys canisters by bare local name already, so no + /// duplicates are added. + #[test] + fn exposed_ids_unchanged_without_a_subproject() { + let backend = principal(1); + let params = params_named("backend", &[("backend", backend)]); + let table = exposed_canister_ids(¶ms); + assert_eq!(table.len(), 1); + assert_eq!(table.get("backend"), Some(&backend)); + } } diff --git a/crates/icp/src/canister/sync/script.rs b/crates/icp/src/canister/sync/script.rs index 7c9d741d7..e26d73171 100644 --- a/crates/icp/src/canister/sync/script.rs +++ b/crates/icp/src/canister/sync/script.rs @@ -145,6 +145,7 @@ mod tests { Params { path: "/work/backend".into(), cid: principal(1), + name: "backend".to_owned(), environment: "production".to_owned(), network: "ic".to_owned(), canister_ids: canister_ids diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index f8310deea..30c1b0c89 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work against a single canister. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced, and lets it make canister calls and read declared files — nothing more. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work against a single canister. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -35,6 +35,7 @@ icp sync ├─ exec(sync-exec-input) called │ canister-id = │ identity-principal = + │ canister-ids = │ dirs / files = what you declared in the manifest │ └─ plugin makes canister-call(...) to the target canister (× N) @@ -68,6 +69,9 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `files` | The files you declared in `files:`, each as a `(name, content)` pair read by the host | | `identity-principal` | Textual principal of the signing identity used for canister calls | | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | +| `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | + +Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister in the project being synced, or a `subproject:canister` key for a dependency canister. Canisters in the same subproject as the one being synced are additionally listed under their bare local name. ### Calling the canister — `canister-call` From 3b31c116629574a74f51843ee94afe29d0c52370 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 10:03:15 -0700 Subject: [PATCH 06/51] Document the actual canister-ids restriction The `canister-ids` table is informational; `canister-call` always targets the canister being synced. The field and rationale docs described a call-target / dependency-declaration permission mechanism that does not exist in this interface, which could mislead plugin authors into expecting they can call other canisters from the table. State the real restriction instead. --- crates/icp-sync-plugin/DESIGN.md | 5 +++-- crates/icp-sync-plugin/sync-plugin.wit | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 0f636dd0b..2a5787b6e 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -33,8 +33,9 @@ docs; the *reasons* behind those choices are recorded here. than a policy the plugin could bypass. - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve - canister names it knows about. It is informational only; calling still - requires a declaration. + canister names it knows about. It is informational only: `canister-call` + still targets the canister being synced, so the table grants no ability to + call other canisters. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 8212e5090..baa2d582c 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -53,8 +53,8 @@ interface types { /// Name→principal mapping for every named canister in the project for /// the environment being synced, sorted by name. Informational: the /// plugin may use it to resolve canister names it knows about. Being - /// listed here does not grant permission to call a canister — that - /// still requires declaring it as a dependency (see `call-target`). + /// listed here does not let the plugin call a canister — the + /// `canister-call` import always targets the canister being synced. canister-ids: list, } From 614c9bad1985570205dd64ee9e9c98defaa69610 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 11:39:46 -0700 Subject: [PATCH 07/51] copilot --- crates/icp-sync-plugin/sync-plugin.wit | 7 ++-- crates/icp/src/canister/sync/plugin.rs | 57 ++++++++++++++++++++++++-- docs/concepts/sync-plugins.md | 2 +- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index baa2d582c..e201568b5 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -20,13 +20,14 @@ interface types { /// The canister's fully-qualified project key: the subproject it belongs /// to (a path relative to the app root) joined to its local name by a /// single colon, e.g. "services/open-accounts:backend". A canister - /// defined directly in the project being synced has no subproject prefix - /// and appears as its bare local name, e.g. "backend". + /// defined directly in the app root has no subproject prefix and appears + /// as its bare local name, e.g. "backend". /// /// Every canister in the same subproject as the canister being synced is /// additionally listed under its bare local name (a duplicate entry with /// the same `id`), so a plugin can address a sibling by the same local - /// name the manifest uses. + /// name the manifest uses. A bare name always means the sibling, so an + /// app-root canister sharing that local name is not listed. name: string, /// Textual principal the name resolves to for this environment. id: string, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 49c636c35..22f749724 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -129,17 +129,22 @@ pub(super) async fn sync( /// project, plus — for canisters in the same subproject as the one being synced /// — a duplicate entry under the bare local name. A store key is /// `:` for a dependency canister and a bare local name for a -/// canister defined directly in the project (see the WIT `canister-id-entry` +/// canister defined directly in the app root (see the WIT `canister-id-entry` /// docs), so the syncing canister's namespace is the prefix of its own key. +/// +/// A local name never contains a colon but a subproject directory may, so keys +/// split on their *last* colon. The bare-name aliases take precedence over an +/// app-root canister of the same local name: a plugin resolving a bare name is +/// naming what the syncing canister's own manifest calls it. fn exposed_canister_ids(params: &Params) -> BTreeMap { - let syncing_namespace = params.name.split_once(':').map(|(namespace, _)| namespace); + let syncing_namespace = params.name.rsplit_once(':').map(|(namespace, _)| namespace); let mut table = params.canister_ids.clone(); for (key, id) in ¶ms.canister_ids { - if let Some((namespace, local)) = key.split_once(':') + if let Some((namespace, local)) = key.rsplit_once(':') && Some(namespace) == syncing_namespace { - table.entry(local.to_owned()).or_insert(*id); + table.insert(local.to_owned(), *id); } } table @@ -217,6 +222,50 @@ mod tests { assert_eq!(table.get("backend"), Some(&backend)); } + /// An app-root canister sharing a local name with a sibling of the syncing + /// canister does not keep the bare name: the syncing subproject's own + /// canister is what that name means to the plugin. + #[test] + fn exposed_ids_sibling_alias_overrides_the_app_root_name() { + let root_backend = principal(1); + let sibling_backend = principal(2); + let params = params_named( + "services/open-accounts:frontend", + &[ + ("backend", root_backend), + ("services/open-accounts:backend", sibling_backend), + ("services/open-accounts:frontend", principal(3)), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("backend"), Some(&sibling_backend)); + // The app-root canister's only key was that bare name, so it drops out + // of the table entirely rather than answering to a sibling's name. + assert!(!table.values().any(|id| *id == root_backend)); + } + + /// A subproject directory may itself contain a colon, so keys are split on + /// their last one — the same rule bundling uses. + #[test] + fn exposed_ids_split_subproject_prefix_at_the_last_colon() { + let backend = principal(1); + let frontend = principal(2); + let params = params_named( + "services/odd:name:backend", + &[ + ("services/odd:name:backend", backend), + ("services/odd:name:frontend", frontend), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + } + /// A single-project layout keys canisters by bare local name already, so no /// duplicates are added. #[test] diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 30c1b0c89..a7a6eb3f9 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -71,7 +71,7 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | | `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | -Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister in the project being synced, or a `subproject:canister` key for a dependency canister. Canisters in the same subproject as the one being synced are additionally listed under their bare local name. +Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister that came from a dependency. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. ### Calling the canister — `canister-call` From 08491a1e8612328ea8c519b57254b08149a41cf4 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 11:13:42 -0700 Subject: [PATCH 08/51] Fix docs --- crates/icp-sync-plugin/src/runtime.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 2cf39ad3b..50406b383 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -39,9 +39,8 @@ use wasmtime_wasi::{DirPerms, FilePerms}; // module so their generated type names don't collide. `run_plugin` reads the // interface version from the component's own metadata (see `detect_plugin_abi`) // and drives it through the matching module, so plugins built against either -// interface load. The two interfaces are currently structurally identical; the -// split exists so later breaking changes to the current interface can land -// without dropping support for already-built plugins. +// interface load. The split exists so breaking changes to the current interface +// can land without dropping support for already-built plugins. mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", From 2d2fa8f98d1dafb7dcd04421586c5cda4139b784 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 04:52:18 -0700 Subject: [PATCH 09/51] Implement cross-canister plugin targeting --- crates/icp-cli/src/operations/bundle.rs | 1 + crates/icp-sync-plugin/DESIGN.md | 49 ++++-- crates/icp-sync-plugin/src/lib.rs | 4 +- crates/icp-sync-plugin/src/runtime.rs | 186 ++++++++++++++++++--- crates/icp-sync-plugin/sync-plugin.wit | 35 +++- crates/icp/src/canister/sync/plugin.rs | 96 ++++++++++- crates/icp/src/manifest/adapter/plugin.rs | 51 ++++++ crates/icp/src/manifest/canister.rs | 2 + docs/concepts/sync-plugins.md | 19 ++- docs/guides/writing-sync-plugins.md | 3 +- docs/reference/configuration.md | 8 +- docs/schemas/canister-yaml-schema.json | 10 ++ docs/schemas/icp-yaml-schema.json | 10 ++ examples/icp-sync-plugin/plugin/src/lib.rs | 2 + 14 files changed, 411 insertions(+), 65 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 6786fc7d5..4b07c5f5d 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -788,6 +788,7 @@ async fn prepare_plugin_step( sha256: Some(plugin_sha256), dirs: bundle_dirs, files: bundle_files, + canisters: None, })) } diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 2a5787b6e..40646741f 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -27,10 +27,13 @@ docs; the *reasons* behind those choices are recorded here. - **Raw Candid bytes at the boundary** — `canister-call-request.arg` is `list`. The plugin owns Candid encoding/decoding; the host forwards bytes unchanged. This keeps the host free of any per-canister type knowledge. -- **`canister-call` takes no canister ID** — the host always calls the canister - from `sync-exec-input.canister-id`. There is deliberately no field for a - different target, so the single-canister restriction is *structural* rather - than a policy the plugin could bypass. +- **`canister-call` takes an explicit `target`** — the plugin selects the + canister being synced (`host`) or a canister it declared as a dependency, by + name or principal. The host resolves the target and *enforces* the + declaration: a target absent from the step's `canisters:` list is rejected + without a call. (In the earlier `@0.1.0` interface `canister-call` had no + target and always reached the canister being synced; see *Interface + versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only: `canister-call` @@ -71,9 +74,12 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin ``` `PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `dirs`, `files`, -`target_canister_id` (the canister being synced), `agent`, `proxy`, -`identity_principal`, `environment`, `compute_limit_secs`, and the exposed -`canister_ids` table, plus `stdio`. +`host_canister_id` (the canister being synced), `agent`, `proxy`, +`identity_principal`, `environment`, `compute_limit_secs`, the exposed +`canister_ids` table, the `callable: CallableCanisters` enforcement set, and +`stdio`. The CLI resolves the manifest's declared `canisters:` into +`CallableCanisters` before calling; this crate stays free of any manifest +knowledge. `dirs` and `files` are the manifest-relative path strings, straight from the adapter. The runtime owns *all* filesystem access anchored at `base_dir`: it @@ -112,7 +118,8 @@ mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugi mod v1 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugin-v1.wit" }); } struct HostState { - target_canister_id: Principal, + host_canister_id: Principal, + callable: CallableCanisters, // by_name + by_id, from the manifest agent: Arc, proxy: Option, wasi_ctx: wasmtime_wasi::WasiCtx, @@ -121,16 +128,18 @@ struct HostState { } // Implemented for both v1::SyncPluginImports and v2::SyncPluginImports; both -// delegate to one shared `do_canister_call(...)`. +// delegate to one shared `do_canister_call(target, ...)`. ``` `HostState` implements `WasiView` so wasmtime_wasi can access the WASI context. `canister_call` uses `tokio::runtime::Handle::current().block_on(...)` because the caller already wraps the synchronous `run_plugin` in -`tokio::task::block_in_place`. Both interface versions call the canister being -synced. When a proxy is configured and the call is a non-`direct` update, it is -encoded as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise -it goes straight to the target via `ic-agent`. +`tokio::task::block_in_place`. For a v0.2.0 plugin the target is resolved from +the request's `call-target` by `resolve_call_target`, which enforces the +`callable` set; for a v0.1.0 plugin the target is always `host_canister_id`. +When a proxy is configured and the call is a non-`direct` update, it is encoded +as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise it goes +straight to the resolved target via `ic-agent`. ### Interface versioning (parallel v0.1.0 / v0.2.0 support) @@ -174,21 +183,27 @@ Deserializes the `canister.yaml` fields into: ```rust pub struct Adapter { - pub source: SourceField, // path: or url: + pub source: SourceField, // path: or url: pub sha256: Option, pub dirs: Option>, pub files: Option>, + pub canisters: Option>, // extra callable canisters } ``` -`Deserialize` is hand-written to reject a `url` source without a `sha256`. +`CanisterRef` is an untagged `Principal | Name` (anything that parses as a +principal is one; everything else is a name), written in the manifest as a plain +string. `Deserialize` is hand-written to reject a `url` source without a +`sha256`. ### `crates/icp/src/canister/sync/plugin.rs` Resolves the wasm (local read or remote HTTP fetch into the package cache), -verifies sha256, builds the exposed canister ID table, then calls +verifies sha256, builds the exposed canister ID table and the `CallableCanisters` +enforcement set (resolving `canisters:` against the project's IDs), then calls `icp_sync_plugin::run_plugin(...)` with a `PluginInvocation`. The runtime — not the CLI — opens the declared paths and enforces the path-safety checks, so the CLI no longer touches the plugin's input files itself. `exposed_canister_ids` adds a bare-local-name duplicate for every canister in the same subproject as -the one being synced. +the one being synced; `resolve_callable` fails the step if a declared dependency +name does not resolve. diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index 84c5f4e41..053be28a3 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -2,6 +2,6 @@ mod path; mod runtime; pub use runtime::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, - run_plugin, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + PluginInvocation, RunPluginError, run_plugin, }; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 50406b383..f45ece92c 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1,5 +1,5 @@ // Host-side Component Model runtime for sync plugins. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex as StdMutex; @@ -39,8 +39,7 @@ use wasmtime_wasi::{DirPerms, FilePerms}; // module so their generated type names don't collide. `run_plugin` reads the // interface version from the component's own metadata (see `detect_plugin_abi`) // and drives it through the matching module, so plugins built against either -// interface load. The split exists so breaking changes to the current interface -// can land without dropping support for already-built plugins. +// interface load. mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", @@ -55,11 +54,62 @@ mod v1 { }); } -use v2::icp::sync_plugin::types::{CallType, CanisterIdEntry}; +use v2::icp::sync_plugin::types::{CallTarget, CallType, CanisterIdEntry}; + +/// The canisters a sync plugin is permitted to call, beyond the canister being +/// synced (which is always reachable via [`CallTarget::Host`]). +/// +/// Built by the CLI from the plugin step's declared `canisters` dependencies, +/// resolved against the project's canister ID table. Keeping the resolution on +/// the CLI side keeps this runtime crate free of any manifest knowledge. +#[derive(Clone, Debug, Default)] +pub struct CallableCanisters { + /// Dependencies callable by name ([`CallTarget::Name`]). Maps the name — as + /// it appears in the canister ID table — to the principal it resolves to. + pub by_name: BTreeMap, + /// Every principal callable by [`CallTarget::Id`]. Includes the principals + /// of the `by_name` entries, so an author may target the same canister + /// either way. + pub by_id: BTreeSet, +} + +/// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing +/// that the plugin declared it as a dependency. The canister being synced +/// (`host`) is always permitted. +fn resolve_call_target( + target: &CallTarget, + host_canister_id: Principal, + callable: &CallableCanisters, +) -> Result { + match target { + CallTarget::Host => Ok(host_canister_id), + CallTarget::Name(name) => callable.by_name.get(name).copied().ok_or_else(|| { + format!( + "plugin is not permitted to call canister '{name}': declare it in the sync step's \ + `canisters` list to allow it" + ) + }), + CallTarget::Id(text) => { + let principal = Principal::from_text(text) + .map_err(|e| format!("invalid target principal '{text}': {e}"))?; + if principal == host_canister_id || callable.by_id.contains(&principal) { + Ok(principal) + } else { + Err(format!( + "plugin is not permitted to call canister '{principal}': declare it in the \ + sync step's `canisters` list to allow it" + )) + } + } + } +} // HostState holds everything the plugin's import functions need. struct HostState { - target_canister_id: Principal, + /// The canister being synced — the target of [`CallTarget::Host`] calls. + host_canister_id: Principal, + /// Canisters the plugin declared as dependencies and may also call. + callable: CallableCanisters, agent: Arc, /// Proxy canister to route update calls through, if configured. proxy: Option, @@ -84,10 +134,12 @@ impl wasmtime_wasi::WasiView for HostState { } impl HostState { - /// Perform a canister call to the canister being synced. Shared by both - /// interface versions. + /// Perform a canister call to an already-resolved target principal. Shared + /// by both interface versions: the v0.1.0 import always passes the canister + /// being synced; the v0.2.0 import passes the resolved `call-target`. fn do_canister_call( &mut self, + target: Principal, method: String, arg_bytes: Vec, call_type: CallType, @@ -96,7 +148,6 @@ impl HostState { ) -> Result, String> { use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; - let cid = self.target_canister_id; let agent = Arc::clone(&self.agent); let proxy = if direct { None } else { self.proxy }; @@ -108,7 +159,7 @@ impl HostState { CallType::Update => { if let Some(proxy_cid) = proxy { let proxy_args = ProxyArgs { - canister_id: cid, + canister_id: target, method: method.clone(), args: arg_bytes, cycles: candid::Nat::from(cycles), @@ -128,14 +179,14 @@ impl HostState { } } else { agent - .update(&cid, &method) + .update(&target, &method) .with_arg(arg_bytes) .await .map_err(|e| format!("canister call failed: {e}")) } } CallType::Query => agent - .query(&cid, &method) + .query(&target, &method) .with_arg(arg_bytes) .call() .await @@ -151,7 +202,7 @@ impl HostState { } } -// -- v0.2.0 interface. --------------------------------------------------------- +// -- v0.2.0 interface: the plugin chooses the target via `call-target`. -------- // `types::Host` is an empty marker trait generated for the `types` interface. impl v2::icp::sync_plugin::types::Host for HostState {} @@ -161,11 +212,19 @@ impl v2::SyncPluginImports for HostState { &mut self, req: v2::icp::sync_plugin::types::CanisterCallRequest, ) -> Result, String> { - self.do_canister_call(req.method, req.arg, req.call_type, req.direct, req.cycles) + let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; + self.do_canister_call( + target, + req.method, + req.arg, + req.call_type, + req.direct, + req.cycles, + ) } } -// -- v0.1.0 interface. --------------------------------------------------------- +// -- v0.1.0 interface: calls always go to the canister being synced. ----------- impl v1::icp::sync_plugin::types::Host for HostState {} @@ -174,12 +233,16 @@ impl v1::SyncPluginImports for HostState { &mut self, req: v1::icp::sync_plugin::types::CanisterCallRequest, ) -> Result, String> { + // The legacy interface has no target field; always call the host canister. + let target = self.host_canister_id; // v1's `call-type` is a distinct generated enum; map it to the shared one. let call_type = match req.call_type { v1::icp::sync_plugin::types::CallType::Update => CallType::Update, v1::icp::sync_plugin::types::CallType::Query => CallType::Query, }; - self.do_canister_call(req.method, req.arg, call_type, req.direct, req.cycles) + self.do_canister_call( + target, req.method, req.arg, call_type, req.direct, req.cycles, + ) } } @@ -265,9 +328,11 @@ pub enum RunPluginError { /// Which version of the sync-plugin interface a component was built against. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum PluginAbi { - /// Current interface (`icp:sync-plugin@0.2.x`). + /// Current interface (`icp:sync-plugin@0.2.x`): `canister-call` chooses a + /// target and `sync-exec-input` carries the canister ID table. V2, - /// Legacy interface (`icp:sync-plugin@0.1.x`). + /// Legacy interface (`icp:sync-plugin@0.1.x`): calls always reach the + /// canister being synced. V1, } @@ -339,8 +404,8 @@ pub struct PluginInvocation { pub dirs: Vec, /// Manifest-relative files to read and pass inline. pub files: Vec, - /// The canister being synced. - pub target_canister_id: Principal, + /// The canister being synced. Reachable via `call-target::host`. + pub host_canister_id: Principal, /// Agent used for canister calls. pub agent: Agent, /// Proxy canister to route update calls through, if configured. @@ -355,6 +420,10 @@ pub struct PluginInvocation { /// plugin. Same-project canisters appear both under their fully-qualified /// key and their bare local name (see the WIT `canister-id-entry` docs). pub canister_ids: BTreeMap, + /// Canisters the plugin declared as dependencies and may call, beyond the + /// canister being synced. Ignored by v0.1.0 plugins, which can only reach + /// the canister being synced. + pub callable: CallableCanisters, /// Channel for live rolling-view output, if any. pub stdio: Option>, } @@ -365,13 +434,14 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin base_dir, dirs, files, - target_canister_id, + host_canister_id, agent, proxy, identity_principal, environment, compute_limit_secs, canister_ids, + callable, stdio, } = invocation; @@ -463,7 +533,8 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let epoch_extension = Arc::new(AtomicU64::new(0)); let host_state = HostState { - target_canister_id, + host_canister_id, + callable, agent: Arc::new(agent), proxy, wasi_ctx: wasi_builder.build(), @@ -485,13 +556,15 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin } }); - let canister_id_text = target_canister_id.to_text(); + let canister_id_text = host_canister_id.to_text(); let identity_text = identity_principal.to_text(); let proxy_text = proxy.map(|p| p.to_text()); // Which interface the plugin was built against is read from the component's // own declared metadata (see `detect_plugin_abi`) rather than probed by - // trial instantiation, then driven through the matching bindgen world. + // trial instantiation. Both are served in parallel: v0.2.0 plugins choose a + // call target and receive the canister ID table; v0.1.0 plugins get neither + // and always call the canister being synced. let call_result = match detect_plugin_abi(&engine, &component, &wasm_path)? { PluginAbi::V2 => { let mut linker: Linker = Linker::new(&engine); @@ -743,7 +816,7 @@ mod tests { } /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister - /// and identity, no proxy, an empty canister ID table, the default compute + /// and identity, no proxy, no declared dependencies, the default compute /// limit, and the current directory as the base. Individual tests override /// the few fields they care about. fn invocation(wasm_path: &str, environment: &str) -> PluginInvocation { @@ -752,17 +825,80 @@ mod tests { base_dir: ".".into(), dirs: vec![], files: vec![], - target_canister_id: anon(), + host_canister_id: anon(), agent: dummy_agent(), proxy: None, identity_principal: anon(), environment: environment.to_string(), compute_limit_secs: DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, canister_ids: BTreeMap::new(), + callable: CallableCanisters::default(), stdio: None, } } + // ------------------------------------------------------------------------- + // Call-target resolution (enforcement) — pure logic, no fixture WASM needed + // ------------------------------------------------------------------------- + + #[test] + fn resolve_target_host_is_always_allowed() { + let host = Principal::from_slice(&[1; 4]); + let callable = CallableCanisters::default(); + assert_eq!( + resolve_call_target(&CallTarget::Host, host, &callable).unwrap(), + host + ); + } + + #[test] + fn resolve_target_name_requires_declaration() { + let host = Principal::from_slice(&[1; 4]); + let dep = Principal::from_slice(&[2; 4]); + let callable = CallableCanisters { + by_name: BTreeMap::from([("backend".to_string(), dep)]), + by_id: BTreeSet::from([dep]), + }; + assert_eq!( + resolve_call_target(&CallTarget::Name("backend".into()), host, &callable).unwrap(), + dep + ); + let err = resolve_call_target(&CallTarget::Name("frontend".into()), host, &callable) + .expect_err("undeclared name must be rejected"); + assert!( + err.contains("not permitted") && err.contains("frontend"), + "got: {err}" + ); + } + + #[test] + fn resolve_target_id_allows_host_and_declared_only() { + let host = Principal::from_slice(&[1; 4]); + let dep = Principal::from_slice(&[2; 4]); + let other = Principal::from_slice(&[3; 4]); + let callable = CallableCanisters { + by_name: BTreeMap::new(), + by_id: BTreeSet::from([dep]), + }; + // A declared principal is allowed; so is the host, implicitly. + assert_eq!( + resolve_call_target(&CallTarget::Id(dep.to_text()), host, &callable).unwrap(), + dep + ); + assert_eq!( + resolve_call_target(&CallTarget::Id(host.to_text()), host, &callable).unwrap(), + host + ); + // An undeclared principal is rejected. + let err = resolve_call_target(&CallTarget::Id(other.to_text()), host, &callable) + .expect_err("undeclared principal must be rejected"); + assert!(err.contains("not permitted"), "got: {err}"); + // Garbage text is a distinct, clearer error. + let err = resolve_call_target(&CallTarget::Id("not a principal".into()), host, &callable) + .expect_err("invalid principal text must be rejected"); + assert!(err.contains("invalid target principal"), "got: {err}"); + } + // ------------------------------------------------------------------------- // Error-path tests — no fixture WASM needed // ------------------------------------------------------------------------- diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index e201568b5..bd7b13d67 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -33,6 +33,25 @@ interface types { id: string, } + /// Which canister a `canister-call-request` targets. + /// + /// A plugin may target the canister being synced, or any canister it + /// declared as a dependency in the sync step's `canisters` list — by that + /// canister's name or by its textual principal. Targeting a canister that + /// was not declared as a dependency is rejected by the host. + variant call-target { + /// The canister being synced (`sync-exec-input.canister-id`). Always + /// permitted, whether or not it also appears in `canisters`. + host, + /// A declared-dependency canister identified by name, spelled exactly as + /// it appears in `sync-exec-input.canister-ids` — a bare local name for a + /// canister in the same subproject, or a `subproject:local` key + /// otherwise. The host resolves it against that mapping table. + name(string), + /// A declared-dependency canister identified by its textual principal. + id(string), + } + /// Input passed by the runtime to the plugin's exec() export. record sync-exec-input { /// Textual principal of the canister being synced. @@ -59,8 +78,12 @@ interface types { canister-ids: list, } - /// A request to call a method on the target canister. + /// A request to call a method on a canister. record canister-call-request { + /// Which canister to call. `host` targets the canister being synced; + /// `name`/`id` target a canister declared as a dependency in the sync + /// step's `canisters` list. + target: call-target, /// The canister method to call. method: string, /// Candid-encoded argument bytes. The plugin is responsible for @@ -84,15 +107,17 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, canister-id-entry, file-input}; + use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin // ------------------------------------------------------------------------- - /// Make an update or query call to the canister being synced. - /// The host always calls the canister from sync-exec-input.canister-id; - /// the plugin does not choose the target. + /// Make an update or query call to a canister. + /// The `req.target` selects the canister: the one being synced (`host`), or + /// a canister declared as a dependency in the sync step's `canisters` list, + /// by name or principal. A target that was not declared as a dependency is + /// rejected without making a call. /// Returns the raw Candid-encoded response bytes on success or an error /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 22f749724..ad50d1d0e 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,16 +1,20 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use camino::Utf8PathBuf; use candid::Principal; use ic_agent::Agent; use icp_sync_plugin::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, - run_plugin, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + PluginInvocation, RunPluginError, run_plugin, }; use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{canister::wasm, manifest::adapter::plugin::Adapter, package::PackageCache}; +use crate::{ + canister::wasm, + manifest::adapter::plugin::{Adapter, CanisterRef}, + package::PackageCache, +}; use super::Params; @@ -29,6 +33,12 @@ pub enum PluginError { #[snafu(display("failed to run plugin"))] Run { source: RunPluginError }, + + #[snafu(display( + "sync plugin declares a dependency on canister '{name}', but no canister by that name \ + is known in environment '{environment}'" + ))] + UnknownDependency { name: String, environment: String }, } /// Resolve the plugin compute-time limit, honoring the @@ -94,8 +104,10 @@ pub(super) async fn sync( let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); - // 3. Build the canister ID table exposed to the plugin. + // 3. Build the canister ID table exposed to the plugin, then resolve the + // plugin's declared callable canisters against it. let canister_ids = exposed_canister_ids(params); + let callable = resolve_callable(adapter, &canister_ids, environment)?; // 4. Run the plugin (blocking call — signal Tokio that this thread will block). let identity_principal = agent @@ -112,13 +124,14 @@ pub(super) async fn sync( base_dir, dirs, files, - target_canister_id: params.cid, + host_canister_id: params.cid, agent: agent_clone, proxy, identity_principal, environment: environment_owned, compute_limit_secs, canister_ids, + callable, stdio: stdio_clone, }) }) @@ -150,6 +163,38 @@ fn exposed_canister_ids(params: &Params) -> BTreeMap { table } +/// Resolve the canisters a plugin declared it may call into a [`CallableCanisters`] +/// enforcement set. Named dependencies are looked up in `canister_ids`; a name +/// that does not resolve is a manifest error. +fn resolve_callable( + adapter: &Adapter, + canister_ids: &BTreeMap, + environment: &str, +) -> Result { + let mut by_name = BTreeMap::new(); + let mut by_id = BTreeSet::new(); + for canister in adapter.canisters.iter().flatten() { + match canister { + CanisterRef::Principal(principal) => { + by_id.insert(*principal); + } + CanisterRef::Name(name) => { + let principal = + canister_ids + .get(name) + .copied() + .context(UnknownDependencySnafu { + name: name.clone(), + environment: environment.to_owned(), + })?; + by_name.insert(name.clone(), principal); + by_id.insert(principal); + } + } + } + Ok(CallableCanisters { by_name, by_id }) +} + #[cfg(test)] mod tests { use super::*; @@ -173,6 +218,8 @@ mod tests { } } + use crate::manifest::adapter::prebuilt::{LocalSource, SourceField}; + fn principal(byte: u8) -> Principal { Principal::from_slice(&[byte; 4]) } @@ -189,6 +236,18 @@ mod tests { } } + fn adapter_with(canisters: Option>) -> Adapter { + Adapter { + source: SourceField::Local(LocalSource { + path: "plugin.wasm".into(), + }), + sha256: None, + dirs: None, + files: None, + canisters, + } + } + /// Canisters sharing the syncing canister's subproject are additionally /// exposed under their bare local name; canisters in other subprojects are /// not. @@ -276,4 +335,29 @@ mod tests { assert_eq!(table.len(), 1); assert_eq!(table.get("backend"), Some(&backend)); } + + #[test] + fn resolve_callable_resolves_names_and_principals() { + let dep = principal(1); + let raw = principal(2); + let table = BTreeMap::from([("backend".to_owned(), dep)]); + let adapter = adapter_with(Some(vec![ + CanisterRef::Name("backend".to_owned()), + CanisterRef::Principal(raw), + ])); + + let callable = resolve_callable(&adapter, &table, "demo").unwrap(); + + assert_eq!(callable.by_name.get("backend"), Some(&dep)); + assert!(callable.by_id.contains(&dep)); + assert!(callable.by_id.contains(&raw)); + } + + #[test] + fn resolve_callable_rejects_unknown_name() { + let adapter = adapter_with(Some(vec![CanisterRef::Name("nope".to_owned())])); + let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") + .expect_err("an undeclared name must fail"); + assert!(matches!(err, PluginError::UnknownDependency { .. })); + } } diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 915b03c1d..5aef5ccfa 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,8 +1,24 @@ +use candid::Principal; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; use super::prebuilt::SourceField; +/// A canister a sync plugin is permitted to call, beyond the canister being +/// synced. Written in the manifest either as a textual principal (e.g. +/// `aaaaa-aa`) or as a canister name resolved against the project's canister ID +/// table for the environment being synced (e.g. `backend`, or a namespaced +/// dependency canister such as `services/open-crm:backend`). Anything that +/// parses as a principal is taken as one; everything else is a name. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum CanisterRef { + /// An explicit principal (e.g. `aaaaa-aa`). + Principal(Principal), + /// A canister name from this project's ID table (e.g. `backend`). + Name(String), +} + /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -45,6 +61,14 @@ pub struct Adapter { /// Files (relative to canister directory) the host reads and passes to /// the plugin as part of `sync-exec-input.files`. pub files: Option>, + + /// Canisters this plugin may call in addition to the canister being synced. + /// Each entry is a canister name (resolved against the project's canister ID + /// table) or a textual principal. The plugin picks a target per call via the + /// `call-target` in its `canister-call` request; a target not listed here is + /// rejected by the host. + #[schemars(with = "Option>")] + pub canisters: Option>, } impl<'de> Deserialize<'de> for Adapter { @@ -56,6 +80,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, + canisters: Option>, } let h = AdapterHelper::deserialize(d)?; @@ -69,6 +94,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: h.sha256, dirs: h.dirs, files: h.files, + canisters: h.canisters, }) } } @@ -94,6 +120,7 @@ mod tests { sha256: None, dirs: None, files: None, + canisters: None, }, ); } @@ -120,6 +147,7 @@ mod tests { sha256: Some("abc123".to_string()), dirs: Some(vec!["assets/seed-data".to_string(), "config".to_string()]), files: Some(vec!["config.txt".to_string()]), + canisters: None, }, ); } @@ -139,6 +167,28 @@ mod tests { ); } + #[test] + fn canisters_parse_as_names_and_principals() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + canisters: + - backend + - services/open-crm:backend + - aaaaa-aa + "#, + ) + .expect("failed to deserialize Adapter with canisters"); + assert_eq!( + adapter.canisters, + Some(vec![ + CanisterRef::Name("backend".to_string()), + CanisterRef::Name("services/open-crm:backend".to_string()), + CanisterRef::Principal(Principal::from_text("aaaaa-aa").unwrap()), + ]), + ); + } + #[test] fn remote_url_with_sha256() { assert_eq!( @@ -156,6 +206,7 @@ mod tests { sha256: Some("a665a45920422f9d417e".to_string()), dirs: None, files: None, + canisters: None, }, ); } diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index b032be7e5..8d2838a5b 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -793,6 +793,7 @@ mod tests { sha256: None, dirs: Some(vec!["assets/seed-data/".to_string()]), files: None, + canisters: None, } )] }), @@ -837,6 +838,7 @@ mod tests { ), dirs: None, files: None, + canisters: None, })] }), }, diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index a7a6eb3f9..aca070959 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work against a single canister. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it declares as dependencies. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -15,7 +15,7 @@ Sync plugins fill that gap. A plugin is: - **Portable** — written in any language that compiles to `wasm32-wasip2`, distributed as one `.wasm` file (local path or remote URL + `sha256`). - **Sandboxed** — it cannot open network sockets, spawn subprocesses, or touch the filesystem outside the directories you explicitly grant it. -- **Scoped to one canister** — it can call update and query methods, but only on the canister being synced. The target is fixed by the host; the plugin cannot choose a different one. +- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister it declares as a dependency in the manifest's `canisters:` list. A call to a canister that was not declared is rejected by the host. The most common way to get a sync plugin is through a [recipe](recipes.md). For example, the `@dfinity/asset-canister` recipe emits a `plugin` sync step (starting with `v2.2.1`) that uploads your built static files to the asset canister — so for everyday frontend deployment you never write a plugin yourself. @@ -38,7 +38,9 @@ icp sync │ canister-ids = │ dirs / files = what you declared in the manifest │ - └─ plugin makes canister-call(...) to the target canister (× N) + └─ plugin makes canister-call({ target, ... }) (× N) + target = host (the canister being synced), or a + declared-dependency canister by name or principal ``` ## The Plugin Interface @@ -47,7 +49,7 @@ The interface is defined as a [WIT](https://component-model.bytecodealliance.org ```wit world sync-plugin { - // Host import: call the canister being synced. + // Host import: call the canister being synced or a declared dependency. import canister-call: func(req: canister-call-request) -> result, string>; // Plugin export: run the sync step. @@ -73,19 +75,20 @@ The authoritative interface, including all record fields, lives in [`sync-plugin Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister that came from a dependency. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. -### Calling the canister — `canister-call` +### Calling a canister — `canister-call` -The plugin calls methods on the target canister through the `canister-call` import. It supplies the method name, **Candid-encoded argument bytes** (the host forwards them unchanged), and a few routing options: +The plugin calls methods through the `canister-call` import. It picks a `target`, supplies the method name, **Candid-encoded argument bytes** (the host forwards them unchanged), and a few routing options: | Request field | Meaning | |---------------|---------| +| `target` | Which canister to call: `host` (the canister being synced), or a canister declared in `canisters:` addressed by `name` or by `id` (principal) | | `method` | The canister method to call | | `arg` | Candid-encoded argument bytes (the plugin encodes; the host forwards as-is) | | `call-type` | `update` or `query` | | `direct` | When `false` (default), update calls are routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, the call always goes directly to the target. Query calls always go directly regardless. | | `cycles` | Cycles to attach to a proxied update call; only meaningful when `direct` is `false`, a proxy is configured, and `call-type` is `update` | -The host always calls the canister named in `sync-exec-input.canister-id`. There is no field for a different canister ID — the single-canister restriction is structural, not a policy the plugin can opt out of. +The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name`/`id` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. ### Logging — stdout and stderr @@ -114,7 +117,7 @@ The plugin runs with a deliberately narrow capability surface. | Read declared `dirs:` | yes | read-only preopens | | Clocks, RNG, `wasi:io` | yes | Rust's `HashMap`, `chrono`, etc. work normally | | `process::exit` / panics | yes | abort the guest cleanly; the host surfaces the error | -| Canister calls | yes | only to the canister being synced | +| Canister calls | yes | to the canister being synced, and to canisters declared in `canisters:` | | Environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | | Network sockets / DNS | blocked | treat the network as unavailable | | Filesystem writes | blocked | no writable preopens | diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 979a43081..0c3004752 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -65,6 +65,7 @@ impl Guest for Plugin { // Call a method on the canister being synced. canister_call(&CanisterCallRequest { + target: CallTarget::Host, // the canister being synced method: "set_uploader".to_string(), arg, call_type: icp::sync_plugin::types::CallType::Update, @@ -84,7 +85,7 @@ export!(Plugin); A few things to note: - **You encode the arguments.** `arg` is raw Candid bytes. Encode with `candid::Encode!`; decode any response (`Vec`) with `candid::Decode!`. -- **The target is fixed.** `canister_call` always reaches the canister in `input.canister_id` — there is no field to target another canister. +- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` or `CallTarget::Id(principal_text)` — a name matches the entries in `input.canister_ids`. The host rejects a target you did not declare. - **`direct` and `cycles` control proxy routing.** With `direct: false`, update calls go through the [proxy canister](proxy-canister.md) when one is configured, and `cycles` can fund the forwarded call. With `direct: true`, the call always goes straight to the target. See [The Plugin Interface](../concepts/sync-plugins.md#the-plugin-interface) for the full semantics. ## Read Declared Files and Directories diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 219eff450..7e594facd 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -151,6 +151,9 @@ sync: - config files: # files read by the host and passed inline - config.txt + canisters: # extra canisters the plugin may call + - ledger # by name (resolved for the environment) + - ryjl3-tyaaa-aaaaa-aaaba-cai # or by principal # Remote plugin (downloaded and verified before execution) - type: plugin @@ -165,10 +168,13 @@ sync: | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | | `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | | `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | +| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name (resolved against the project's canister IDs for the environment) or a textual principal | Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. -The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. +A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a dependency canister. A name that does not resolve to a known canister for the environment fails the sync step. + +The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. ## Recipes diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 7559c5cb2..4dcd4d0b8 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -69,6 +69,16 @@ ], "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { + "canisters": { + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "dirs": { "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", "items": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index b83a4e7a9..1c4758c8e 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -69,6 +69,16 @@ ], "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { + "canisters": { + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "dirs": { "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", "items": { diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index 6f8d25508..f6d7534bd 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -24,6 +24,7 @@ impl Guest for Plugin { .map_err(|e| format!("invalid identity principal: {e}"))?; let arg = Encode!(&uploader).map_err(|e| format!("encode set_uploader arg: {e}"))?; canister_call(&CanisterCallRequest { + target: CallTarget::Host, method: "set_uploader".to_string(), arg, call_type: icp::sync_plugin::types::CallType::Update, @@ -68,6 +69,7 @@ fn register_dir(dir: &Path) -> Result { let arg = Encode!(&path_str, &content_trimmed) .map_err(|e| format!("encode register arg: {e}"))?; canister_call(&CanisterCallRequest { + target: CallTarget::Host, method: "register".to_string(), arg, call_type: icp::sync_plugin::types::CallType::Update, From 2d531968ddfc7457dea430d3d6c265c68b32ed31 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 10:14:38 -0700 Subject: [PATCH 10/51] Document canister-ids call-target permission on the targeting interface With cross-canister targeting present, the `canister-ids` table's field doc and the DESIGN rationale should describe the real permission model: the table is informational, and calling a listed canister requires declaring it as a dependency (`call-target`). The mappings-branch wording ("canister-call always targets the canister being synced") was correct only before this interface added targeting. --- crates/icp-sync-plugin/DESIGN.md | 5 ++--- crates/icp-sync-plugin/sync-plugin.wit | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 40646741f..fcc8a82fa 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -36,9 +36,8 @@ docs; the *reasons* behind those choices are recorded here. versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve - canister names it knows about. It is informational only: `canister-call` - still targets the canister being synced, so the table grants no ability to - call other canisters. + canister names it knows about. It is informational only; calling still + requires a declaration. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index bd7b13d67..751e263ad 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -73,8 +73,8 @@ interface types { /// Name→principal mapping for every named canister in the project for /// the environment being synced, sorted by name. Informational: the /// plugin may use it to resolve canister names it knows about. Being - /// listed here does not let the plugin call a canister — the - /// `canister-call` import always targets the canister being synced. + /// listed here does not grant permission to call a canister — that + /// still requires declaring it as a dependency (see `call-target`). canister-ids: list, } From ea5c6663ee6bc666554bbb2a05b7814ce20438bb Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 10:34:51 -0700 Subject: [PATCH 11/51] copilot --- crates/icp-cli/src/operations/bundle.rs | 29 +++++- crates/icp-cli/tests/bundle_tests.rs | 130 ++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 4b07c5f5d..af861f99b 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -17,7 +17,9 @@ use icp::{ ArgsFormat, BuildStep, BuildSteps, CanisterManifest, DependencyManifest, EnvironmentManifest, Instructions, Item, LoadManifestFromPathError, ManagedMode, ManifestInitArgs, Mode, NetworkManifest, PROJECT_MANIFEST, ProjectManifest, SyncStep, - SyncSteps, load_manifest_from_path, plugin, prebuilt, + SyncSteps, load_manifest_from_path, plugin, + plugin::CanisterRef, + prebuilt, prebuilt::{LocalSource, SourceField}, }, package::PackageCache, @@ -650,6 +652,7 @@ async fn prepare_canister( canister_path, &path_name, idx, + local_names, pkg_cache, out, ) @@ -710,6 +713,27 @@ fn localize_controllers( settings } +/// Rewrite a plugin's declared call targets from workspace store keys back to the +/// local names of the instance being written, on the same grounds as +/// [`localize_controllers`]. Principals are already absolute and pass through. +fn localize_call_targets( + canisters: Option<&[CanisterRef]>, + local_names: &HashMap<&str, &str>, +) -> Option> { + canisters.map(|canisters| { + canisters + .iter() + .map(|target| match target { + CanisterRef::Name(name) => match local_names.get(name.as_str()) { + Some(local) => CanisterRef::Name((*local).to_owned()), + None => target.clone(), + }, + CanisterRef::Principal(_) => target.clone(), + }) + .collect() + }) +} + #[allow(clippy::too_many_arguments)] async fn prepare_plugin_step( adapter: &plugin::Adapter, @@ -718,6 +742,7 @@ async fn prepare_plugin_step( canister_path: &Path, path_name: &str, idx: usize, + local_names: &HashMap<&str, &str>, pkg_cache: &PackageCache, out: &mut BundleArtifacts, ) -> Result { @@ -788,7 +813,7 @@ async fn prepare_plugin_step( sha256: Some(plugin_sha256), dirs: bundle_dirs, files: bundle_files, - canisters: None, + canisters: localize_call_targets(adapter.canisters.as_deref(), local_names), })) } diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 74748e4e4..9619fb10a 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1088,6 +1088,136 @@ fn bundle_packages_plugin_sync_steps() { ); } +/// A plugin's declared call targets must survive bundling — dropping them would turn a +/// working project into a bundle whose cross-canister calls are all rejected. Names of +/// the writing instance's own canisters come back out as local names; principals and +/// names that already resolved against the workspace are left alone. +#[test] +fn bundle_preserves_plugin_call_targets() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let build_step = formatdoc! {r#" + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + "#}; + + // Bundling only repackages the plugin wasm bytes, so any non-empty content works. + write(&project_dir.join("plugin.wasm"), b"\x00asm\x01\x00\x00\x00") + .expect("failed to write plugin wasm"); + + let dep_dir = project_dir.join("vendor/openemail"); + create_dir_all(&dep_dir).expect("failed to create dependency dir"); + write(&dep_dir.join("plugin.wasm"), b"\x00asm\x01\x00\x00\x00") + .expect("failed to write dependency plugin wasm"); + + // The dependency's plugin names its own sibling, both bare and by store key. + write_string( + &dep_dir.join("icp.yaml"), + &formatdoc! {r#" + canisters: + - name: backend + {build_step} + sync: + steps: + - type: plugin + path: plugin.wasm + canisters: + - helper + - vendor/openemail:helper + - name: helper + {build_step} + "#}, + ) + .expect("failed to write dependency manifest"); + + // The root's plugin names a root sibling, a dependency canister by store key, and a + // literal principal. + write_string( + &project_dir.join("icp.yaml"), + &formatdoc! {r#" + canisters: + - name: frontend + {build_step} + sync: + steps: + - type: plugin + path: plugin.wasm + canisters: + - api + - vendor/openemail:backend + - aaaaa-aa + - name: api + {build_step} + + dependencies: + - name: openemail + path: ./vendor/openemail + canisters: [backend] + "#}, + ) + .expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut manifests: std::collections::HashMap = std::collections::HashMap::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path.ends_with("icp.yaml") { + let mut yaml = String::new(); + entry + .read_to_string(&mut yaml) + .expect("failed to read manifest"); + manifests.insert(path, yaml); + } + } + + let plugin_targets = |yaml: &str, canister: &str| -> Vec { + let parsed: serde_yaml::Value = + serde_yaml::from_str(yaml).expect("manifest yaml is invalid"); + let canisters = parsed["canisters"] + .as_sequence() + .expect("manifest has no canisters"); + let entry = canisters + .iter() + .find(|c| c["name"].as_str() == Some(canister)) + .unwrap_or_else(|| panic!("{canister} not found in bundled manifest: {yaml}")); + entry["sync"]["steps"][0]["canisters"] + .as_sequence() + .unwrap_or_else(|| panic!("{canister} plugin step lost its canisters: {yaml}")) + .iter() + .map(|t| t.as_str().expect("call target is not a string").to_owned()) + .collect() + }; + + assert_eq!( + plugin_targets(&manifests["icp.yaml"], "frontend"), + ["api", "vendor/openemail:backend", "aaaaa-aa"], + ); + // Both spellings of the dependency's own sibling come out as its local name. + assert_eq!( + plugin_targets(&manifests["vendor/openemail/icp.yaml"], "backend"), + ["helper", "helper"], + ); +} + /// An `icp_appmanifest.yaml` next to the project manifest must be included in the bundle, with its /// top-level `images` paths relocated under a top-level `images/` folder and the /// referenced image files copied alongside. Unrelated metadata is preserved. From 2b487753a7fa84b3894ac60914ae214c453d99c1 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 10:05:59 -0700 Subject: [PATCH 12/51] Remove id target --- crates/icp-cli/src/operations/bundle.rs | 19 +++---- crates/icp-cli/tests/bundle_tests.rs | 6 +-- crates/icp-sync-plugin/DESIGN.md | 23 ++++---- crates/icp-sync-plugin/src/runtime.rs | 47 +--------------- crates/icp-sync-plugin/sync-plugin.wit | 14 +++-- crates/icp/src/canister/sync/plugin.rs | 65 ++++++++++------------- crates/icp/src/manifest/adapter/plugin.rs | 38 ++++--------- docs/concepts/sync-plugins.md | 6 +-- docs/guides/writing-sync-plugins.md | 2 +- docs/reference/configuration.md | 6 +-- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- 12 files changed, 75 insertions(+), 155 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index af861f99b..bd460d576 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -17,9 +17,7 @@ use icp::{ ArgsFormat, BuildStep, BuildSteps, CanisterManifest, DependencyManifest, EnvironmentManifest, Instructions, Item, LoadManifestFromPathError, ManagedMode, ManifestInitArgs, Mode, NetworkManifest, PROJECT_MANIFEST, ProjectManifest, SyncStep, - SyncSteps, load_manifest_from_path, plugin, - plugin::CanisterRef, - prebuilt, + SyncSteps, load_manifest_from_path, plugin, prebuilt, prebuilt::{LocalSource, SourceField}, }, package::PackageCache, @@ -715,20 +713,17 @@ fn localize_controllers( /// Rewrite a plugin's declared call targets from workspace store keys back to the /// local names of the instance being written, on the same grounds as -/// [`localize_controllers`]. Principals are already absolute and pass through. +/// [`localize_controllers`]. fn localize_call_targets( - canisters: Option<&[CanisterRef]>, + canisters: Option<&[String]>, local_names: &HashMap<&str, &str>, -) -> Option> { +) -> Option> { canisters.map(|canisters| { canisters .iter() - .map(|target| match target { - CanisterRef::Name(name) => match local_names.get(name.as_str()) { - Some(local) => CanisterRef::Name((*local).to_owned()), - None => target.clone(), - }, - CanisterRef::Principal(_) => target.clone(), + .map(|target| match local_names.get(target.as_str()) { + Some(local) => (*local).to_owned(), + None => target.clone(), }) .collect() }) diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 9619fb10a..67b8a014f 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1134,8 +1134,7 @@ fn bundle_preserves_plugin_call_targets() { ) .expect("failed to write dependency manifest"); - // The root's plugin names a root sibling, a dependency canister by store key, and a - // literal principal. + // The root's plugin names a root sibling and a dependency canister by store key. write_string( &project_dir.join("icp.yaml"), &formatdoc! {r#" @@ -1149,7 +1148,6 @@ fn bundle_preserves_plugin_call_targets() { canisters: - api - vendor/openemail:backend - - aaaaa-aa - name: api {build_step} @@ -1209,7 +1207,7 @@ fn bundle_preserves_plugin_call_targets() { assert_eq!( plugin_targets(&manifests["icp.yaml"], "frontend"), - ["api", "vendor/openemail:backend", "aaaaa-aa"], + ["api", "vendor/openemail:backend"], ); // Both spellings of the dependency's own sibling come out as its local name. assert_eq!( diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index fcc8a82fa..e9270dc79 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -29,11 +29,13 @@ docs; the *reasons* behind those choices are recorded here. unchanged. This keeps the host free of any per-canister type knowledge. - **`canister-call` takes an explicit `target`** — the plugin selects the canister being synced (`host`) or a canister it declared as a dependency, by - name or principal. The host resolves the target and *enforces* the - declaration: a target absent from the step's `canisters:` list is rejected - without a call. (In the earlier `@0.1.0` interface `canister-call` had no - target and always reached the canister being synced; see *Interface - versioning* below.) + name. The host resolves the target and *enforces* the declaration: a target + absent from the step's `canisters:` list is rejected without a call. Names are + the only way to address a dependency: the name→principal mapping is the host's + to make, since it varies per environment, and a plugin that hardcodes a + principal is pinned to one deployment. (In the earlier `@0.1.0` interface + `canister-call` had no target and always reached the canister being synced; see + *Interface versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still @@ -118,7 +120,7 @@ mod v1 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugi struct HostState { host_canister_id: Principal, - callable: CallableCanisters, // by_name + by_id, from the manifest + callable: CallableCanisters, // name → principal, from the manifest agent: Arc, proxy: Option, wasi_ctx: wasmtime_wasi::WasiCtx, @@ -186,14 +188,13 @@ pub struct Adapter { pub sha256: Option, pub dirs: Option>, pub files: Option>, - pub canisters: Option>, // extra callable canisters + pub canisters: Option>, // extra callable canisters, by name } ``` -`CanisterRef` is an untagged `Principal | Name` (anything that parses as a -principal is one; everything else is a name), written in the manifest as a plain -string. `Deserialize` is hand-written to reject a `url` source without a -`sha256`. +Each `canisters:` entry is a canister name resolved against the project's ID +table for the environment being synced. `Deserialize` is hand-written to reject a +`url` source without a `sha256`. ### `crates/icp/src/canister/sync/plugin.rs` diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index f45ece92c..6c75caae2 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1,5 +1,5 @@ // Host-side Component Model runtime for sync plugins. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex as StdMutex; @@ -67,10 +67,6 @@ pub struct CallableCanisters { /// Dependencies callable by name ([`CallTarget::Name`]). Maps the name — as /// it appears in the canister ID table — to the principal it resolves to. pub by_name: BTreeMap, - /// Every principal callable by [`CallTarget::Id`]. Includes the principals - /// of the `by_name` entries, so an author may target the same canister - /// either way. - pub by_id: BTreeSet, } /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing @@ -89,18 +85,6 @@ fn resolve_call_target( `canisters` list to allow it" ) }), - CallTarget::Id(text) => { - let principal = Principal::from_text(text) - .map_err(|e| format!("invalid target principal '{text}': {e}"))?; - if principal == host_canister_id || callable.by_id.contains(&principal) { - Ok(principal) - } else { - Err(format!( - "plugin is not permitted to call canister '{principal}': declare it in the \ - sync step's `canisters` list to allow it" - )) - } - } } } @@ -857,7 +841,6 @@ mod tests { let dep = Principal::from_slice(&[2; 4]); let callable = CallableCanisters { by_name: BTreeMap::from([("backend".to_string(), dep)]), - by_id: BTreeSet::from([dep]), }; assert_eq!( resolve_call_target(&CallTarget::Name("backend".into()), host, &callable).unwrap(), @@ -871,34 +854,6 @@ mod tests { ); } - #[test] - fn resolve_target_id_allows_host_and_declared_only() { - let host = Principal::from_slice(&[1; 4]); - let dep = Principal::from_slice(&[2; 4]); - let other = Principal::from_slice(&[3; 4]); - let callable = CallableCanisters { - by_name: BTreeMap::new(), - by_id: BTreeSet::from([dep]), - }; - // A declared principal is allowed; so is the host, implicitly. - assert_eq!( - resolve_call_target(&CallTarget::Id(dep.to_text()), host, &callable).unwrap(), - dep - ); - assert_eq!( - resolve_call_target(&CallTarget::Id(host.to_text()), host, &callable).unwrap(), - host - ); - // An undeclared principal is rejected. - let err = resolve_call_target(&CallTarget::Id(other.to_text()), host, &callable) - .expect_err("undeclared principal must be rejected"); - assert!(err.contains("not permitted"), "got: {err}"); - // Garbage text is a distinct, clearer error. - let err = resolve_call_target(&CallTarget::Id("not a principal".into()), host, &callable) - .expect_err("invalid principal text must be rejected"); - assert!(err.contains("invalid target principal"), "got: {err}"); - } - // ------------------------------------------------------------------------- // Error-path tests — no fixture WASM needed // ------------------------------------------------------------------------- diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 751e263ad..cace1e32c 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -36,9 +36,9 @@ interface types { /// Which canister a `canister-call-request` targets. /// /// A plugin may target the canister being synced, or any canister it - /// declared as a dependency in the sync step's `canisters` list — by that - /// canister's name or by its textual principal. Targeting a canister that - /// was not declared as a dependency is rejected by the host. + /// declared as a dependency in the sync step's `canisters` list — always by + /// that canister's name. Targeting a canister that was not declared as a + /// dependency is rejected by the host. variant call-target { /// The canister being synced (`sync-exec-input.canister-id`). Always /// permitted, whether or not it also appears in `canisters`. @@ -48,8 +48,6 @@ interface types { /// canister in the same subproject, or a `subproject:local` key /// otherwise. The host resolves it against that mapping table. name(string), - /// A declared-dependency canister identified by its textual principal. - id(string), } /// Input passed by the runtime to the plugin's exec() export. @@ -81,7 +79,7 @@ interface types { /// A request to call a method on a canister. record canister-call-request { /// Which canister to call. `host` targets the canister being synced; - /// `name`/`id` target a canister declared as a dependency in the sync + /// `name` targets a canister declared as a dependency in the sync /// step's `canisters` list. target: call-target, /// The canister method to call. @@ -116,8 +114,8 @@ world sync-plugin { /// Make an update or query call to a canister. /// The `req.target` selects the canister: the one being synced (`host`), or /// a canister declared as a dependency in the sync step's `canisters` list, - /// by name or principal. A target that was not declared as a dependency is - /// rejected without making a call. + /// by name. A target that was not declared as a dependency is rejected + /// without making a call. /// Returns the raw Candid-encoded response bytes on success or an error /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index ad50d1d0e..1e525dc22 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use camino::Utf8PathBuf; use candid::Principal; @@ -10,11 +10,7 @@ use icp_sync_plugin::{ use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{ - canister::wasm, - manifest::adapter::plugin::{Adapter, CanisterRef}, - package::PackageCache, -}; +use crate::{canister::wasm, manifest::adapter::plugin::Adapter, package::PackageCache}; use super::Params; @@ -164,7 +160,7 @@ fn exposed_canister_ids(params: &Params) -> BTreeMap { } /// Resolve the canisters a plugin declared it may call into a [`CallableCanisters`] -/// enforcement set. Named dependencies are looked up in `canister_ids`; a name +/// enforcement set. Each declared name is looked up in `canister_ids`; a name /// that does not resolve is a manifest error. fn resolve_callable( adapter: &Adapter, @@ -172,27 +168,17 @@ fn resolve_callable( environment: &str, ) -> Result { let mut by_name = BTreeMap::new(); - let mut by_id = BTreeSet::new(); - for canister in adapter.canisters.iter().flatten() { - match canister { - CanisterRef::Principal(principal) => { - by_id.insert(*principal); - } - CanisterRef::Name(name) => { - let principal = - canister_ids - .get(name) - .copied() - .context(UnknownDependencySnafu { - name: name.clone(), - environment: environment.to_owned(), - })?; - by_name.insert(name.clone(), principal); - by_id.insert(principal); - } - } + for name in adapter.canisters.iter().flatten() { + let principal = canister_ids + .get(name) + .copied() + .context(UnknownDependencySnafu { + name: name.clone(), + environment: environment.to_owned(), + })?; + by_name.insert(name.clone(), principal); } - Ok(CallableCanisters { by_name, by_id }) + Ok(CallableCanisters { by_name }) } #[cfg(test)] @@ -236,7 +222,7 @@ mod tests { } } - fn adapter_with(canisters: Option>) -> Adapter { + fn adapter_with(canisters: Option>) -> Adapter { Adapter { source: SourceField::Local(LocalSource { path: "plugin.wasm".into(), @@ -337,25 +323,30 @@ mod tests { } #[test] - fn resolve_callable_resolves_names_and_principals() { + fn resolve_callable_resolves_names() { let dep = principal(1); - let raw = principal(2); - let table = BTreeMap::from([("backend".to_owned(), dep)]); + let sibling = principal(2); + let table = BTreeMap::from([ + ("backend".to_owned(), sibling), + ("services/open-crm:backend".to_owned(), dep), + ]); let adapter = adapter_with(Some(vec![ - CanisterRef::Name("backend".to_owned()), - CanisterRef::Principal(raw), + "backend".to_owned(), + "services/open-crm:backend".to_owned(), ])); let callable = resolve_callable(&adapter, &table, "demo").unwrap(); - assert_eq!(callable.by_name.get("backend"), Some(&dep)); - assert!(callable.by_id.contains(&dep)); - assert!(callable.by_id.contains(&raw)); + assert_eq!(callable.by_name.get("backend"), Some(&sibling)); + assert_eq!( + callable.by_name.get("services/open-crm:backend"), + Some(&dep) + ); } #[test] fn resolve_callable_rejects_unknown_name() { - let adapter = adapter_with(Some(vec![CanisterRef::Name("nope".to_owned())])); + let adapter = adapter_with(Some(vec!["nope".to_owned()])); let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") .expect_err("an undeclared name must fail"); assert!(matches!(err, PluginError::UnknownDependency { .. })); diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 5aef5ccfa..68a76c686 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,24 +1,8 @@ -use candid::Principal; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; use super::prebuilt::SourceField; -/// A canister a sync plugin is permitted to call, beyond the canister being -/// synced. Written in the manifest either as a textual principal (e.g. -/// `aaaaa-aa`) or as a canister name resolved against the project's canister ID -/// table for the environment being synced (e.g. `backend`, or a namespaced -/// dependency canister such as `services/open-crm:backend`). Anything that -/// parses as a principal is taken as one; everything else is a name. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CanisterRef { - /// An explicit principal (e.g. `aaaaa-aa`). - Principal(Principal), - /// A canister name from this project's ID table (e.g. `backend`). - Name(String), -} - /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -63,12 +47,12 @@ pub struct Adapter { pub files: Option>, /// Canisters this plugin may call in addition to the canister being synced. - /// Each entry is a canister name (resolved against the project's canister ID - /// table) or a textual principal. The plugin picks a target per call via the - /// `call-target` in its `canister-call` request; a target not listed here is - /// rejected by the host. - #[schemars(with = "Option>")] - pub canisters: Option>, + /// Each entry is a canister name resolved against the project's canister ID + /// table for the environment being synced (e.g. `backend`, or a namespaced + /// dependency canister such as `services/open-crm:backend`). The plugin + /// picks a target per call via the `call-target` in its `canister-call` + /// request; a target not listed here is rejected by the host. + pub canisters: Option>, } impl<'de> Deserialize<'de> for Adapter { @@ -80,7 +64,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, - canisters: Option>, + canisters: Option>, } let h = AdapterHelper::deserialize(d)?; @@ -168,23 +152,21 @@ mod tests { } #[test] - fn canisters_parse_as_names_and_principals() { + fn canisters_parse_as_names() { let adapter = serde_yaml::from_str::( r#" path: plugins/my-sync.wasm canisters: - backend - services/open-crm:backend - - aaaaa-aa "#, ) .expect("failed to deserialize Adapter with canisters"); assert_eq!( adapter.canisters, Some(vec![ - CanisterRef::Name("backend".to_string()), - CanisterRef::Name("services/open-crm:backend".to_string()), - CanisterRef::Principal(Principal::from_text("aaaaa-aa").unwrap()), + "backend".to_string(), + "services/open-crm:backend".to_string(), ]), ); } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index aca070959..4dae4da32 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -40,7 +40,7 @@ icp sync │ └─ plugin makes canister-call({ target, ... }) (× N) target = host (the canister being synced), or a - declared-dependency canister by name or principal + declared-dependency canister by name ``` ## The Plugin Interface @@ -81,14 +81,14 @@ The plugin calls methods through the `canister-call` import. It picks a `target` | Request field | Meaning | |---------------|---------| -| `target` | Which canister to call: `host` (the canister being synced), or a canister declared in `canisters:` addressed by `name` or by `id` (principal) | +| `target` | Which canister to call: `host` (the canister being synced), or a canister declared in `canisters:` addressed by `name` | | `method` | The canister method to call | | `arg` | Candid-encoded argument bytes (the plugin encodes; the host forwards as-is) | | `call-type` | `update` or `query` | | `direct` | When `false` (default), update calls are routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, the call always goes directly to the target. Query calls always go directly regardless. | | `cycles` | Cycles to attach to a proxied update call; only meaningful when `direct` is `false`, a proxy is configured, and `call-type` is `update` | -The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name`/`id` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. +The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. A name is the only way to address another canister — the host owns the name→principal mapping, which differs per environment. ### Logging — stdout and stderr diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 0c3004752..80cf1f46e 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -85,7 +85,7 @@ export!(Plugin); A few things to note: - **You encode the arguments.** `arg` is raw Candid bytes. Encode with `candid::Encode!`; decode any response (`Vec`) with `candid::Decode!`. -- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` or `CallTarget::Id(principal_text)` — a name matches the entries in `input.canister_ids`. The host rejects a target you did not declare. +- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` — the name matches the entries in `input.canister_ids`. Names are the only way to reach another canister; the host resolves them per environment. The host rejects a target you did not declare. - **`direct` and `cycles` control proxy routing.** With `direct: false`, update calls go through the [proxy canister](proxy-canister.md) when one is configured, and `cycles` can fund the forwarded call. With `direct: true`, the call always goes straight to the target. See [The Plugin Interface](../concepts/sync-plugins.md#the-plugin-interface) for the full semantics. ## Read Declared Files and Directories diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 7e594facd..7d885a292 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -151,9 +151,9 @@ sync: - config files: # files read by the host and passed inline - config.txt - canisters: # extra canisters the plugin may call + canisters: # extra canisters the plugin may call, - ledger # by name (resolved for the environment) - - ryjl3-tyaaa-aaaaa-aaaba-cai # or by principal + - services/open-crm:backend # Remote plugin (downloaded and verified before execution) - type: plugin @@ -168,7 +168,7 @@ sync: | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | | `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | | `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | -| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name (resolved against the project's canister IDs for the environment) or a textual principal | +| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 4dcd4d0b8..c361b4562 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 1c4758c8e..5f27fa902 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, From e083734546c03a4d699961642723bdffb0cb0b35 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 10:13:56 -0700 Subject: [PATCH 13/51] Remove project dependencies from sandboxing logic --- crates/icp-sync-plugin/DESIGN.md | 18 +++++++-------- crates/icp-sync-plugin/src/runtime.rs | 22 +++++++++--------- crates/icp-sync-plugin/sync-plugin.wit | 27 +++++++++++------------ crates/icp/src/canister/sync/mod.rs | 2 +- crates/icp/src/canister/sync/plugin.rs | 23 ++++++++++--------- crates/icp/src/manifest/adapter/plugin.rs | 2 +- docs/concepts/sync-plugins.md | 10 ++++----- docs/reference/configuration.md | 2 +- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index e9270dc79..9d57338c9 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -28,18 +28,18 @@ docs; the *reasons* behind those choices are recorded here. `list`. The plugin owns Candid encoding/decoding; the host forwards bytes unchanged. This keeps the host free of any per-canister type knowledge. - **`canister-call` takes an explicit `target`** — the plugin selects the - canister being synced (`host`) or a canister it declared as a dependency, by - name. The host resolves the target and *enforces* the declaration: a target - absent from the step's `canisters:` list is rejected without a call. Names are - the only way to address a dependency: the name→principal mapping is the host's - to make, since it varies per environment, and a plugin that hardcodes a - principal is pinned to one deployment. (In the earlier `@0.1.0` interface + canister being synced (`host`) or a canister from the step's `canisters:` + list, by name. The host resolves the target and *enforces* the list: a target + absent from it is rejected without a call. Names are the only way to address + another canister: the name→principal mapping is the host's to make, since it + varies per environment, and a plugin that hardcodes a principal is pinned to + one deployment. (In the earlier `@0.1.0` interface `canister-call` had no target and always reached the canister being synced; see *Interface versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still - requires a declaration. + requires an entry in `canisters:`. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. @@ -205,5 +205,5 @@ enforcement set (resolving `canisters:` against the project's IDs), then calls the CLI — opens the declared paths and enforces the path-safety checks, so the CLI no longer touches the plugin's input files itself. `exposed_canister_ids` adds a bare-local-name duplicate for every canister in the same subproject as -the one being synced; `resolve_callable` fails the step if a declared dependency -name does not resolve. +the one being synced; `resolve_callable` fails the step if a name in +`canisters:` does not resolve. diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 6c75caae2..4632a1ab9 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -59,19 +59,19 @@ use v2::icp::sync_plugin::types::{CallTarget, CallType, CanisterIdEntry}; /// The canisters a sync plugin is permitted to call, beyond the canister being /// synced (which is always reachable via [`CallTarget::Host`]). /// -/// Built by the CLI from the plugin step's declared `canisters` dependencies, -/// resolved against the project's canister ID table. Keeping the resolution on -/// the CLI side keeps this runtime crate free of any manifest knowledge. +/// Built by the CLI from the plugin step's `canisters` list, resolved against +/// the project's canister ID table. Keeping the resolution on the CLI side +/// keeps this runtime crate free of any manifest knowledge. #[derive(Clone, Debug, Default)] pub struct CallableCanisters { - /// Dependencies callable by name ([`CallTarget::Name`]). Maps the name — as - /// it appears in the canister ID table — to the principal it resolves to. + /// Canisters callable by name ([`CallTarget::Name`]). Maps the name — as it + /// appears in the canister ID table — to the principal it resolves to. pub by_name: BTreeMap, } /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing -/// that the plugin declared it as a dependency. The canister being synced -/// (`host`) is always permitted. +/// that the plugin listed it in `canisters`. The canister being synced (`host`) +/// is always permitted. fn resolve_call_target( target: &CallTarget, host_canister_id: Principal, @@ -92,7 +92,7 @@ fn resolve_call_target( struct HostState { /// The canister being synced — the target of [`CallTarget::Host`] calls. host_canister_id: Principal, - /// Canisters the plugin declared as dependencies and may also call. + /// Canisters the plugin declared in `canisters` and may also call. callable: CallableCanisters, agent: Arc, /// Proxy canister to route update calls through, if configured. @@ -404,7 +404,7 @@ pub struct PluginInvocation { /// plugin. Same-project canisters appear both under their fully-qualified /// key and their bare local name (see the WIT `canister-id-entry` docs). pub canister_ids: BTreeMap, - /// Canisters the plugin declared as dependencies and may call, beyond the + /// Canisters the plugin declared in `canisters` and may call, beyond the /// canister being synced. Ignored by v0.1.0 plugins, which can only reach /// the canister being synced. pub callable: CallableCanisters, @@ -800,8 +800,8 @@ mod tests { } /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister - /// and identity, no proxy, no declared dependencies, the default compute - /// limit, and the current directory as the base. Individual tests override + /// and identity, no proxy, no declared callable canisters, the default + /// compute limit, and the current directory as the base. Tests override /// the few fields they care about. fn invocation(wasm_path: &str, environment: &str) -> PluginInvocation { PluginInvocation { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index cace1e32c..c2d51fde2 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -35,18 +35,18 @@ interface types { /// Which canister a `canister-call-request` targets. /// - /// A plugin may target the canister being synced, or any canister it - /// declared as a dependency in the sync step's `canisters` list — always by - /// that canister's name. Targeting a canister that was not declared as a - /// dependency is rejected by the host. + /// A plugin may target the canister being synced, or any canister listed in + /// the sync step's `canisters` list — always by that canister's name. + /// Targeting a canister that was not listed is rejected by the host. variant call-target { /// The canister being synced (`sync-exec-input.canister-id`). Always /// permitted, whether or not it also appears in `canisters`. host, - /// A declared-dependency canister identified by name, spelled exactly as - /// it appears in `sync-exec-input.canister-ids` — a bare local name for a - /// canister in the same subproject, or a `subproject:local` key - /// otherwise. The host resolves it against that mapping table. + /// A canister from the `canisters` list, identified by name, spelled + /// exactly as it appears in `sync-exec-input.canister-ids` — a bare + /// local name for a canister in the same subproject, or a + /// `subproject:local` key otherwise. The host resolves it against that + /// mapping table. name(string), } @@ -72,15 +72,15 @@ interface types { /// the environment being synced, sorted by name. Informational: the /// plugin may use it to resolve canister names it knows about. Being /// listed here does not grant permission to call a canister — that - /// still requires declaring it as a dependency (see `call-target`). + /// still requires listing it in `canisters` (see `call-target`). canister-ids: list, } /// A request to call a method on a canister. record canister-call-request { /// Which canister to call. `host` targets the canister being synced; - /// `name` targets a canister declared as a dependency in the sync - /// step's `canisters` list. + /// `name` targets a canister listed in the sync step's `canisters` + /// list. target: call-target, /// The canister method to call. method: string, @@ -113,9 +113,8 @@ world sync-plugin { /// Make an update or query call to a canister. /// The `req.target` selects the canister: the one being synced (`host`), or - /// a canister declared as a dependency in the sync step's `canisters` list, - /// by name. A target that was not declared as a dependency is rejected - /// without making a call. + /// a canister listed in the sync step's `canisters` list, by name. A target + /// that was not listed is rejected without making a call. /// Returns the raw Candid-encoded response bytes on success or an error /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index 93dfdd0bd..a90ff93be 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -20,7 +20,7 @@ pub struct Params { pub path: PathBuf, pub cid: Principal, /// Fully-qualified store key of the canister being synced (e.g. `backend`, - /// or `services/open-crm:backend` for a dependency canister). Its namespace + /// or `services/open-crm:backend` for a canister in a subproject). Its namespace /// prefix identifies which other canisters are in the same subproject. pub name: String, /// Name of the environment being synced (e.g. "local", "production"). diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 1e525dc22..73a111306 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -31,10 +31,10 @@ pub enum PluginError { Run { source: RunPluginError }, #[snafu(display( - "sync plugin declares a dependency on canister '{name}', but no canister by that name \ + "sync plugin lists canister '{name}' as callable, but no canister by that name \ is known in environment '{environment}'" ))] - UnknownDependency { name: String, environment: String }, + UnknownCallableCanister { name: String, environment: String }, } /// Resolve the plugin compute-time limit, honoring the @@ -101,7 +101,7 @@ pub(super) async fn sync( let files: Vec = adapter.files.clone().unwrap_or_default(); // 3. Build the canister ID table exposed to the plugin, then resolve the - // plugin's declared callable canisters against it. + // step's `canisters` list against it. let canister_ids = exposed_canister_ids(params); let callable = resolve_callable(adapter, &canister_ids, environment)?; @@ -137,9 +137,10 @@ pub(super) async fn sync( /// The canister ID table exposed to a sync plugin: every named canister in the /// project, plus — for canisters in the same subproject as the one being synced /// — a duplicate entry under the bare local name. A store key is -/// `:` for a dependency canister and a bare local name for a -/// canister defined directly in the app root (see the WIT `canister-id-entry` -/// docs), so the syncing canister's namespace is the prefix of its own key. +/// `:` for a canister in a subproject and a bare local name +/// for a canister defined directly in the app root (see the WIT +/// `canister-id-entry` docs), so the syncing canister's namespace is the prefix +/// of its own key. /// /// A local name never contains a colon but a subproject directory may, so keys /// split on their *last* colon. The bare-name aliases take precedence over an @@ -159,9 +160,9 @@ fn exposed_canister_ids(params: &Params) -> BTreeMap { table } -/// Resolve the canisters a plugin declared it may call into a [`CallableCanisters`] -/// enforcement set. Each declared name is looked up in `canister_ids`; a name -/// that does not resolve is a manifest error. +/// Resolve the step's `canisters` list into a [`CallableCanisters`] enforcement +/// set. Each listed name is looked up in `canister_ids`; a name that does not +/// resolve is a manifest error. fn resolve_callable( adapter: &Adapter, canister_ids: &BTreeMap, @@ -172,7 +173,7 @@ fn resolve_callable( let principal = canister_ids .get(name) .copied() - .context(UnknownDependencySnafu { + .context(UnknownCallableCanisterSnafu { name: name.clone(), environment: environment.to_owned(), })?; @@ -349,6 +350,6 @@ mod tests { let adapter = adapter_with(Some(vec!["nope".to_owned()])); let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") .expect_err("an undeclared name must fail"); - assert!(matches!(err, PluginError::UnknownDependency { .. })); + assert!(matches!(err, PluginError::UnknownCallableCanister { .. })); } } diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 68a76c686..099825626 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -49,7 +49,7 @@ pub struct Adapter { /// Canisters this plugin may call in addition to the canister being synced. /// Each entry is a canister name resolved against the project's canister ID /// table for the environment being synced (e.g. `backend`, or a namespaced - /// dependency canister such as `services/open-crm:backend`). The plugin + /// subproject canister such as `services/open-crm:backend`). The plugin /// picks a target per call via the `call-target` in its `canister-call` /// request; a target not listed here is rejected by the host. pub canisters: Option>, diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 4dae4da32..144c30ec6 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it declares as dependencies. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -15,7 +15,7 @@ Sync plugins fill that gap. A plugin is: - **Portable** — written in any language that compiles to `wasm32-wasip2`, distributed as one `.wasm` file (local path or remote URL + `sha256`). - **Sandboxed** — it cannot open network sockets, spawn subprocesses, or touch the filesystem outside the directories you explicitly grant it. -- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister it declares as a dependency in the manifest's `canisters:` list. A call to a canister that was not declared is rejected by the host. +- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister listed in the manifest's `canisters:` list. A call to a canister that was not listed is rejected by the host. The most common way to get a sync plugin is through a [recipe](recipes.md). For example, the `@dfinity/asset-canister` recipe emits a `plugin` sync step (starting with `v2.2.1`) that uploads your built static files to the asset canister — so for everyday frontend deployment you never write a plugin yourself. @@ -40,7 +40,7 @@ icp sync │ └─ plugin makes canister-call({ target, ... }) (× N) target = host (the canister being synced), or a - declared-dependency canister by name + canister from `canisters:` by name ``` ## The Plugin Interface @@ -49,7 +49,7 @@ The interface is defined as a [WIT](https://component-model.bytecodealliance.org ```wit world sync-plugin { - // Host import: call the canister being synced or a declared dependency. + // Host import: call the canister being synced or one listed in `canisters:`. import canister-call: func(req: canister-call-request) -> result, string>; // Plugin export: run the sync step. @@ -73,7 +73,7 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | | `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | -Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister that came from a dependency. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. +Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister defined in a subproject. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. ### Calling a canister — `canister-call` diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 7d885a292..3ba6ccc62 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -172,7 +172,7 @@ sync: Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. -A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a dependency canister. A name that does not resolve to a known canister for the environment fails the sync step. +A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a canister defined in a subproject. A name that does not resolve to a known canister for the environment fails the sync step. The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index c361b4562..9b9db63d9 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 5f27fa902..87186951b 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, From 668646bb6b8aef7cd25f3d8074a4c6a79339374b Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Wed, 19 Aug 2026 07:37:27 -0700 Subject: [PATCH 14/51] Add kv fields to plugin input --- crates/icp-cli/src/operations/bundle.rs | 1 + crates/icp-sync-plugin/DESIGN.md | 5 ++- crates/icp-sync-plugin/src/runtime.rs | 43 +++++++++++++++++++ crates/icp-sync-plugin/sync-plugin.wit | 15 ++++++- .../tests/fixtures/test-plugin/src/lib.rs | 17 ++++++++ crates/icp/src/canister/sync/plugin.rs | 8 ++++ crates/icp/src/manifest/adapter/plugin.rs | 33 ++++++++++++++ crates/icp/src/manifest/canister.rs | 2 + docs/concepts/sync-plugins.md | 5 ++- docs/guides/writing-sync-plugins.md | 14 +++++- docs/reference/configuration.md | 5 ++- docs/schemas/canister-yaml-schema.json | 12 +++++- docs/schemas/icp-yaml-schema.json | 12 +++++- 13 files changed, 163 insertions(+), 9 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index bd460d576..dd90010b4 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -809,6 +809,7 @@ async fn prepare_plugin_step( dirs: bundle_dirs, files: bundle_files, canisters: localize_call_targets(adapter.canisters.as_deref(), local_names), + fields: adapter.fields.clone(), })) } diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 9d57338c9..56b563944 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -75,7 +75,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin ``` `PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `dirs`, `files`, -`host_canister_id` (the canister being synced), `agent`, `proxy`, +`fields`, `host_canister_id` (the canister being synced), `agent`, `proxy`, `identity_principal`, `environment`, `compute_limit_secs`, the exposed `canister_ids` table, the `callable: CallableCanisters` enforcement set, and `stdio`. The CLI resolves the manifest's declared `canisters:` into @@ -188,7 +188,8 @@ pub struct Adapter { pub sha256: Option, pub dirs: Option>, pub files: Option>, - pub canisters: Option>, // extra callable canisters, by name + pub fields: Option>, // inline key-value fields + pub canisters: Option>, // extra callable canisters, by name } ``` diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 4632a1ab9..46c02cda5 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -388,6 +388,9 @@ pub struct PluginInvocation { pub dirs: Vec, /// Manifest-relative files to read and pass inline. pub files: Vec, + /// Key-value fields to pass inline. Passed to v0.2.0 plugins; ignored by + /// v0.1.0 plugins, whose interface has no `fields`. + pub fields: BTreeMap, /// The canister being synced. Reachable via `call-target::host`. pub host_canister_id: Principal, /// Agent used for canister calls. @@ -418,6 +421,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin base_dir, dirs, files, + fields, host_canister_id, agent, proxy, @@ -573,6 +577,10 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin .into_iter() .map(|(name, content)| v2::FileInput { name, content }) .collect(), + fields: fields + .into_iter() + .map(|(name, value)| v2::FieldInput { name, value }) + .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, canister_ids: canister_ids @@ -809,6 +817,7 @@ mod tests { base_dir: ".".into(), dirs: vec![], files: vec![], + fields: BTreeMap::new(), host_canister_id: anon(), agent: dummy_agent(), proxy: None, @@ -1008,6 +1017,40 @@ mod tests { assert!(msg.contains("stdout from plugin"), "got: {msg}"); } + #[tokio::test(flavor = "multi_thread")] + async fn plugin_fields_are_passed_through() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let (tx, mut rx) = tokio::sync::mpsc::channel::(16); + let result = tokio::task::block_in_place(|| { + let mut inv = invocation(wasm_path, "fields"); + inv.fields = BTreeMap::from([ + ("greeting".to_string(), "hi".to_string()), + ("audience".to_string(), "world".to_string()), + ]); + inv.stdio = Some(tx); + run_plugin(inv) + }); + assert!(result.is_ok()); + let echoed = rx + .try_recv() + .expect("expected the plugin to echo its fields"); + assert_eq!(echoed, "audience=world,greeting=hi"); + } + + #[test] + fn plugin_missing_expected_field_fails() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + // The "fields" fixture requires a `greeting` field; passing none fails. + assert!(matches!( + run_plugin(invocation(wasm_path, "fields")), + Err(RunPluginError::PluginFailed { ref message }) if message == "missing 'greeting' field" + )); + } + #[test] fn legacy_v1_plugin_is_detected_and_driven() { // A plugin built against the v0.1.0 interface must still load: the host diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index c2d51fde2..a19c609f2 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -14,6 +14,16 @@ interface types { content: string, } + /// A key-value field declared in the manifest step's `fields` setting. + /// The host passes the plugin's declared fields inline; how they are + /// interpreted is up to the plugin. + record field-input { + /// Field name, as spelled in the manifest. + name: string, + /// Field value. + value: string, + } + /// An entry in the project's canister ID mapping table: a canister name /// and the textual principal it resolves to in the environment being synced. record canister-id-entry { @@ -63,6 +73,9 @@ interface types { /// Files declared in the manifest step's `files` setting, read by /// the host and passed inline. The plugin decides how to use them. files: list, + /// Key-value fields declared in the manifest step's `fields` setting, + /// passed inline. The plugin decides how to use them. + fields: list, /// Textual principal of the signing identity used for canister calls. identity-principal: string, /// Textual principal of the proxy canister, if one was configured via @@ -105,7 +118,7 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input}; + use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 47b82c26a..4093ef226 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -19,6 +19,23 @@ impl Guest for TestPlugin { println!("stdout from plugin"); Ok(()) } + // Echo the fields back so the host can assert on ordering and + // values; fail if the expected field is missing. + "fields" => { + let rendered = input + .fields + .iter() + .map(|f| format!("{}={}", f.name, f.value)) + .collect::>() + .join(","); + match input.fields.iter().find(|f| f.name == "greeting") { + Some(_) => { + eprintln!("{rendered}"); + Ok(()) + } + None => Err("missing 'greeting' field".to_string()), + } + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 73a111306..da28b7288 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -99,6 +99,12 @@ pub(super) async fn sync( let base_dir = Utf8PathBuf::from(params.path.as_str()); let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); + let fields: BTreeMap = adapter + .fields + .clone() + .unwrap_or_default() + .into_iter() + .collect(); // 3. Build the canister ID table exposed to the plugin, then resolve the // step's `canisters` list against it. @@ -120,6 +126,7 @@ pub(super) async fn sync( base_dir, dirs, files, + fields, host_canister_id: params.cid, agent: agent_clone, proxy, @@ -231,6 +238,7 @@ mod tests { sha256: None, dirs: None, files: None, + fields: None, canisters, } } diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 099825626..e0daaa664 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; @@ -20,6 +22,8 @@ use super::prebuilt::SourceField; /// - assets/seed-data /// files: # files read by the host and passed inline /// - config.txt +/// fields: # key-value fields passed inline +/// api_url: https://example.com /// ``` /// /// Example (remote URL — `sha256` is required): @@ -46,6 +50,10 @@ pub struct Adapter { /// the plugin as part of `sync-exec-input.files`. pub files: Option>, + /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. + /// Values are strings; the plugin decides how to interpret them. + pub fields: Option>, + /// Canisters this plugin may call in addition to the canister being synced. /// Each entry is a canister name resolved against the project's canister ID /// table for the environment being synced (e.g. `backend`, or a namespaced @@ -64,6 +72,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, + fields: Option>, canisters: Option>, } @@ -78,6 +87,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: h.sha256, dirs: h.dirs, files: h.files, + fields: h.fields, canisters: h.canisters, }) } @@ -104,6 +114,7 @@ mod tests { sha256: None, dirs: None, files: None, + fields: None, canisters: None, }, ); @@ -131,11 +142,32 @@ mod tests { sha256: Some("abc123".to_string()), dirs: Some(vec!["assets/seed-data".to_string(), "config".to_string()]), files: Some(vec!["config.txt".to_string()]), + fields: None, canisters: None, }, ); } + #[test] + fn fields_parse_as_a_string_map() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + fields: + api_url: https://example.com + token: abc123 + "#, + ) + .expect("failed to deserialize Adapter with fields"); + assert_eq!( + adapter.fields, + Some(HashMap::from([ + ("api_url".to_string(), "https://example.com".to_string()), + ("token".to_string(), "abc123".to_string()), + ])), + ); + } + #[test] fn remote_url_without_sha256_is_rejected() { let err = serde_yaml::from_str::( @@ -188,6 +220,7 @@ mod tests { sha256: Some("a665a45920422f9d417e".to_string()), dirs: None, files: None, + fields: None, canisters: None, }, ); diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 8d2838a5b..28a576f9b 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -793,6 +793,7 @@ mod tests { sha256: None, dirs: Some(vec!["assets/seed-data/".to_string()]), files: None, + fields: None, canisters: None, } )] @@ -838,6 +839,7 @@ mod tests { ), dirs: None, files: None, + fields: None, canisters: None, })] }), diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 144c30ec6..a4b027dbb 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -25,7 +25,7 @@ When a `plugin` sync step executes for a canister, icp-cli: 1. Resolves the wasm — reads the local `path`, or downloads the `url` to the package cache. 2. Verifies the `sha256` checksum if one is given (required for `url`). -3. Reads any files listed in `files:` and preopens any directories listed in `dirs:` read-only. +3. Reads any files listed in `files:`, preopens any directories listed in `dirs:` read-only, and collects any key-value pairs listed in `fields:`. 4. Instantiates the component in a WASI sandbox and calls its `exec()` export. 5. Forwards the plugin's output to the CLI and reports success or the returned error. @@ -36,7 +36,7 @@ icp sync │ canister-id = │ identity-principal = │ canister-ids = - │ dirs / files = what you declared in the manifest + │ dirs/files/fields = what you declared in the manifest │ └─ plugin makes canister-call({ target, ... }) (× N) target = host (the canister being synced), or a @@ -69,6 +69,7 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `environment` | Name of the environment being synced (e.g. `local`, `production`) | | `dirs` | The directories you declared in `dirs:`; the host preopened each one read-only | | `files` | The files you declared in `files:`, each as a `(name, content)` pair read by the host | +| `fields` | The key-value fields you declared in `fields:`, each as a `(name, value)` pair; values are strings | | `identity-principal` | Textual principal of the signing identity used for canister calls | | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | | `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 80cf1f46e..0a4428397 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -114,6 +114,16 @@ for file in &input.files { Writes, paths outside a preopen, and `..` traversal are all rejected by the sandbox. See [The Sandbox](../concepts/sync-plugins.md#the-sandbox) for the full capability list and resource limits. +## Read Declared Fields + +Key-value pairs declared in the manifest's `fields:` are passed inline as string values. Use them for small configuration a plugin needs without shipping a file: + +```rust +for field in &input.fields { + println!("{} = {}", field.name, field.value); +} +``` + ## Build ```bash @@ -124,7 +134,7 @@ The output `.wasm` (under `target/wasm32-wasip2/release/`) is loaded directly by ## Wire It Into the Manifest -Reference the built wasm from a `plugin` sync step and declare the files and directories the plugin needs: +Reference the built wasm from a `plugin` sync step and declare the files, directories, and fields the plugin needs: ```yaml sync: @@ -135,6 +145,8 @@ sync: - seed-data files: - config.txt + fields: + api_url: https://example.com ``` Then run the sync phase: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 3ba6ccc62..95539c6f8 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -151,7 +151,9 @@ sync: - config files: # files read by the host and passed inline - config.txt - canisters: # extra canisters the plugin may call, + fields: # key-value fields passed inline + api_url: https://example.com + canisters: # extra canisters the plugin may call - ledger # by name (resolved for the environment) - services/open-crm:backend @@ -168,6 +170,7 @@ sync: | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | | `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | | `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | +| `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | | `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 9b9db63d9..711169c87 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -89,6 +89,16 @@ "null" ] }, + "fields": { + "additionalProperties": { + "type": "string" + }, + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "type": [ + "object", + "null" + ] + }, "files": { "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", "items": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 87186951b..303db6e37 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -89,6 +89,16 @@ "null" ] }, + "fields": { + "additionalProperties": { + "type": "string" + }, + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "type": [ + "object", + "null" + ] + }, "files": { "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", "items": { From 4eb273a22c202902a47cf4a9b8e64544f775b2eb Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 08:34:39 -0700 Subject: [PATCH 15/51] improvements --- crates/icp-cli/tests/bundle_tests.rs | 65 ++++++++ crates/icp-sync-plugin/DESIGN.md | 20 ++- crates/icp-sync-plugin/src/runtime.rs | 29 ++-- .../tests/fixtures/test-plugin/src/lib.rs | 15 +- crates/icp/src/canister/sync/plugin.rs | 7 +- crates/icp/src/manifest/adapter/plugin.rs | 153 +++++++++++++++++- docs/guides/writing-sync-plugins.md | 3 + docs/reference/configuration.md | 3 + docs/schemas/canister-yaml-schema.json | 10 +- docs/schemas/icp-yaml-schema.json | 10 +- 10 files changed, 267 insertions(+), 48 deletions(-) diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 67b8a014f..1500ec340 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -768,6 +768,71 @@ fn bundle_normalizes_dotdot_within_project() { ); } +/// Unlike `dirs`/`files`, a plugin step's `fields` reference nothing on disk, so bundling must +/// carry them into the rewritten manifest verbatim — a deploy from the bundle sees the same +/// configuration the original project declared. +#[test] +fn bundle_preserves_plugin_fields() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + write(&project_dir.join("plugin.wasm"), b"\x00asm\x01\x00\x00\x00") + .expect("failed to write plugin wasm"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: plugin.wasm + fields: + api_url: https://example.com + port: 8080 + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut manifest_yaml = String::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path == "icp.yaml" { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + } + + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let fields = &parsed["canisters"][0]["sync"]["steps"][0]["fields"]; + assert_eq!(fields["api_url"].as_str(), Some("https://example.com")); + // `port` was written unquoted; loading stringifies it, so the rewritten + // manifest carries a string too. + assert_eq!(fields["port"].as_str(), Some("8080")); +} + /// A plugin sync step whose `dirs` entry resolves *outside* the project directory must be /// rejected. Bundles can only reference files inside the project so the produced archive is portable. #[test] diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 56b563944..8737dffab 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -153,7 +153,10 @@ instantiates the matching `bindgen!` world and builds the matching trial instantiation: it is unambiguous and needs no throwaway `Store`. A component with no recognized `icp:sync-plugin/types@` import, or an unsupported version, is rejected with `UnsupportedInterface`. Both `.wit` files -are checked in; `sync-plugin-v1.wit` is the frozen v0.1.0 contract. +are checked in; `sync-plugin-v1.wit` is the frozen v0.1.0 contract. Inputs the +v0.1.0 `sync-exec-input` has no field for — `canister-ids` and `fields` — are +simply dropped for a v1 plugin; a v1 plugin cannot observe them, so declaring +`fields:` alongside one has no effect. ### Compute budget (epoch interruption) @@ -188,14 +191,25 @@ pub struct Adapter { pub sha256: Option, pub dirs: Option>, pub files: Option>, - pub fields: Option>, // inline key-value fields + pub fields: Option>, // inline key-value fields pub canisters: Option>, // extra callable canisters, by name } ``` Each `canisters:` entry is a canister name resolved against the project's ID table for the environment being synced. `Deserialize` is hand-written to reject a -`url` source without a `sha256`. +`url` source without a `sha256`. `fields` is a `BTreeMap` rather than a `HashMap` +so that re-serializing the adapter (the bundler writes a consolidated manifest) +is byte-stable; the WIT interface itself makes no promise about the order fields +arrive in. + +Each `fields` value deserializes through `FieldValue`, which takes any YAML +scalar and stringifies it, so `retries: 3` need not be quoted. `serde_yaml` does +that coercion itself when reading YAML *text*, but a canister's build/sync +section reaches the adapter as an already-parsed `serde_yaml::Value` (see +`CanisterManifest`'s hand-written `Deserialize`), and re-deserializing from a +`Value` keeps a number a number — hence the explicit visitor. Lists, mappings, +and empty values are rejected: there is no string to hand the plugin. ### `crates/icp/src/canister/sync/plugin.rs` diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 46c02cda5..708dc241f 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1017,26 +1017,21 @@ mod tests { assert!(msg.contains("stdout from plugin"), "got: {msg}"); } - #[tokio::test(flavor = "multi_thread")] - async fn plugin_fields_are_passed_through() { + #[test] + fn plugin_fields_are_passed_through() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let (tx, mut rx) = tokio::sync::mpsc::channel::(16); - let result = tokio::task::block_in_place(|| { - let mut inv = invocation(wasm_path, "fields"); - inv.fields = BTreeMap::from([ - ("greeting".to_string(), "hi".to_string()), - ("audience".to_string(), "world".to_string()), - ]); - inv.stdio = Some(tx); - run_plugin(inv) - }); - assert!(result.is_ok()); - let echoed = rx - .try_recv() - .expect("expected the plugin to echo its fields"); - assert_eq!(echoed, "audience=world,greeting=hi"); + let mut inv = invocation(wasm_path, "fields"); + inv.fields = BTreeMap::from([ + ("greeting".to_string(), "hi".to_string()), + ("audience".to_string(), "world".to_string()), + ]); + // The "fields" fixture echoes what it received to stderr, which + // run_plugin returns. The interface promises no field order, but the + // BTreeMap makes the host's order name-sorted in practice. + let lines = run_plugin(inv).expect("plugin should succeed"); + assert_eq!(lines, vec!["audience=world,greeting=hi".to_string()]); } #[test] diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 4093ef226..63fc360fd 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -19,22 +19,19 @@ impl Guest for TestPlugin { println!("stdout from plugin"); Ok(()) } - // Echo the fields back so the host can assert on ordering and - // values; fail if the expected field is missing. "fields" => { + if !input.fields.iter().any(|f| f.name == "greeting") { + return Err("missing 'greeting' field".to_string()); + } + // Echo the fields back so the host can assert on what arrived. let rendered = input .fields .iter() .map(|f| format!("{}={}", f.name, f.value)) .collect::>() .join(","); - match input.fields.iter().find(|f| f.name == "greeting") { - Some(_) => { - eprintln!("{rendered}"); - Ok(()) - } - None => Err("missing 'greeting' field".to_string()), - } + eprintln!("{rendered}"); + Ok(()) } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index da28b7288..eee54a1b4 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -99,12 +99,7 @@ pub(super) async fn sync( let base_dir = Utf8PathBuf::from(params.path.as_str()); let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); - let fields: BTreeMap = adapter - .fields - .clone() - .unwrap_or_default() - .into_iter() - .collect(); + let fields: BTreeMap = adapter.fields.clone().unwrap_or_default(); // 3. Build the canister ID table exposed to the plugin, then resolve the // step's `canisters` list against it. diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index e0daaa664..c84ab6e3c 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,10 +1,83 @@ -use std::collections::HashMap; +use std::{collections::BTreeMap, fmt}; use schemars::JsonSchema; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{ + Deserialize, Deserializer, Serialize, + de::{self, Visitor}, +}; use super::prebuilt::SourceField; +/// One `fields:` value on its way in from the manifest. A plugin always receives +/// a string, but writing `port: 8080` should not have to be quoted, so any YAML +/// scalar is accepted and stringified. Lists, mappings, and empty values are +/// rejected: there is no string to hand the plugin. +/// +/// Note this cannot be left to serde's own `String` handling. `serde_yaml` +/// coerces scalars when deserializing straight from YAML text, but the manifest +/// is parsed into a `serde_yaml::Value` first (see `CanisterManifest`'s +/// `Deserialize`), and re-deserializing from a `Value` keeps a number a number. +struct FieldValue(String); + +impl<'de> Deserialize<'de> for FieldValue { + fn deserialize>(d: D) -> Result { + struct ScalarVisitor; + + impl Visitor<'_> for ScalarVisitor { + type Value = FieldValue; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, number, or boolean") + } + + fn visit_str(self, v: &str) -> Result { + Ok(FieldValue(v.to_owned())) + } + + fn visit_i64(self, v: i64) -> Result { + Ok(FieldValue(v.to_string())) + } + + fn visit_u64(self, v: u64) -> Result { + Ok(FieldValue(v.to_string())) + } + + fn visit_f64(self, v: f64) -> Result { + Ok(FieldValue(v.to_string())) + } + + fn visit_bool(self, v: bool) -> Result { + Ok(FieldValue(v.to_string())) + } + } + + d.deserialize_any(ScalarVisitor) + } +} + +impl JsonSchema for FieldValue { + fn schema_name() -> std::borrow::Cow<'static, str> { + "FieldValue".into() + } + + fn inline_schema() -> bool { + true + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": ["string", "number", "boolean"], + }) + } +} + +fn deserialize_fields<'de, D: Deserializer<'de>>( + d: D, +) -> Result>, D::Error> { + let fields = Option::>::deserialize(d)?; + Ok(fields.map(|fields| fields.into_iter().map(|(k, v)| (k, v.0)).collect())) +} + /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -24,6 +97,7 @@ use super::prebuilt::SourceField; /// - config.txt /// fields: # key-value fields passed inline /// api_url: https://example.com +/// retries: 3 /// ``` /// /// Example (remote URL — `sha256` is required): @@ -51,8 +125,10 @@ pub struct Adapter { pub files: Option>, /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. - /// Values are strings; the plugin decides how to interpret them. - pub fields: Option>, + /// A plugin receives every value as a string; a number or boolean written + /// unquoted arrives as its text form. The plugin decides how to interpret them. + #[schemars(with = "Option>")] + pub fields: Option>, /// Canisters this plugin may call in addition to the canister being synced. /// Each entry is a canister name resolved against the project's canister ID @@ -72,7 +148,8 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, - fields: Option>, + #[serde(default, deserialize_with = "deserialize_fields")] + fields: Option>, canisters: Option>, } @@ -95,6 +172,8 @@ impl<'de> Deserialize<'de> for Adapter { #[cfg(test)] mod tests { + use indoc::indoc; + use super::*; use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; @@ -148,9 +227,19 @@ mod tests { ); } + /// Parse an adapter the way the manifest loader does: YAML text into a + /// `serde_yaml::Value`, then that value into the typed adapter. Going + /// through the value matters for `fields` — deserializing straight from + /// text lets `serde_yaml` coerce scalars to strings on its own, which + /// would hide whether `FieldValue` accepts them. + fn adapter_via_value(yaml: &str) -> Result { + let value: serde_yaml::Value = serde_yaml::from_str(yaml).expect("invalid yaml"); + serde_yaml::from_value(value) + } + #[test] fn fields_parse_as_a_string_map() { - let adapter = serde_yaml::from_str::( + let adapter = adapter_via_value( r#" path: plugins/my-sync.wasm fields: @@ -161,13 +250,63 @@ mod tests { .expect("failed to deserialize Adapter with fields"); assert_eq!( adapter.fields, - Some(HashMap::from([ + Some(BTreeMap::from([ ("api_url".to_string(), "https://example.com".to_string()), ("token".to_string(), "abc123".to_string()), ])), ); } + #[test] + fn scalar_field_values_are_stringified() { + let adapter = adapter_via_value( + r#" + path: plugins/my-sync.wasm + fields: + port: 8080 + enabled: true + ratio: 1.5 + "#, + ) + .expect("failed to deserialize Adapter with scalar fields"); + assert_eq!( + adapter.fields, + Some(BTreeMap::from([ + ("port".to_string(), "8080".to_string()), + ("enabled".to_string(), "true".to_string()), + ("ratio".to_string(), "1.5".to_string()), + ])), + ); + } + + #[test] + fn non_scalar_field_values_are_rejected() { + for yaml in [ + // A plugin can only receive a string, so there is nothing sensible + // to hand it for a nested mapping... + indoc! {r#" + path: plugins/my-sync.wasm + fields: + nested: + a: b + "#}, + // ...or for a key written with no value at all. + indoc! {r#" + path: plugins/my-sync.wasm + fields: + blank: + "#}, + ] { + let err = + adapter_via_value(yaml).expect_err("non-scalar field value should be rejected"); + assert!( + err.to_string() + .contains("expected a string, number, or boolean"), + "unexpected error: {err}" + ); + } + } + #[test] fn remote_url_without_sha256_is_rejected() { let err = serde_yaml::from_str::( diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 0a4428397..9b6f28282 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -124,6 +124,8 @@ for field in &input.fields { } ``` +A value always arrives as a string, so parse the ones you want as another type — a manifest may write `retries: 3` unquoted, and the plugin receives `"3"`. + ## Build ```bash @@ -147,6 +149,7 @@ sync: - config.txt fields: api_url: https://example.com + retries: 3 ``` Then run the sync phase: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 95539c6f8..0518287d9 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -153,6 +153,7 @@ sync: - config.txt fields: # key-value fields passed inline api_url: https://example.com + retries: 3 canisters: # extra canisters the plugin may call - ledger # by name (resolved for the environment) - services/open-crm:backend @@ -175,6 +176,8 @@ sync: Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. +A plugin receives every `fields:` value as a string. Numbers and booleans need no quoting — `port: 8080` arrives as `"8080"` — but a value may not be a list, a mapping, or empty. + A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a canister defined in a subproject. A name that does not resolve to a known canister for the environment fails the sync step. The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 711169c87..4632f9cb0 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -91,9 +91,13 @@ }, "fields": { "additionalProperties": { - "type": "string" + "type": [ + "string", + "number", + "boolean" + ] }, - "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nA plugin receives every value as a string; a number or boolean written\nunquoted arrives as its text form. The plugin decides how to interpret them.", "type": [ "object", "null" diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 303db6e37..3121bf499 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -91,9 +91,13 @@ }, "fields": { "additionalProperties": { - "type": "string" + "type": [ + "string", + "number", + "boolean" + ] }, - "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nA plugin receives every value as a string; a number or boolean written\nunquoted arrives as its text form. The plugin decides how to interpret them.", "type": [ "object", "null" From d09ed9961408f2cacc49132dfdbb73ce13d9587e Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 06:32:52 -0700 Subject: [PATCH 16/51] Add key names to files/directories for plugins --- crates/icp-cli/src/operations/bundle.rs | 38 ++- crates/icp-cli/tests/sync_tests.rs | 69 +++++ crates/icp-sync-plugin/DESIGN.md | 21 +- crates/icp-sync-plugin/src/lib.rs | 2 +- crates/icp-sync-plugin/src/runtime.rs | 107 +++++-- crates/icp-sync-plugin/sync-plugin.wit | 23 +- .../tests/fixtures/test-plugin/src/lib.rs | 11 + crates/icp/src/canister/sync/plugin.rs | 25 +- crates/icp/src/manifest/adapter/plugin.rs | 278 +++++++++++++++++- crates/icp/src/manifest/canister.rs | 9 +- docs/concepts/sync-plugins.md | 6 +- docs/guides/writing-sync-plugins.md | 10 +- docs/reference/configuration.md | 18 +- docs/schemas/canister-yaml-schema.json | 67 +++-- docs/schemas/icp-yaml-schema.json | 67 +++-- examples/icp-sync-plugin/plugin/src/lib.rs | 2 +- 16 files changed, 654 insertions(+), 99 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index dd90010b4..94eef4268 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -766,39 +766,49 @@ async fn prepare_plugin_step( // Plugin preopened dirs go under a `dirs/` subdir so a user-supplied dir literally named // `files` cannot collide with the `files/` area used for plugin input files. + // The declared paths are rewritten to their archive locations; each entry's + // map key is carried through unchanged. let bundle_dirs = adapter.dirs.as_ref().map(|dirs| { - dirs.iter() + dirs.entries() + .iter() .map(|d| { let manifest_path = format!( "plugins/{path_name}/{idx}/dirs/{}", - normalize_archive_dir(d) + normalize_archive_dir(&d.path) ); out.plugin_dirs.push(DirEntry { - src_path: canister_path.join(d), + src_path: canister_path.join(&d.path), archive_prefix: archive_join(prefix, &manifest_path), }); - manifest_path + plugin::NamedPath { + key: d.key.clone(), + path: manifest_path, + } }) - .collect::>() + .collect::() }); let bundle_files = adapter.files.as_ref().map(|files| { files + .entries() .iter() .map(|f| { let manifest_path = format!( "plugins/{path_name}/{idx}/files/{}", - normalize_archive_dir(f) + normalize_archive_dir(&f.path) ); out.plugin_files.push(PluginFile { - src_path: canister_path.join(f), + src_path: canister_path.join(&f.path), archive_path: archive_join(prefix, &manifest_path), canister_name: canister.name.clone(), - orig_file: f.clone(), + orig_file: f.path.clone(), }); - manifest_path + plugin::NamedPath { + key: f.key.clone(), + path: manifest_path, + } }) - .collect::>() + .collect::() }); Ok(SyncStep::Plugin(plugin::Adapter { @@ -1301,8 +1311,8 @@ fn validate_source_paths( SyncStep::Script(_) => {} SyncStep::Plugin(adapter) => { if let Some(dirs) = &adapter.dirs { - for d in dirs { - let src = canister_path.join(d); + for d in dirs.entries() { + let src = canister_path.join(&d.path); let resolved = resolve_within_project( &src, project_dir, @@ -1313,8 +1323,8 @@ fn validate_source_paths( } } if let Some(files) = &adapter.files { - for f in files { - let src = canister_path.join(f); + for f in files.entries() { + let src = canister_path.join(&f.path); resolve_within_project( &src, project_dir, diff --git a/crates/icp-cli/tests/sync_tests.rs b/crates/icp-cli/tests/sync_tests.rs index 31f72ef48..888dabdb6 100644 --- a/crates/icp-cli/tests/sync_tests.rs +++ b/crates/icp-cli/tests/sync_tests.rs @@ -465,6 +465,75 @@ async fn sync_plugin_registers_seed_data() { ); } +/// `dirs:` may be written as a map (name → path, or name → list of paths) +/// instead of a plain list. The declared paths are still preopened and traversed +/// the same way, so registration works end-to-end; this proves the map form +/// deserializes and reaches the runtime. +#[tokio::test] +async fn sync_plugin_accepts_map_form_dirs() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + + let (canister_wasm, plugin_wasm) = build_sync_plugin_example(); + + // Two directories, declared under one map key as a list. + let fruit = project_dir.join("fruit"); + let veg = project_dir.join("veg"); + create_dir_all(&fruit).expect("failed to create fruit dir"); + create_dir_all(&veg).expect("failed to create veg dir"); + write_string(&fruit.join("fruit-01.txt"), "apple").expect("failed to write fruit-01.txt"); + write_string(&veg.join("veg-01.txt"), "carrot").expect("failed to write veg-01.txt"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{canister_wasm}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: {plugin_wasm} + dirs: + produce: + - fruit + - veg + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let _g = ctx.start_network_in(&project_dir, "random-network").await; + ctx.ping_until_healthy(&project_dir, "random-network"); + + clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) + .mint_cycles(10 * TRILLION); + + ctx.icp() + .current_dir(&project_dir) + .args(["deploy", "--environment", "random-environment"]) + .assert() + .success(); + + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "call", + "my-canister", + "show", + "()", + "--query", + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout(contains("apple").and(contains("carrot"))); +} + /// A malformed `ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS` must abort the sync with an /// actionable error rather than being silently ignored. This also exercises the /// end-to-end wiring: it proves the override is actually read on the real plugin diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 8737dffab..326a9b6a5 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -82,10 +82,11 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin `CallableCanisters` before calling; this crate stays free of any manifest knowledge. -`dirs` and `files` are the manifest-relative path strings, straight from the -adapter. The runtime owns *all* filesystem access anchored at `base_dir`: it -preopens each `dir` from `base_dir.join(dir)` and reads each `file` from -`base_dir.join(file)`, passing the contents inline in `SyncExecInput`. Keeping +`dirs` and `files` are the manifest-relative paths (as `KeyedPath`s carrying the +map key each was declared under, if any), straight from the adapter. The runtime +owns *all* filesystem access anchored at `base_dir`: it preopens each `dir` from +`base_dir.join(dir.path)` and reads each `file` from `base_dir.join(file.path)`, +passing the contents — and the keys — inline in `SyncExecInput`. Keeping both inside the runtime means the path-safety logic (below) lives in one place and stays private to this crate — the CLI just forwards strings. The returned `Vec` is the plugin's persistent stderr lines (see stdio capture below); @@ -189,13 +190,21 @@ Deserializes the `canister.yaml` fields into: pub struct Adapter { pub source: SourceField, // path: or url: pub sha256: Option, - pub dirs: Option>, - pub files: Option>, + pub dirs: Option, + pub files: Option, pub fields: Option>, // inline key-value fields pub canisters: Option>, // extra callable canisters, by name } ``` +`NamedPaths` deserializes `dirs:`/`files:` from either a plain list of paths or a +map of name → path (or list of paths), flattening to an ordered list of +`(key, path)` entries: `key` is `None` for a list entry and `Some(name)` for a +map entry, and is *non-unique* — a map key that resolves to a list of paths +produces one entry per path, all sharing the key. The CLI passes these to the +runtime as `KeyedPath`s (this crate stays free of manifest types), which surface +in `sync-exec-input.dirs`/`files` as each entry's `key`. + Each `canisters:` entry is a canister name resolved against the project's ID table for the environment being synced. `Deserialize` is hand-written to reject a `url` source without a `sha256`. `fields` is a `BTreeMap` rather than a `HashMap` diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index 053be28a3..d2fa023b4 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -2,6 +2,6 @@ mod path; mod runtime; pub use runtime::{ - CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, }; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 708dc241f..719a920a9 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -56,6 +56,20 @@ mod v1 { use v2::icp::sync_plugin::types::{CallTarget, CallType, CanisterIdEntry}; +/// A manifest path passed to a plugin, tagged with the map key it was declared +/// under. Both `dirs` and `files` are lists of these. +/// +/// The key is `None` when the manifest wrote the setting as a plain list, and +/// `Some(name)` when it wrote a map. It is *non-unique*: several paths share a +/// key when a map key resolves to a list of paths. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KeyedPath { + /// The map key this path was declared under, or `None` for a plain-list entry. + pub key: Option, + /// Manifest-relative path, anchored at the invocation's `base_dir`. + pub path: String, +} + /// The canisters a sync plugin is permitted to call, beyond the canister being /// synced (which is always reachable via [`CallTarget::Host`]). /// @@ -384,10 +398,12 @@ pub struct PluginInvocation { pub wasm_path: Utf8PathBuf, /// Directory the declared `dirs`/`files` are anchored at (the canister dir). pub base_dir: Utf8PathBuf, - /// Manifest-relative directories to preopen read-only. - pub dirs: Vec, - /// Manifest-relative files to read and pass inline. - pub files: Vec, + /// Manifest-relative directories to preopen read-only, each tagged with the + /// map key it was declared under (if any). + pub dirs: Vec, + /// Manifest-relative files to read and pass inline, each tagged with the map + /// key it was declared under (if any). + pub files: Vec, /// Key-value fields to pass inline. Passed to v0.2.0 plugins; ignored by /// v0.1.0 plugins, whose interface has no `fields`. pub fields: BTreeMap, @@ -475,7 +491,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin // Preopen each declared directory read-only. The guest sees it at the // same relative path it used in the manifest. let mut wasi_builder = wasmtime_wasi::WasiCtxBuilder::new(); - for dir in &dirs { + for KeyedPath { path: dir, .. } in &dirs { ensure!(!crate::path::escapes_base(dir), UnsafeDirSnafu { dir }); // Reject symlinks in the declared path: neither the final entry nor any // intermediate component may be a symlink, so the preopen cannot escape @@ -498,10 +514,11 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin // Read each declared file on the host and pass its content inline. The same // path-safety checks as `dirs` apply: reject escaping or symlinked paths so // a read cannot leave `base_dir`. - // Held as plain (name, content) pairs so they can be converted to whichever - // interface version's `file-input` record the plugin turns out to use. - let mut file_contents: Vec<(String, String)> = Vec::with_capacity(files.len()); - for name in &files { + // Held as plain (key, name, content) triples so they can be converted to + // whichever interface version's `file-input` record the plugin turns out to + // use (v0.1.0 has no `key`, so it is dropped there). + let mut file_contents: Vec<(Option, String, String)> = Vec::with_capacity(files.len()); + for KeyedPath { key, path: name } in &files { ensure!(!crate::path::escapes_base(name), UnsafeFileSnafu { name }); if let Some(link) = crate::path::first_symlink_component(&base_dir, name) { return SymlinkFileSnafu { name, link }.fail(); @@ -509,7 +526,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let path = base_dir.join(name); let content = std::fs::read_to_string(path.as_std_path()).context(ReadFileSnafu { path })?; - file_contents.push((name.clone(), content)); + file_contents.push((key.clone(), name.clone(), content)); } let persistent_stderr: Arc>> = Arc::default(); @@ -572,10 +589,13 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let input = v2::SyncExecInput { canister_id: canister_id_text, environment, - dirs, + dirs: dirs + .into_iter() + .map(|KeyedPath { key, path }| v2::DirInput { key, path }) + .collect(), files: file_contents .into_iter() - .map(|(name, content)| v2::FileInput { name, content }) + .map(|(key, name, content)| v2::FileInput { key, name, content }) .collect(), fields: fields .into_iter() @@ -611,10 +631,14 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let input = v1::SyncExecInput { canister_id: canister_id_text, environment, - dirs, + // The v0.1.0 interface has no per-entry key; pass just the paths. + dirs: dirs + .into_iter() + .map(|KeyedPath { path, .. }| path) + .collect(), files: file_contents .into_iter() - .map(|(name, content)| v1::FileInput { name, content }) + .map(|(_key, name, content)| v1::FileInput { name, content }) .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, @@ -807,6 +831,17 @@ mod tests { Principal::anonymous() } + /// Plain (unkeyed) [`KeyedPath`]s, as a plain-list manifest entry produces. + fn unkeyed(paths: &[&str]) -> Vec { + paths + .iter() + .map(|p| KeyedPath { + key: None, + path: (*p).to_string(), + }) + .collect() + } + /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister /// and identity, no proxy, no declared callable canisters, the default /// compute limit, and the current directory as the base. Tests override @@ -896,7 +931,7 @@ mod tests { return; }; let mut inv = invocation(wasm_path, "test"); - inv.dirs = vec!["nonexistent_dir".to_string()]; + inv.dirs = unkeyed(&["nonexistent_dir"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::PreopenDir { .. }) @@ -917,7 +952,7 @@ mod tests { let mut inv = invocation(wasm_path, "test"); inv.base_dir = base.to_path_buf(); - inv.dirs = vec!["link".to_string()]; + inv.dirs = unkeyed(&["link"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::SymlinkDir { .. }) @@ -930,7 +965,7 @@ mod tests { return; }; let mut inv = invocation(wasm_path, "test"); - inv.files = vec!["nonexistent_file.txt".to_string()]; + inv.files = unkeyed(&["nonexistent_file.txt"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::ReadFile { .. }) @@ -951,7 +986,7 @@ mod tests { let mut inv = invocation(wasm_path, "test"); inv.base_dir = base.to_path_buf(); - inv.files = vec!["link.txt".to_string()]; + inv.files = unkeyed(&["link.txt"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::SymlinkFile { .. }) @@ -1046,6 +1081,42 @@ mod tests { )); } + #[tokio::test(flavor = "multi_thread")] + async fn dir_and_file_keys_reach_the_plugin() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + // A real dir and file must exist: the host preopens the dir and reads + // the file before calling exec(). + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let base = tmp.path(); + std::fs::create_dir_all(base.join("seeds")).expect("create dir"); + std::fs::write(base.join("cfg.txt"), b"data").expect("write file"); + + let (tx, mut rx) = tokio::sync::mpsc::channel::(16); + let result = tokio::task::block_in_place(|| { + let mut inv = invocation(wasm_path, "keys"); + inv.base_dir = base.to_path_buf(); + inv.dirs = vec![KeyedPath { + key: Some("assets".to_string()), + path: "seeds".to_string(), + }]; + inv.files = vec![KeyedPath { + key: None, + path: "cfg.txt".to_string(), + }]; + inv.stdio = Some(tx); + run_plugin(inv) + }); + let lines = result.expect("plugin should succeed"); + assert_eq!( + lines, + vec!["dir assets=seeds".to_string(), "file -=cfg.txt".to_string()], + ); + // The same lines are forwarded live to the rolling-view channel. + assert!(rx.try_recv().is_ok()); + } + #[test] fn legacy_v1_plugin_is_detected_and_driven() { // A plugin built against the v0.1.0 interface must still load: the host diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index a19c609f2..d5a4481cc 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -5,8 +5,23 @@ interface types { /// Whether a canister call is an update or a query. enum call-type { update, query } + /// A directory the host preopened on behalf of the plugin. + record dir-input { + /// The map key this directory was declared under in the manifest, or + /// `none` when `dirs` was written as a plain list. Several entries share + /// one key when a key maps to a list of directories. + key: option, + /// Path of the directory as declared in the manifest (relative to the + /// canister directory). The host preopens it at this same path. + path: string, + } + /// A file the host read on behalf of the plugin. record file-input { + /// The map key this file was declared under in the manifest, or `none` + /// when `files` was written as a plain list. Several entries share one + /// key when a key maps to a list of files. + key: option, /// Path of the file as declared in the manifest (relative to /// the canister directory). name: string, @@ -69,9 +84,13 @@ interface types { /// Directories declared in the manifest step's `dirs` setting. /// The host preopens each entry via WASI; the plugin can traverse /// them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). - dirs: list, + /// Each entry carries the map key it was declared under, if any (see + /// `dir-input`). + dirs: list, /// Files declared in the manifest step's `files` setting, read by /// the host and passed inline. The plugin decides how to use them. + /// Each entry carries the map key it was declared under, if any (see + /// `file-input`). files: list, /// Key-value fields declared in the manifest step's `fields` setting, /// passed inline. The plugin decides how to use them. @@ -118,7 +137,7 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input, field-input}; + use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, dir-input, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 63fc360fd..2c7db0cb7 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -33,6 +33,17 @@ impl Guest for TestPlugin { eprintln!("{rendered}"); Ok(()) } + // Echo each dir/file entry as `kind key=path`, using "-" for an + // absent key, so the host can assert keys survive the boundary. + "keys" => { + for dir in &input.dirs { + eprintln!("dir {}={}", dir.key.as_deref().unwrap_or("-"), dir.path); + } + for file in &input.files { + eprintln!("file {}={}", file.key.as_deref().unwrap_or("-"), file.name); + } + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index eee54a1b4..60ce522bc 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -4,13 +4,30 @@ use camino::Utf8PathBuf; use candid::Principal; use ic_agent::Agent; use icp_sync_plugin::{ - CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, }; use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{canister::wasm, manifest::adapter::plugin::Adapter, package::PackageCache}; +use crate::{ + canister::wasm, + manifest::adapter::plugin::{Adapter, NamedPaths}, + package::PackageCache, +}; + +/// Convert a manifest [`NamedPaths`] (or its absence) into the runtime's +/// key-tagged path list. A missing setting yields an empty list. +fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { + paths + .into_iter() + .flat_map(NamedPaths::entries) + .map(|entry| KeyedPath { + key: entry.key.clone(), + path: entry.path.clone(), + }) + .collect() +} use super::Params; @@ -97,8 +114,8 @@ pub(super) async fn sync( // subject to the runtime's path-safety checks (no escaping or symlinked // paths). let base_dir = Utf8PathBuf::from(params.path.as_str()); - let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); - let files: Vec = adapter.files.clone().unwrap_or_default(); + let dirs = keyed_paths(adapter.dirs.as_ref()); + let files = keyed_paths(adapter.files.as_ref()); let fields: BTreeMap = adapter.fields.clone().unwrap_or_default(); // 3. Build the canister ID table exposed to the plugin, then resolve the diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index c84ab6e3c..56a0fb8c5 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,9 +1,14 @@ -use std::{collections::BTreeMap, fmt}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt, +}; +use indexmap::IndexMap; use schemars::JsonSchema; use serde::{ - Deserialize, Deserializer, Serialize, + Deserialize, Deserializer, Serialize, Serializer, de::{self, Visitor}, + ser::SerializeMap, }; use super::prebuilt::SourceField; @@ -78,6 +83,156 @@ fn deserialize_fields<'de, D: Deserializer<'de>>( Ok(fields.map(|fields| fields.into_iter().map(|(k, v)| (k, v.0)).collect())) } +/// A single manifest path together with the map key it was declared under. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NamedPath { + /// The map key this path was declared under, or `None` for a plain-list + /// entry. Non-unique: several paths share a key when the key maps to a list. + pub key: Option, + /// The path itself, relative to the canister directory. + pub path: String, +} + +/// A set of manifest paths declared either as a plain list or as a map of +/// name → path(s). Used for a plugin step's `dirs` and `files`. +/// +/// In `canister.yaml` this accepts three shapes: +/// ```yaml +/// # a plain list — entries carry no key +/// files: +/// - config.txt +/// - data.json +/// # a map whose keys each name a single path... +/// files: +/// main: config.txt +/// # ...or a list of paths, which all share that key +/// files: +/// seeds: +/// - a.json +/// - b.json +/// ``` +/// +/// Order is preserved: list entries in written order; map entries in written +/// key order, each key's paths in written order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NamedPaths(Vec); + +/// A list of paths, or a map of name → path (or list of paths). The map form +/// tags each path with its key for the plugin; a key may map to several paths. +/// +/// This type exists only to describe [`NamedPaths`] in the generated JSON schema +/// (see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`] +/// owns the actual (de)serialization. +#[derive(JsonSchema)] +#[serde(untagged)] +#[allow(dead_code)] +enum NamedPathsSchema { + List(Vec), + Map(HashMap), +} + +/// One map value in [`NamedPathsSchema`]: a single path, or a list of paths that +/// share the key. +#[derive(JsonSchema)] +#[serde(untagged)] +#[allow(dead_code)] +enum PathOrListSchema { + One(String), + Many(Vec), +} + +impl NamedPaths { + /// Build from an ordered list of key-tagged paths. + pub fn from_entries(entries: Vec) -> Self { + NamedPaths(entries) + } + + /// The declared paths, in order, each tagged with its map key (if any). + pub fn entries(&self) -> &[NamedPath] { + &self.0 + } + + /// Consume into the ordered list of key-tagged paths. + pub fn into_entries(self) -> Vec { + self.0 + } +} + +impl FromIterator for NamedPaths { + fn from_iter>(iter: I) -> Self { + NamedPaths(iter.into_iter().collect()) + } +} + +impl<'de> Deserialize<'de> for NamedPaths { + fn deserialize>(d: D) -> Result { + /// A map value: a single path or a list of paths sharing the key. + #[derive(Deserialize)] + #[serde(untagged)] + enum PathOrList { + One(String), + Many(Vec), + } + + #[derive(Deserialize)] + #[serde(untagged)] + enum Repr { + List(Vec), + Map(IndexMap), + } + + let entries = match Repr::deserialize(d)? { + Repr::List(paths) => paths + .into_iter() + .map(|path| NamedPath { key: None, path }) + .collect(), + Repr::Map(map) => map + .into_iter() + .flat_map(|(key, value)| { + let paths = match value { + PathOrList::One(path) => vec![path], + PathOrList::Many(paths) => paths, + }; + paths.into_iter().map(move |path| NamedPath { + key: Some(key.clone()), + path, + }) + }) + .collect(), + }; + Ok(NamedPaths(entries)) + } +} + +impl Serialize for NamedPaths { + fn serialize(&self, s: S) -> Result { + // Deserialization yields either all-unkeyed (list form) or all-keyed + // (map form) entries; serialize back to whichever it was. + if self.0.iter().all(|e| e.key.is_none()) { + let paths: Vec<&str> = self.0.iter().map(|e| e.path.as_str()).collect(); + paths.serialize(s) + } else { + // Group paths by key, preserving order. A key with one path + // serializes as a scalar; multiple as a list. + let mut groups: IndexMap<&str, Vec<&str>> = IndexMap::new(); + for e in &self.0 { + groups + .entry(e.key.as_deref().unwrap_or_default()) + .or_default() + .push(&e.path); + } + let mut map = s.serialize_map(Some(groups.len()))?; + for (key, paths) in groups { + match paths.as_slice() { + [one] => map.serialize_entry(key, one)?, + many => map.serialize_entry(key, many)?, + } + } + map.end() + } + } +} + /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -100,6 +255,18 @@ fn deserialize_fields<'de, D: Deserializer<'de>>( /// retries: 3 /// ``` /// +/// `dirs` and `files` may instead be written as a map, tagging each entry with a +/// `key` surfaced to the plugin; a key may map to a single path or a list: +/// ```yaml +/// - type: plugin +/// path: ./plugins/populate-data.wasm +/// dirs: +/// seed: assets/seed-data # keyed single path +/// migrations: # keyed list — entries share the key +/// - migrations/2025 +/// - migrations/2026 +/// ``` +/// /// Example (remote URL — `sha256` is required): /// ```yaml /// - type: plugin @@ -117,12 +284,18 @@ pub struct Adapter { /// Directories (relative to canister directory) the plugin may read from. /// Each entry must be a directory; it is preopened via WASI so the plugin - /// can traverse it using standard filesystem APIs. - pub dirs: Option>, + /// can traverse it using standard filesystem APIs. Written as a plain list + /// of paths, or as a map of name → path (or list of paths); the name is + /// surfaced to the plugin as each entry's `key`. + #[schemars(with = "Option")] + pub dirs: Option, /// Files (relative to canister directory) the host reads and passes to - /// the plugin as part of `sync-exec-input.files`. - pub files: Option>, + /// the plugin as part of `sync-exec-input.files`. Written as a plain list + /// of paths, or as a map of name → path (or list of paths); the name is + /// surfaced to the plugin as each entry's `key`. + #[schemars(with = "Option")] + pub files: Option, /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. /// A plugin receives every value as a string; a number or boolean written @@ -146,8 +319,8 @@ impl<'de> Deserialize<'de> for Adapter { #[serde(flatten)] source: SourceField, sha256: Option, - dirs: Option>, - files: Option>, + dirs: Option, + files: Option, #[serde(default, deserialize_with = "deserialize_fields")] fields: Option>, canisters: Option>, @@ -175,6 +348,27 @@ mod tests { use indoc::indoc; use super::*; + + /// [`NamedPaths`] with no keys, as a plain-list manifest entry produces. + fn unkeyed(paths: [&str; N]) -> NamedPaths { + NamedPaths::from_entries( + paths + .into_iter() + .map(|path| NamedPath { + key: None, + path: path.to_string(), + }) + .collect(), + ) + } + + /// A single key-tagged [`NamedPath`]. + fn keyed(key: &str, path: &str) -> NamedPath { + NamedPath { + key: Some(key.to_string()), + path: path.to_string(), + } + } use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; #[test] @@ -219,8 +413,8 @@ mod tests { path: "plugins/my-sync.wasm".into(), }), sha256: Some("abc123".to_string()), - dirs: Some(vec!["assets/seed-data".to_string(), "config".to_string()]), - files: Some(vec!["config.txt".to_string()]), + dirs: Some(unkeyed(["assets/seed-data", "config"])), + files: Some(unkeyed(["config.txt"])), fields: None, canisters: None, }, @@ -237,6 +431,70 @@ mod tests { serde_yaml::from_value(value) } + /// The list form leaves every entry keyless. + #[test] + fn dirs_and_files_as_plain_lists_have_no_keys() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + dirs: + - assets + files: + - a.txt + - b.txt + "#, + ) + .expect("failed to deserialize Adapter with list dirs/files"); + assert_eq!(adapter.dirs, Some(unkeyed(["assets"]))); + assert_eq!(adapter.files, Some(unkeyed(["a.txt", "b.txt"]))); + } + + /// The map form tags each entry with its key. A key mapping to a list yields + /// several entries sharing that (non-unique) key, in written order. + #[test] + fn dirs_and_files_as_maps_carry_keys() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + dirs: + seed: assets/seed-data + extra: + - one + - two + files: + main: config.txt + "#, + ) + .expect("failed to deserialize Adapter with map dirs/files"); + assert_eq!( + adapter.dirs.map(NamedPaths::into_entries), + Some(vec![ + keyed("seed", "assets/seed-data"), + keyed("extra", "one"), + keyed("extra", "two"), + ]), + ); + assert_eq!( + adapter.files.map(NamedPaths::into_entries), + Some(vec![keyed("main", "config.txt")]), + ); + } + + /// The list and map forms round-trip through serialization back to their + /// natural YAML shape. + #[test] + fn named_paths_round_trip() { + for yaml in [ + "- a.txt\n- b.txt\n", + "single: one.txt\nmany:\n- x.txt\n- y.txt\n", + ] { + let parsed: NamedPaths = + serde_yaml::from_str(yaml).expect("failed to parse NamedPaths"); + let reserialized = serde_yaml::to_string(&parsed).expect("failed to serialize"); + assert_eq!(reserialized, yaml, "round-trip changed the YAML shape"); + } + } + #[test] fn fields_parse_as_a_string_map() { let adapter = adapter_via_value( diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 28a576f9b..23c9ca853 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -791,7 +791,14 @@ mod tests { path: "./plugins/my-sync.wasm".into(), }), sha256: None, - dirs: Some(vec!["assets/seed-data/".to_string()]), + dirs: Some( + crate::manifest::adapter::plugin::NamedPaths::from_entries( + vec![crate::manifest::adapter::plugin::NamedPath { + key: None, + path: "assets/seed-data/".to_string(), + }], + ) + ), files: None, fields: None, canisters: None, diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index a4b027dbb..853cab13e 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -67,8 +67,8 @@ The authoritative interface, including all record fields, lives in [`sync-plugin |-------|-------------| | `canister-id` | Textual principal of the canister being synced | | `environment` | Name of the environment being synced (e.g. `local`, `production`) | -| `dirs` | The directories you declared in `dirs:`; the host preopened each one read-only | -| `files` | The files you declared in `files:`, each as a `(name, content)` pair read by the host | +| `dirs` | The directories you declared in `dirs:`; the host preopened each one read-only. Each entry carries its `key` (see below) and `path` | +| `files` | The files you declared in `files:`, each with its `key`, `name` (path), and `content` read by the host | | `fields` | The key-value fields you declared in `fields:`, each as a `(name, value)` pair; values are strings | | `identity-principal` | Textual principal of the signing identity used for canister calls | | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | @@ -76,6 +76,8 @@ The authoritative interface, including all record fields, lives in [`sync-plugin Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister defined in a subproject. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. +`dirs` and `files` each carry a `key`: the map key the entry was declared under in the manifest, or absent when `dirs:`/`files:` was written as a plain list. A key that maps to a list of paths produces several entries sharing that key, so the key is not unique. Use it to group or label declared paths — e.g. distinguish `seed:` directories from `migrations:` directories — without hardcoding paths in the plugin. + ### Calling a canister — `canister-call` The plugin calls methods through the `canister-call` import. It picks a `target`, supplies the method name, **Candid-encoded argument bytes** (the host forwards them unchanged), and a few routing options: diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 9b6f28282..8e466b585 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -92,19 +92,19 @@ A few things to note: A plugin can't see the filesystem freely — only what you grant it in the manifest's `dirs:` and `files:`. -Directories in `dirs:` are preopened read-only at the same relative path. Traverse them with standard `std::fs`: +Directories in `dirs:` are preopened read-only at the same relative path. Each entry gives you its `path` plus a `key` (the map key it was declared under, or `None` for a plain-list entry). Traverse them with standard `std::fs`: ```rust for dir in &input.dirs { - for entry in std::fs::read_dir(dir).map_err(|e| e.to_string())? { + for entry in std::fs::read_dir(&dir.path).map_err(|e| e.to_string())? { let path = entry.map_err(|e| e.to_string())?.path(); let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; - // ... encode and send to the canister ... + // ... encode and send to the canister; dir.key groups related dirs ... } } ``` -Files in `files:` are read by the host up front and passed inline — read them from the input struct, not from disk: +Files in `files:` are read by the host up front and passed inline — read them from the input struct, not from disk. Each entry carries its `key`, `name` (the path), and `content`: ```rust for file in &input.files { @@ -112,6 +112,8 @@ for file in &input.files { } ``` +Declaring `dirs:`/`files:` as a map instead of a list tags each entry with a `key`, so a plugin can group or label paths (for example, tell `seed:` directories from `migrations:`) without hardcoding paths. A key that maps to a list of paths yields several entries sharing that key. + Writes, paths outside a preopen, and `..` traversal are all rejected by the sandbox. See [The Sandbox](../concepts/sync-plugins.md#the-sandbox) for the full capability list and resource limits. ## Read Declared Fields diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0518287d9..d91e58c77 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -169,11 +169,25 @@ sync: | `path` | string | One of `path` or `url` | Local path to the wasm, relative to the canister directory | | `url` | string | One of `path` or `url` | URL to download the wasm from | | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | -| `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | -| `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | +| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | +| `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | | `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | | `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | +`dirs:` and `files:` each accept either a plain list of paths or a map. As a map, each key names a single path or a list of paths, and the key is surfaced to the plugin as that entry's `key` (a key mapping to a list produces several entries sharing it). A plain-list entry has no key. For example: + +```yaml + - type: plugin + path: ./plugins/populate-data.wasm + dirs: + seed: assets/seed-data # one path under a key + migrations: # several paths sharing a key + - migrations/2025 + - migrations/2026 + files: + - config.txt # a plain list is still fine +``` + Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. A plugin receives every `fields:` value as a string. Numbers and booleans need no quoting — `port: 8080` arrives as `"8080"` — but a value may not be a list, a mapping, or empty. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 4632f9cb0..a5c7e262d 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -80,14 +80,15 @@ ] }, "dirs": { - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "fields": { "additionalProperties": { @@ -104,14 +105,15 @@ ] }, "files": { - "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "sha256": { "description": "Optional sha256 checksum of the wasm file.\nOptional for `path`; required for `url`.", @@ -339,6 +341,37 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, + "NamedPathsSchema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "additionalProperties": { + "$ref": "#/$defs/PathOrListSchema" + }, + "type": "object" + } + ], + "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + }, + "PathOrListSchema": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + }, "Recipe": { "properties": { "configuration": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 3121bf499..4121c9bb4 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -80,14 +80,15 @@ ] }, "dirs": { - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "fields": { "additionalProperties": { @@ -104,14 +105,15 @@ ] }, "files": { - "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "sha256": { "description": "Optional sha256 checksum of the wasm file.\nOptional for `path`; required for `url`.", @@ -795,6 +797,23 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, + "NamedPathsSchema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "additionalProperties": { + "$ref": "#/$defs/PathOrListSchema" + }, + "type": "object" + } + ], + "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + }, "NetworkManifest": { "description": "A network definition for the project", "oneOf": [ @@ -835,6 +854,20 @@ ], "type": "object" }, + "PathOrListSchema": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + }, "Recipe": { "properties": { "configuration": { diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index f6d7534bd..c3fdee356 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -38,7 +38,7 @@ impl Guest for Plugin { // uploader principal, which is the current identity — not the proxy. let mut registered = 0u32; for dir in &input.dirs { - registered += register_dir(Path::new(dir))?; + registered += register_dir(Path::new(&dir.path))?; } // Persisted after the step completes; use stderr. From 07f5c51e3680368b074c578dd7f0cb8cebc3e3c1 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 07:21:32 -0700 Subject: [PATCH 17/51] fixes --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/icp-cli/src/operations/bundle.rs | 73 +++---- crates/icp-cli/tests/bundle_tests.rs | 102 +++++++++ crates/icp-sync-plugin/DESIGN.md | 16 +- crates/icp-sync-plugin/src/runtime.rs | 26 ++- crates/icp/src/canister/sync/plugin.rs | 8 +- crates/icp/src/manifest/adapter/plugin.rs | 242 +++++++++------------- crates/icp/src/manifest/canister.rs | 41 ++-- docs/schemas/canister-yaml-schema.json | 18 +- docs/schemas/icp-yaml-schema.json | 18 +- 11 files changed, 304 insertions(+), 243 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 330024485..9ca77dd4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6357,6 +6357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", + "indexmap", "ref-cast", "schemars_derive", "serde", diff --git a/Cargo.toml b/Cargo.toml index 30710a835..60dc41e1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,7 @@ pkcs8 = { version = "0.10.2", features = ["encryption", "std"] } rand = "0.10.1" regex = "1.12.2" reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "json", "stream"] } -schemars = { version = "1.0.4", features = ["derive", "url2"] } +schemars = { version = "1.0.4", features = ["derive", "indexmap2", "url2"] } scrypt = "0.11.0" sec1 = { version = "0.7.3", features = ["pkcs8"] } send_ctrlc = "0.6" diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 94eef4268..5bd881d9e 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -769,49 +769,36 @@ async fn prepare_plugin_step( // The declared paths are rewritten to their archive locations; each entry's // map key is carried through unchanged. let bundle_dirs = adapter.dirs.as_ref().map(|dirs| { - dirs.entries() - .iter() - .map(|d| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/dirs/{}", - normalize_archive_dir(&d.path) - ); - out.plugin_dirs.push(DirEntry { - src_path: canister_path.join(&d.path), - archive_prefix: archive_join(prefix, &manifest_path), - }); - plugin::NamedPath { - key: d.key.clone(), - path: manifest_path, - } - }) - .collect::() + dirs.map_paths(|dir| { + let manifest_path = format!( + "plugins/{path_name}/{idx}/dirs/{}", + normalize_archive_dir(dir) + ); + out.plugin_dirs.push(DirEntry { + src_path: canister_path.join(dir), + archive_prefix: archive_join(prefix, &manifest_path), + }); + manifest_path + }) }); let bundle_files = adapter.files.as_ref().map(|files| { - files - .entries() - .iter() - .map(|f| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/files/{}", - normalize_archive_dir(&f.path) - ); - out.plugin_files.push(PluginFile { - src_path: canister_path.join(&f.path), - archive_path: archive_join(prefix, &manifest_path), - canister_name: canister.name.clone(), - orig_file: f.path.clone(), - }); - plugin::NamedPath { - key: f.key.clone(), - path: manifest_path, - } - }) - .collect::() + files.map_paths(|file| { + let manifest_path = format!( + "plugins/{path_name}/{idx}/files/{}", + normalize_archive_dir(file) + ); + out.plugin_files.push(PluginFile { + src_path: canister_path.join(file), + archive_path: archive_join(prefix, &manifest_path), + canister_name: canister.name.clone(), + orig_file: file.to_string(), + }); + manifest_path + }) }); - Ok(SyncStep::Plugin(plugin::Adapter { + Ok(SyncStep::Plugin(Box::new(plugin::Adapter { source: SourceField::Local(LocalSource { path: plugin_wasm_path.as_str().into(), }), @@ -820,7 +807,7 @@ async fn prepare_plugin_step( files: bundle_files, canisters: localize_call_targets(adapter.canisters.as_deref(), local_names), fields: adapter.fields.clone(), - })) + }))) } async fn inline_networks( @@ -1311,8 +1298,8 @@ fn validate_source_paths( SyncStep::Script(_) => {} SyncStep::Plugin(adapter) => { if let Some(dirs) = &adapter.dirs { - for d in dirs.entries() { - let src = canister_path.join(&d.path); + for dir in dirs.entries() { + let src = canister_path.join(dir.path); let resolved = resolve_within_project( &src, project_dir, @@ -1323,8 +1310,8 @@ fn validate_source_paths( } } if let Some(files) = &adapter.files { - for f in files.entries() { - let src = canister_path.join(&f.path); + for file in files.entries() { + let src = canister_path.join(file.path); resolve_within_project( &src, project_dir, diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 1500ec340..66feb2c84 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1281,6 +1281,108 @@ fn bundle_preserves_plugin_call_targets() { ); } +/// Map-form `dirs:`/`files:` must survive bundling: the paths are rewritten to their +/// archive locations, but each stays under the key it was declared with, so a plugin sees +/// the same keys whether it runs from the project or from the bundle. +#[test] +fn bundle_preserves_plugin_path_keys() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let plugin_bytes: &[u8] = b"\x00asm\x01\x00\x00\x00plugin"; + write(&project_dir.join("plugin.wasm"), plugin_bytes).expect("failed to write plugin"); + + for (dir, file) in [("seed", "s.txt"), ("m2025", "a.txt"), ("m2026", "b.txt")] { + let path = project_dir.join(dir); + create_dir_all(&path).expect("failed to create dir"); + write_string(&path.join(file), "data").expect("failed to write file"); + } + write_string(&project_dir.join("config.toml"), "key=value").expect("failed to write config"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: plugin.wasm + dirs: + seed: seed + migrations: + - m2025 + - m2026 + files: + main: config.toml + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut archived: Vec = Vec::new(); + let mut manifest_yaml = String::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path == "icp.yaml" { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + archived.push(path); + } + + for expected in [ + "plugins/my-canister/0/dirs/seed/s.txt", + "plugins/my-canister/0/dirs/m2025/a.txt", + "plugins/my-canister/0/dirs/m2026/b.txt", + "plugins/my-canister/0/files/config.toml", + ] { + assert!( + archived.iter().any(|path| path == expected), + "{expected} not found in bundle; archive holds {archived:?}" + ); + } + + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let step = &parsed["canisters"][0]["sync"]["steps"][0]; + assert_eq!( + step["dirs"]["seed"].as_str(), + Some("plugins/my-canister/0/dirs/seed") + ); + assert_eq!( + step["dirs"]["migrations"][0].as_str(), + Some("plugins/my-canister/0/dirs/m2025") + ); + assert_eq!( + step["dirs"]["migrations"][1].as_str(), + Some("plugins/my-canister/0/dirs/m2026") + ); + assert_eq!( + step["files"]["main"].as_str(), + Some("plugins/my-canister/0/files/config.toml") + ); +} + /// An `icp_appmanifest.yaml` next to the project manifest must be included in the bundle, with its /// top-level `images` paths relocated under a top-level `images/` folder and the /// referenced image files copied alongside. Unrelated metadata is preserved. diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 326a9b6a5..27617fc14 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -197,13 +197,15 @@ pub struct Adapter { } ``` -`NamedPaths` deserializes `dirs:`/`files:` from either a plain list of paths or a -map of name → path (or list of paths), flattening to an ordered list of -`(key, path)` entries: `key` is `None` for a list entry and `Some(name)` for a -map entry, and is *non-unique* — a map key that resolves to a list of paths -produces one entry per path, all sharing the key. The CLI passes these to the -runtime as `KeyedPath`s (this crate stays free of manifest types), which surface -in `sync-exec-input.dirs`/`files` as each entry's `key`. +`NamedPaths` is an untagged `List(Vec) | Map(IndexMap)` +— the two shapes `dirs:`/`files:` may be written in — keeping the written form +exact, so bundling can rewrite the paths (`map_paths`) and serialize the step +back out unchanged in shape. `entries()` flattens either form to ordered +`(key, path)` pairs: `key` is `None` for a list entry and `Some(name)` for a map +entry, and is *non-unique* — a map key holding a list of paths yields one entry +per path, all sharing the key. The CLI passes those to the runtime as +`KeyedPath`s (this crate stays free of manifest types), which surface in +`sync-exec-input.dirs`/`files` as each entry's `key`. Each `canisters:` entry is a canister name resolved against the project's ID table for the environment being synced. `Deserialize` is hand-written to reject a diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 719a920a9..ae3d76f91 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -70,6 +70,16 @@ pub struct KeyedPath { pub path: String, } +/// A declared file the host read: the key and path it was declared under, plus +/// its content. Held version-agnostically so it can be converted to whichever +/// interface version's `file-input` record the plugin turns out to use — the +/// v0.1.0 record has no `key`, so it is dropped there. +struct FileContent { + key: Option, + name: String, + content: String, +} + /// The canisters a sync plugin is permitted to call, beyond the canister being /// synced (which is always reachable via [`CallTarget::Host`]). /// @@ -514,10 +524,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin // Read each declared file on the host and pass its content inline. The same // path-safety checks as `dirs` apply: reject escaping or symlinked paths so // a read cannot leave `base_dir`. - // Held as plain (key, name, content) triples so they can be converted to - // whichever interface version's `file-input` record the plugin turns out to - // use (v0.1.0 has no `key`, so it is dropped there). - let mut file_contents: Vec<(Option, String, String)> = Vec::with_capacity(files.len()); + let mut file_contents: Vec = Vec::with_capacity(files.len()); for KeyedPath { key, path: name } in &files { ensure!(!crate::path::escapes_base(name), UnsafeFileSnafu { name }); if let Some(link) = crate::path::first_symlink_component(&base_dir, name) { @@ -526,7 +533,11 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let path = base_dir.join(name); let content = std::fs::read_to_string(path.as_std_path()).context(ReadFileSnafu { path })?; - file_contents.push((key.clone(), name.clone(), content)); + file_contents.push(FileContent { + key: key.clone(), + name: name.clone(), + content, + }); } let persistent_stderr: Arc>> = Arc::default(); @@ -595,7 +606,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin .collect(), files: file_contents .into_iter() - .map(|(key, name, content)| v2::FileInput { key, name, content }) + .map(|FileContent { key, name, content }| v2::FileInput { key, name, content }) .collect(), fields: fields .into_iter() @@ -636,9 +647,10 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin .into_iter() .map(|KeyedPath { path, .. }| path) .collect(), + // The v0.1.0 `file-input` record has no `key`; drop it. files: file_contents .into_iter() - .map(|(_key, name, content)| v1::FileInput { name, content }) + .map(|FileContent { name, content, .. }| v1::FileInput { name, content }) .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 60ce522bc..fbde4dde4 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -16,6 +16,8 @@ use crate::{ package::PackageCache, }; +use super::Params; + /// Convert a manifest [`NamedPaths`] (or its absence) into the runtime's /// key-tagged path list. A missing setting yields an empty list. fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { @@ -23,14 +25,12 @@ fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { .into_iter() .flat_map(NamedPaths::entries) .map(|entry| KeyedPath { - key: entry.key.clone(), - path: entry.path.clone(), + key: entry.key.map(str::to_string), + path: entry.path.to_string(), }) .collect() } -use super::Params; - #[derive(Debug, Snafu)] pub enum PluginError { #[snafu(transparent)] diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 56a0fb8c5..d3814f7d3 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,14 +1,11 @@ -use std::{ - collections::{BTreeMap, HashMap}, - fmt, -}; +use std::{collections::BTreeMap, fmt}; use indexmap::IndexMap; +use itertools::Either; use schemars::JsonSchema; use serde::{ - Deserialize, Deserializer, Serialize, Serializer, + Deserialize, Deserializer, Serialize, de::{self, Visitor}, - ser::SerializeMap, }; use super::prebuilt::SourceField; @@ -83,20 +80,9 @@ fn deserialize_fields<'de, D: Deserializer<'de>>( Ok(fields.map(|fields| fields.into_iter().map(|(k, v)| (k, v.0)).collect())) } -/// A single manifest path together with the map key it was declared under. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct NamedPath { - /// The map key this path was declared under, or `None` for a plain-list - /// entry. Non-unique: several paths share a key when the key maps to a list. - pub key: Option, - /// The path itself, relative to the canister directory. - pub path: String, -} - -/// A set of manifest paths declared either as a plain list or as a map of -/// name → path(s). Used for a plugin step's `dirs` and `files`. +/// The paths declared for a plugin step's `dirs` or `files`: either a plain list +/// of paths, or a map of name → path(s) whose keys are surfaced to the plugin. /// -/// In `canister.yaml` this accepts three shapes: /// ```yaml /// # a plain list — entries carry no key /// files: @@ -105,130 +91,90 @@ pub struct NamedPath { /// # a map whose keys each name a single path... /// files: /// main: config.txt -/// # ...or a list of paths, which all share that key +/// # ...or a list of paths, which then all share that key /// files: /// seeds: /// - a.json /// - b.json /// ``` /// -/// Order is preserved: list entries in written order; map entries in written -/// key order, each key's paths in written order. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct NamedPaths(Vec); - -/// A list of paths, or a map of name → path (or list of paths). The map form -/// tags each path with its key for the plugin; a key may map to several paths. -/// -/// This type exists only to describe [`NamedPaths`] in the generated JSON schema -/// (see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`] -/// owns the actual (de)serialization. -#[derive(JsonSchema)] +/// Order is preserved in both forms: list entries in written order; map entries +/// in written key order, each key's paths in written order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] -#[allow(dead_code)] -enum NamedPathsSchema { +pub enum NamedPaths { + /// A plain list of paths, carrying no keys. List(Vec), - Map(HashMap), + /// A map of name → path(s), tagging each path with the key it sits under. + Map(IndexMap), } -/// One map value in [`NamedPathsSchema`]: a single path, or a list of paths that -/// share the key. -#[derive(JsonSchema)] +/// One value of a [`NamedPaths::Map`]: a single path, or a list of paths that +/// all share the key. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] -#[allow(dead_code)] -enum PathOrListSchema { +pub enum PathOrList { + /// A single path under the key. One(String), + /// Several paths, all sharing the key. Many(Vec), } -impl NamedPaths { - /// Build from an ordered list of key-tagged paths. - pub fn from_entries(entries: Vec) -> Self { - NamedPaths(entries) - } - - /// The declared paths, in order, each tagged with its map key (if any). - pub fn entries(&self) -> &[NamedPath] { - &self.0 - } - - /// Consume into the ordered list of key-tagged paths. - pub fn into_entries(self) -> Vec { - self.0 - } -} - -impl FromIterator for NamedPaths { - fn from_iter>(iter: I) -> Self { - NamedPaths(iter.into_iter().collect()) - } +/// A declared path together with the map key it sits under, as yielded by +/// [`NamedPaths::entries`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NamedPath<'a> { + /// The map key this path sits under, or `None` for a plain-list entry. + /// Non-unique: the paths of a key that maps to a list all share it. + pub key: Option<&'a str>, + /// The path itself, relative to the canister directory. + pub path: &'a str, } -impl<'de> Deserialize<'de> for NamedPaths { - fn deserialize>(d: D) -> Result { - /// A map value: a single path or a list of paths sharing the key. - #[derive(Deserialize)] - #[serde(untagged)] - enum PathOrList { - One(String), - Many(Vec), - } - - #[derive(Deserialize)] - #[serde(untagged)] - enum Repr { - List(Vec), - Map(IndexMap), +impl NamedPaths { + /// The declared paths in written order, each tagged with its key (if any). + pub fn entries(&self) -> impl Iterator> { + match self { + Self::List(paths) => Either::Left(paths.iter().map(|path| NamedPath { + key: None, + path: path.as_str(), + })), + Self::Map(map) => Either::Right(map.iter().flat_map(|(key, value)| { + value.paths().iter().map(move |path| NamedPath { + key: Some(key.as_str()), + path: path.as_str(), + }) + })), } + } - let entries = match Repr::deserialize(d)? { - Repr::List(paths) => paths - .into_iter() - .map(|path| NamedPath { key: None, path }) - .collect(), - Repr::Map(map) => map - .into_iter() - .flat_map(|(key, value)| { - let paths = match value { - PathOrList::One(path) => vec![path], - PathOrList::Many(paths) => paths, - }; - paths.into_iter().map(move |path| NamedPath { - key: Some(key.clone()), - path, + /// Rewrite every path, leaving the keys and the written shape intact. + pub fn map_paths(&self, mut f: impl FnMut(&str) -> String) -> Self { + match self { + Self::List(paths) => Self::List(paths.iter().map(|path| f(path)).collect()), + Self::Map(map) => Self::Map( + map.iter() + .map(|(key, value)| { + let value = match value { + PathOrList::One(path) => PathOrList::One(f(path)), + PathOrList::Many(paths) => { + PathOrList::Many(paths.iter().map(|path| f(path)).collect()) + } + }; + (key.clone(), value) }) - }) - .collect(), - }; - Ok(NamedPaths(entries)) + .collect(), + ), + } } } -impl Serialize for NamedPaths { - fn serialize(&self, s: S) -> Result { - // Deserialization yields either all-unkeyed (list form) or all-keyed - // (map form) entries; serialize back to whichever it was. - if self.0.iter().all(|e| e.key.is_none()) { - let paths: Vec<&str> = self.0.iter().map(|e| e.path.as_str()).collect(); - paths.serialize(s) - } else { - // Group paths by key, preserving order. A key with one path - // serializes as a scalar; multiple as a list. - let mut groups: IndexMap<&str, Vec<&str>> = IndexMap::new(); - for e in &self.0 { - groups - .entry(e.key.as_deref().unwrap_or_default()) - .or_default() - .push(&e.path); - } - let mut map = s.serialize_map(Some(groups.len()))?; - for (key, paths) in groups { - match paths.as_slice() { - [one] => map.serialize_entry(key, one)?, - many => map.serialize_entry(key, many)?, - } - } - map.end() +impl PathOrList { + /// The paths sitting under this key. + fn paths(&self) -> &[String] { + match self { + Self::One(path) => std::slice::from_ref(path), + Self::Many(paths) => paths, } } } @@ -287,14 +233,12 @@ pub struct Adapter { /// can traverse it using standard filesystem APIs. Written as a plain list /// of paths, or as a map of name → path (or list of paths); the name is /// surfaced to the plugin as each entry's `key`. - #[schemars(with = "Option")] pub dirs: Option, /// Files (relative to canister directory) the host reads and passes to /// the plugin as part of `sync-exec-input.files`. Written as a plain list /// of paths, or as a map of name → path (or list of paths); the name is /// surfaced to the plugin as each entry's `key`. - #[schemars(with = "Option")] pub files: Option, /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. @@ -349,27 +293,25 @@ mod tests { use super::*; - /// [`NamedPaths`] with no keys, as a plain-list manifest entry produces. - fn unkeyed(paths: [&str; N]) -> NamedPaths { - NamedPaths::from_entries( - paths - .into_iter() - .map(|path| NamedPath { - key: None, - path: path.to_string(), - }) - .collect(), - ) + use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; + + /// The plain-list form of `dirs:`/`files:`. + fn list(paths: [&str; N]) -> NamedPaths { + NamedPaths::List(paths.into_iter().map(str::to_string).collect()) } - /// A single key-tagged [`NamedPath`]. - fn keyed(key: &str, path: &str) -> NamedPath { + /// A key-tagged entry, as [`NamedPaths::entries`] yields for the map form. + fn keyed<'a>(key: &'a str, path: &'a str) -> NamedPath<'a> { NamedPath { - key: Some(key.to_string()), - path: path.to_string(), + key: Some(key), + path, } } - use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; + + /// The flattened entries of an optional `dirs:`/`files:` setting. + fn entries(paths: &Option) -> Option>> { + paths.as_ref().map(|paths| paths.entries().collect()) + } #[test] fn local_path() { @@ -413,8 +355,8 @@ mod tests { path: "plugins/my-sync.wasm".into(), }), sha256: Some("abc123".to_string()), - dirs: Some(unkeyed(["assets/seed-data", "config"])), - files: Some(unkeyed(["config.txt"])), + dirs: Some(list(["assets/seed-data", "config"])), + files: Some(list(["config.txt"])), fields: None, canisters: None, }, @@ -445,8 +387,8 @@ mod tests { "#, ) .expect("failed to deserialize Adapter with list dirs/files"); - assert_eq!(adapter.dirs, Some(unkeyed(["assets"]))); - assert_eq!(adapter.files, Some(unkeyed(["a.txt", "b.txt"]))); + assert_eq!(adapter.dirs, Some(list(["assets"]))); + assert_eq!(adapter.files, Some(list(["a.txt", "b.txt"]))); } /// The map form tags each entry with its key. A key mapping to a list yields @@ -467,7 +409,7 @@ mod tests { ) .expect("failed to deserialize Adapter with map dirs/files"); assert_eq!( - adapter.dirs.map(NamedPaths::into_entries), + entries(&adapter.dirs), Some(vec![ keyed("seed", "assets/seed-data"), keyed("extra", "one"), @@ -475,11 +417,23 @@ mod tests { ]), ); assert_eq!( - adapter.files.map(NamedPaths::into_entries), + entries(&adapter.files), Some(vec![keyed("main", "config.txt")]), ); } + /// Rewriting paths (as bundling does) leaves keys and the written shape alone. + #[test] + fn map_paths_preserves_keys_and_shape() { + let paths: NamedPaths = serde_yaml::from_str("single: one.txt\nmany:\n- x.txt\n- y.txt\n") + .expect("failed to parse NamedPaths"); + let mapped = paths.map_paths(|path| format!("bundled/{path}")); + assert_eq!( + serde_yaml::to_string(&mapped).expect("failed to serialize"), + "single: bundled/one.txt\nmany:\n- bundled/x.txt\n- bundled/y.txt\n", + ); + } + /// The list and map forms round-trip through serialization back to their /// natural YAML shape. #[test] diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 23c9ca853..8efab30de 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -318,7 +318,8 @@ pub enum SyncStep { /// Represents a sync step executed by a WebAssembly plugin running inside /// a wasmtime WASI sandbox. The plugin can call canister methods on exactly /// the canister being synced and read files from the declared `dirs`. - Plugin(adapter::plugin::Adapter), + // Boxed: a plugin step carries far more configuration than a script one. + Plugin(Box), } impl<'de> Deserialize<'de> for SyncStep { @@ -332,7 +333,7 @@ impl<'de> Deserialize<'de> for SyncStep { #[serde(tag = "type", rename_all = "lowercase")] enum Helper { Script(adapter::script::Adapter), - Plugin(adapter::plugin::Adapter), + Plugin(Box), Assets(serde::de::IgnoredAny), } @@ -383,6 +384,7 @@ mod tests { use crate::{ manifest::{ adapter::{ + plugin, prebuilt::{self, RemoteSource, SourceField}, script, }, @@ -785,25 +787,18 @@ mod tests { })] }, sync: Some(SyncSteps { - steps: vec![SyncStep::Plugin( - crate::manifest::adapter::plugin::Adapter { - source: prebuilt::SourceField::Local(prebuilt::LocalSource { - path: "./plugins/my-sync.wasm".into(), - }), - sha256: None, - dirs: Some( - crate::manifest::adapter::plugin::NamedPaths::from_entries( - vec![crate::manifest::adapter::plugin::NamedPath { - key: None, - path: "assets/seed-data/".to_string(), - }], - ) - ), - files: None, - fields: None, - canisters: None, - } - )] + steps: vec![SyncStep::Plugin(Box::new(plugin::Adapter { + source: prebuilt::SourceField::Local(prebuilt::LocalSource { + path: "./plugins/my-sync.wasm".into(), + }), + sha256: None, + dirs: Some(plugin::NamedPaths::List(vec![ + "assets/seed-data/".to_string() + ])), + files: None, + fields: None, + canisters: None, + }))] }), }, }, @@ -836,7 +831,7 @@ mod tests { })] }, sync: Some(SyncSteps { - steps: vec![SyncStep::Plugin(crate::manifest::adapter::plugin::Adapter { + steps: vec![SyncStep::Plugin(Box::new(plugin::Adapter { source: prebuilt::SourceField::Remote(prebuilt::RemoteSource { url: "https://example.com/plugins/migrate-v2.wasm".to_string(), }), @@ -848,7 +843,7 @@ mod tests { files: None, fields: None, canisters: None, - })] + }))] }), }, }, diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index a5c7e262d..c8aaeae3f 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -82,7 +82,7 @@ "dirs": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -107,7 +107,7 @@ "files": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -341,9 +341,10 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, - "NamedPathsSchema": { + "NamedPaths": { "anyOf": [ { + "description": "A plain list of paths, carrying no keys.", "items": { "type": "string" }, @@ -351,26 +352,29 @@ }, { "additionalProperties": { - "$ref": "#/$defs/PathOrListSchema" + "$ref": "#/$defs/PathOrList" }, + "description": "A map of name → path(s), tagging each path with the key it sits under.", "type": "object" } ], - "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + "description": "The paths declared for a plugin step's `dirs` or `files`: either a plain list\nof paths, or a map of name → path(s) whose keys are surfaced to the plugin.\n\n```yaml\n# a plain list — entries carry no key\nfiles:\n - config.txt\n - data.json\n# a map whose keys each name a single path...\nfiles:\n main: config.txt\n# ...or a list of paths, which then all share that key\nfiles:\n seeds:\n - a.json\n - b.json\n```\n\nOrder is preserved in both forms: list entries in written order; map entries\nin written key order, each key's paths in written order." }, - "PathOrListSchema": { + "PathOrList": { "anyOf": [ { + "description": "A single path under the key.", "type": "string" }, { + "description": "Several paths, all sharing the key.", "items": { "type": "string" }, "type": "array" } ], - "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + "description": "One value of a [`NamedPaths::Map`]: a single path, or a list of paths that\nall share the key." }, "Recipe": { "properties": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 4121c9bb4..b0ca7f2d8 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -82,7 +82,7 @@ "dirs": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -107,7 +107,7 @@ "files": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -797,9 +797,10 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, - "NamedPathsSchema": { + "NamedPaths": { "anyOf": [ { + "description": "A plain list of paths, carrying no keys.", "items": { "type": "string" }, @@ -807,12 +808,13 @@ }, { "additionalProperties": { - "$ref": "#/$defs/PathOrListSchema" + "$ref": "#/$defs/PathOrList" }, + "description": "A map of name → path(s), tagging each path with the key it sits under.", "type": "object" } ], - "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + "description": "The paths declared for a plugin step's `dirs` or `files`: either a plain list\nof paths, or a map of name → path(s) whose keys are surfaced to the plugin.\n\n```yaml\n# a plain list — entries carry no key\nfiles:\n - config.txt\n - data.json\n# a map whose keys each name a single path...\nfiles:\n main: config.txt\n# ...or a list of paths, which then all share that key\nfiles:\n seeds:\n - a.json\n - b.json\n```\n\nOrder is preserved in both forms: list entries in written order; map entries\nin written key order, each key's paths in written order." }, "NetworkManifest": { "description": "A network definition for the project", @@ -854,19 +856,21 @@ ], "type": "object" }, - "PathOrListSchema": { + "PathOrList": { "anyOf": [ { + "description": "A single path under the key.", "type": "string" }, { + "description": "Several paths, all sharing the key.", "items": { "type": "string" }, "type": "array" } ], - "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + "description": "One value of a [`NamedPaths::Map`]: a single path, or a list of paths that\nall share the key." }, "Recipe": { "properties": { From 5ac68237515e6ef3f6996f2047ff6cb777490f7b Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 14:12:10 -0700 Subject: [PATCH 18/51] Handle duplicate paths properly --- Cargo.lock | 1 + crates/icp-cli/Cargo.toml | 1 + crates/icp-cli/src/operations/bundle.rs | 75 ++++++++----- crates/icp-cli/tests/bundle_tests.rs | 102 ++++++++++++++++++ crates/icp-sync-plugin/src/lib.rs | 1 + crates/icp-sync-plugin/src/path.rs | 101 +++++++++++++++++ crates/icp-sync-plugin/src/runtime.rs | 60 ++++++++++- crates/icp-sync-plugin/sync-plugin.wit | 11 +- .../tests/fixtures/test-plugin/src/lib.rs | 19 ++++ crates/icp/src/manifest/adapter/plugin.rs | 10 +- docs/concepts/sync-plugins.md | 3 +- docs/reference/configuration.md | 2 +- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- 14 files changed, 346 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ca77dd4d..b68d3f5b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3728,6 +3728,7 @@ dependencies = [ "ic-utils", "icp", "icp-canister-interfaces", + "icp-sync-plugin", "icrc-ledger-types", "indicatif", "indoc", diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index 3b85f287b..7e326084c 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -44,6 +44,7 @@ ic-management-canister-types.workspace = true ic-utils.workspace = true icp-canister-interfaces.workspace = true icp = { workspace = true, features = ["clap"] } +icp-sync-plugin.workspace = true icrc-ledger-types.workspace = true indicatif.workspace = true indoc.workspace = true diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 5bd881d9e..918bb599f 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -25,6 +25,7 @@ use icp::{ project::{WorkspaceInstance, WorkspaceInstancesError, workspace_instances}, store_artifact, }; +use icp_sync_plugin::{covering_dirs, distinct_paths}; use snafu::{OptionExt, ResultExt, Snafu}; use tar::Builder; @@ -768,36 +769,56 @@ async fn prepare_plugin_step( // `files` cannot collide with the `files/` area used for plugin input files. // The declared paths are rewritten to their archive locations; each entry's // map key is carried through unchanged. - let bundle_dirs = adapter.dirs.as_ref().map(|dirs| { - dirs.map_paths(|dir| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/dirs/{}", - normalize_archive_dir(dir) - ); - out.plugin_dirs.push(DirEntry { - src_path: canister_path.join(dir), - archive_prefix: archive_join(prefix, &manifest_path), - }); - manifest_path - }) - }); - + let dirs_prefix = format!("plugins/{path_name}/{idx}/dirs"); + let files_prefix = format!("plugins/{path_name}/{idx}/files"); + let bundle_dirs = adapter + .dirs + .as_ref() + .map(|dirs| dirs.map_paths(|dir| format!("{dirs_prefix}/{}", normalize_archive_dir(dir)))); let bundle_files = adapter.files.as_ref().map(|files| { - files.map_paths(|file| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/files/{}", - normalize_archive_dir(file) - ); - out.plugin_files.push(PluginFile { - src_path: canister_path.join(file), - archive_path: archive_join(prefix, &manifest_path), - canister_name: canister.name.clone(), - orig_file: file.to_string(), - }); - manifest_path - }) + files.map_paths(|file| format!("{files_prefix}/{}", normalize_archive_dir(file))) }); + // The rewritten manifest above keeps every declared entry; the archive holds + // the trees and files behind them, of which there are fewer. A directory + // named under two keys is one tree to copy, and a declared subdirectory of + // another is already inside its copy — writing either twice would collide in + // the archive. The reduction runs over the paths as declared, so two that + // only *look* alike once rewritten (`../shared` and `shared` both normalize + // to `shared`) stay separate and are still caught as a collision. + for dir in covering_dirs( + adapter + .dirs + .iter() + .flat_map(plugin::NamedPaths::entries) + .map(|entry| entry.path), + ) { + out.plugin_dirs.push(DirEntry { + src_path: canister_path.join(dir), + archive_prefix: archive_join( + prefix, + &format!("{dirs_prefix}/{}", normalize_archive_dir(dir)), + ), + }); + } + for file in distinct_paths( + adapter + .files + .iter() + .flat_map(plugin::NamedPaths::entries) + .map(|entry| entry.path), + ) { + out.plugin_files.push(PluginFile { + src_path: canister_path.join(file), + archive_path: archive_join( + prefix, + &format!("{files_prefix}/{}", normalize_archive_dir(file)), + ), + canister_name: canister.name.clone(), + orig_file: file.to_string(), + }); + } + Ok(SyncStep::Plugin(Box::new(plugin::Adapter { source: SourceField::Local(LocalSource { path: plugin_wasm_path.as_str().into(), diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 66feb2c84..49b6c2a99 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1383,6 +1383,108 @@ fn bundle_preserves_plugin_path_keys() { ); } +/// `dirs:`/`files:` are configuration as well as sandbox grants, so the same path may be +/// named under several keys, and one key's directory may sit inside another's. The bundled +/// manifest keeps every entry as declared; the archive holds one copy of each tree, since +/// two copies of one directory (or a copy of a directory already inside another) cannot be +/// written to the archive at all. +#[test] +fn bundle_archives_aliased_plugin_paths_once() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let plugin_bytes: &[u8] = b"\x00asm\x01\x00\x00\x00plugin"; + write(&project_dir.join("plugin.wasm"), plugin_bytes).expect("failed to write plugin"); + + let inner = project_dir.join("data/inner"); + create_dir_all(&inner).expect("failed to create dir"); + write_string(&project_dir.join("data/top.txt"), "top").expect("failed to write file"); + write_string(&inner.join("deep.txt"), "deep").expect("failed to write file"); + write_string(&project_dir.join("config.toml"), "key=value").expect("failed to write config"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: plugin.wasm + dirs: + seed: data + backup: data + sub: data/inner + files: + main: config.toml + fallback: ./config.toml + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut archived: Vec = Vec::new(); + let mut manifest_yaml = String::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path == "icp.yaml" { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + archived.push(path); + } + + // One copy of the tree, holding what the nested entry points at. + for expected in [ + "plugins/my-canister/0/dirs/data/top.txt", + "plugins/my-canister/0/dirs/data/inner/deep.txt", + "plugins/my-canister/0/files/config.toml", + ] { + assert_eq!( + archived.iter().filter(|path| *path == expected).count(), + 1, + "{expected} should appear exactly once; archive holds {archived:?}" + ); + } + + // Every declared entry survives, keys and all. + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let step = &parsed["canisters"][0]["sync"]["steps"][0]; + for (key, expected) in [ + ("seed", "plugins/my-canister/0/dirs/data"), + ("backup", "plugins/my-canister/0/dirs/data"), + ("sub", "plugins/my-canister/0/dirs/data/inner"), + ] { + assert_eq!(step["dirs"][key].as_str(), Some(expected)); + } + for key in ["main", "fallback"] { + assert_eq!( + step["files"][key].as_str(), + Some("plugins/my-canister/0/files/config.toml") + ); + } +} + /// An `icp_appmanifest.yaml` next to the project manifest must be included in the bundle, with its /// top-level `images` paths relocated under a top-level `images/` folder and the /// referenced image files copied alongside. Unrelated metadata is preserved. diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index d2fa023b4..a6f212fca 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -1,6 +1,7 @@ mod path; mod runtime; +pub use path::{covering_dirs, distinct_paths}; pub use runtime::{ CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, diff --git a/crates/icp-sync-plugin/src/path.rs b/crates/icp-sync-plugin/src/path.rs index 625f7ac68..3f8615c70 100644 --- a/crates/icp-sync-plugin/src/path.rs +++ b/crates/icp-sync-plugin/src/path.rs @@ -1,6 +1,8 @@ //! Path-safety helpers used by the host runtime to validate declared `dirs`/`files` //! entries before preopening directories or reading files under the canister base dir. +use std::collections::HashSet; + use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; /// Returns `true` if `rel` cannot be safely joined onto a base directory @@ -59,6 +61,105 @@ pub(crate) fn first_symlink_component(base: &Utf8Path, rel: &str) -> Option Vec<&str> { + path.split('/') + .filter(|part| !part.is_empty() && *part != ".") + .collect() +} + +/// Reduce declared directories to the ones that actually have to be opened. +/// +/// `dirs` is configuration as much as it is a sandbox grant: a plugin may +/// legitimately be handed the same tree under several keys, or a tree and a +/// subtree of it, and it is told about every entry that was declared. The grant +/// behind those entries has no such multiplicity — opening a directory twice, or +/// opening one already reachable through an ancestor, conveys no further access. +/// Callers keep the declared list as configuration and open only what this +/// returns; a nested declared directory is reached through the ancestor covering +/// it. +/// +/// Retained paths keep their written spelling and first-occurrence order. +/// Comparison is component-wise, so `data` covers `./data/inner` but not +/// `database`. Paths are expected to be relative and free of `..` (see +/// [`escapes_base`]); a `..` compares as an ordinary name, which can only leave +/// the result less reduced. +pub fn covering_dirs<'a>(dirs: impl IntoIterator) -> Vec<&'a str> { + let dirs: Vec<&str> = dirs.into_iter().collect(); + let parts: Vec> = dirs.iter().map(|dir| components(dir)).collect(); + dirs.iter() + .enumerate() + .filter(|(i, _)| { + !parts.iter().enumerate().any(|(j, other)| { + j != *i + && parts[*i].starts_with(other) + // A strict ancestor always covers; between equals, the first written wins. + && (other.len() < parts[*i].len() || j < *i) + }) + }) + .map(|(_, dir)| *dir) + .collect() +} + +/// Reduce declared paths to the distinct ones, keeping the written spelling and +/// first-occurrence order. +/// +/// [`covering_dirs`] without the containment rule, for entries that name files: +/// `./a.json` and `a.json` are one file, but a file never subsumes another the +/// way a directory subsumes its contents. +pub fn distinct_paths<'a>(paths: impl IntoIterator) -> Vec<&'a str> { + let mut seen: HashSet> = HashSet::new(); + paths + .into_iter() + .filter(|path| seen.insert(components(path))) + .collect() +} + +#[cfg(test)] +mod covering_tests { + use super::*; + + #[test] + fn unrelated_dirs_are_all_kept() { + assert_eq!( + covering_dirs(["assets", "config", "data/seed"]), + ["assets", "config", "data/seed"], + ); + } + + #[test] + fn duplicates_collapse_to_the_first_spelling() { + assert_eq!(covering_dirs(["./data", "data", "data/"]), ["./data"]); + } + + #[test] + fn nested_dirs_collapse_to_their_ancestor_whichever_is_written_first() { + assert_eq!(covering_dirs(["data", "data/inner"]), ["data"]); + assert_eq!(covering_dirs(["data/inner", "data"]), ["data"]); + // Transitive: `data` covers `data/a` covers `data/a/b`. + assert_eq!(covering_dirs(["data/a/b", "data/a", "data"]), ["data"]); + } + + #[test] + fn a_name_prefix_is_not_an_ancestor() { + assert_eq!(covering_dirs(["data", "database"]), ["data", "database"]); + } + + #[test] + fn distinct_paths_dedupes_without_containment() { + assert_eq!( + distinct_paths(["./a.json", "a.json", "b.json", "dir/a.json"]), + ["./a.json", "b.json", "dir/a.json"], + ); + } +} + #[cfg(test)] mod escapes_base_tests { use super::*; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index ae3d76f91..c7690e7b4 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -289,6 +289,9 @@ pub enum RunPluginError { ))] SymlinkDir { dir: String, link: Utf8PathBuf }, + #[snafu(display("plugin dir '{dir}' is not an existing directory"))] + MissingDir { dir: String }, + #[snafu(display("failed to preopen directory '{dir}' for the plugin"))] PreopenDir { source: wasmtime::Error, @@ -498,9 +501,9 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin path: wasm_path.clone(), })?; - // Preopen each declared directory read-only. The guest sees it at the - // same relative path it used in the manifest. - let mut wasi_builder = wasmtime_wasi::WasiCtxBuilder::new(); + // Check every declared directory: each one is handed to the plugin as + // configuration, so it is rejected for being unsafe or unusable whether or + // not it ends up needing a preopen of its own. for KeyedPath { path: dir, .. } in &dirs { ensure!(!crate::path::escapes_base(dir), UnsafeDirSnafu { dir }); // Reject symlinks in the declared path: neither the final entry nor any @@ -510,6 +513,17 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin if let Some(link) = crate::path::first_symlink_component(&base_dir, dir) { return SymlinkDirSnafu { dir, link }.fail(); } + let host_path = base_dir.join(dir); + let is_dir = std::fs::metadata(host_path.as_std_path()).is_ok_and(|meta| meta.is_dir()); + ensure!(is_dir, MissingDirSnafu { dir }); + } + + // Preopen read-only, one per distinct tree — a directory declared twice, or + // one already reachable through a declared ancestor, needs no preopen of its + // own. The guest sees each preopen at the same relative path it used in the + // manifest, and reaches a nested declared directory through its ancestor. + let mut wasi_builder = wasmtime_wasi::WasiCtxBuilder::new(); + for dir in crate::path::covering_dirs(dirs.iter().map(|d| d.path.as_str())) { let host_path = base_dir.join(dir); wasi_builder .preopened_dir( @@ -938,7 +952,7 @@ mod tests { // ------------------------------------------------------------------------- #[test] - fn preopen_dir_error_on_missing_dir() { + fn missing_dir_is_rejected() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; @@ -946,10 +960,46 @@ mod tests { inv.dirs = unkeyed(&["nonexistent_dir"]); assert!(matches!( run_plugin(inv), - Err(RunPluginError::PreopenDir { .. }) + Err(RunPluginError::MissingDir { .. }) )); } + /// A directory declared under several keys, or nested inside another + /// declared one, reaches the plugin as every entry it was written as. Only + /// the preopens behind those entries collapse — `data/inner` has none of its + /// own here, and is read through the `data` preopen that covers it. + #[test] + fn aliased_and_nested_dirs_are_all_readable() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let base = tmp.path(); + std::fs::create_dir_all(base.join("data/inner")).expect("create dir"); + std::fs::write(base.join("data/top.txt"), b"top").expect("write file"); + std::fs::write(base.join("data/inner/deep.txt"), b"deep").expect("write file"); + + let mut inv = invocation(wasm_path, "read-dirs"); + inv.base_dir = base.to_path_buf(); + inv.dirs = [("seed", "data"), ("backup", "data"), ("sub", "data/inner")] + .into_iter() + .map(|(key, path)| KeyedPath { + key: Some(key.to_owned()), + path: path.to_owned(), + }) + .collect(); + + let lines = run_plugin(inv).expect("plugin should succeed"); + assert_eq!( + lines, + [ + "seed=inner,top.txt".to_string(), + "backup=inner,top.txt".to_string(), + "sub=deep.txt".to_string(), + ], + ); + } + #[cfg(unix)] #[test] fn symlinked_dir_is_rejected() { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index d5a4481cc..ef6fa7587 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -5,14 +5,17 @@ interface types { /// Whether a canister call is an update or a query. enum call-type { update, query } - /// A directory the host preopened on behalf of the plugin. + /// A directory the host made readable for the plugin. record dir-input { /// The map key this directory was declared under in the manifest, or /// `none` when `dirs` was written as a plain list. Several entries share /// one key when a key maps to a list of directories. key: option, /// Path of the directory as declared in the manifest (relative to the - /// canister directory). The host preopens it at this same path. + /// canister directory). It is readable at this same path. Entries may + /// repeat a path or name a directory inside another entry's; the host + /// preopens each distinct tree once, so such an entry is read through + /// the preopen that covers it. path: string, } @@ -82,8 +85,8 @@ interface types { /// Name of the environment being synced (e.g. "production", "local"). environment: string, /// Directories declared in the manifest step's `dirs` setting. - /// The host preopens each entry via WASI; the plugin can traverse - /// them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). + /// The host makes each entry readable via WASI preopens; the plugin + /// traverses them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). /// Each entry carries the map key it was declared under, if any (see /// `dir-input`). dirs: list, diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 2c7db0cb7..c3c849f83 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -44,6 +44,25 @@ impl Guest for TestPlugin { } Ok(()) } + // List each declared dir as `key=entry,entry`, so the host can + // assert a dir reached only through a preopened ancestor is still + // readable. + "read-dirs" => { + for dir in &input.dirs { + let mut names = std::fs::read_dir(&dir.path) + .and_then(|entries| { + entries + .map(|entry| { + entry.map(|e| e.file_name().to_string_lossy().into_owned()) + }) + .collect::, _>>() + }) + .map_err(|err| format!("reading '{}': {err}", dir.path))?; + names.sort(); + eprintln!("{}={}", dir.key.as_deref().unwrap_or("-"), names.join(",")); + } + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index d3814f7d3..9c7069ecf 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -229,10 +229,12 @@ pub struct Adapter { pub sha256: Option, /// Directories (relative to canister directory) the plugin may read from. - /// Each entry must be a directory; it is preopened via WASI so the plugin - /// can traverse it using standard filesystem APIs. Written as a plain list - /// of paths, or as a map of name → path (or list of paths); the name is - /// surfaced to the plugin as each entry's `key`. + /// Each entry must be a directory; it is made readable via WASI so the + /// plugin can traverse it using standard filesystem APIs. Written as a plain + /// list of paths, or as a map of name → path (or list of paths); the name is + /// surfaced to the plugin as each entry's `key`. Entries may repeat a + /// directory or name one inside another's — the plugin is told about each + /// entry as written, and reads them all. pub dirs: Option, /// Files (relative to canister directory) the host reads and passes to diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 853cab13e..2c53a347f 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -108,7 +108,8 @@ The plugin runs with a deliberately narrow capability surface. ### Filesystem -- Each directory in `dirs:` is preopened **read-only**. The plugin sees it at the same relative path it used in the manifest (e.g. `dirs: ["assets"]` is visible as `assets/` inside the guest) and traverses it with standard filesystem APIs (`std::fs` in Rust). +- Each directory in `dirs:` is readable **read-only**. The plugin sees it at the same relative path it used in the manifest (e.g. `dirs: ["assets"]` is visible as `assets/` inside the guest) and traverses it with standard filesystem APIs (`std::fs` in Rust). +- Entries may name the same directory under several keys, or name a directory inside another entry's, and the plugin is told about each entry as written. The preopens behind them are one per distinct tree: an entry nested inside another is read through the preopen covering it, which grants nothing extra. - Files in `files:` are read by the host up front and passed inline in `sync-exec-input.files`. The plugin reads their content from the input struct, not from disk. - Any path outside a preopen is invisible. Writes, creates, deletes, renames, and symlinks that escape a preopen are rejected by the sandbox at runtime. - Paths in `dirs:`/`files:` must be relative and may not contain `..`. They also may not be — or traverse — a symlink: each declared entry is rejected if it or any of its parent components is a symlink, so a declared path cannot resolve to a target outside the canister directory. (This restriction may be relaxed later if a safe use case emerges.) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d91e58c77..b1a46582e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -169,7 +169,7 @@ sync: | `path` | string | One of `path` or `url` | Local path to the wasm, relative to the canister directory | | `url` | string | One of `path` or `url` | URL to download the wasm from | | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | -| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | +| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is made readable read-only via WASI | | `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | | `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | | `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index c8aaeae3f..9c2c60912 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -88,7 +88,7 @@ "type": "null" } ], - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is made readable via WASI so the\nplugin can traverse it using standard filesystem APIs. Written as a plain\nlist of paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`. Entries may repeat a\ndirectory or name one inside another's — the plugin is told about each\nentry as written, and reads them all." }, "fields": { "additionalProperties": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index b0ca7f2d8..ef3096767 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -88,7 +88,7 @@ "type": "null" } ], - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is made readable via WASI so the\nplugin can traverse it using standard filesystem APIs. Written as a plain\nlist of paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`. Entries may repeat a\ndirectory or name one inside another's — the plugin is told about each\nentry as written, and reads them all." }, "fields": { "additionalProperties": { From b6ea66c0e3048b0b0e04660acef7540114114597 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Sat, 22 Aug 2026 21:34:47 -0700 Subject: [PATCH 19/51] Add get-metadata-section function --- Cargo.lock | 1 + crates/icp-cli/tests/sync_tests.rs | 18 ++- crates/icp-sync-plugin/Cargo.toml | 1 + crates/icp-sync-plugin/DESIGN.md | 46 ++++++- crates/icp-sync-plugin/src/runtime.rs | 128 ++++++++++++++++-- crates/icp-sync-plugin/sync-plugin.wit | 36 ++++- .../tests/fixtures/test-plugin/src/lib.rs | 13 ++ crates/icp/src/manifest/adapter/plugin.rs | 11 +- docs/concepts/sync-plugins.md | 30 +++- docs/guides/writing-sync-plugins.md | 22 ++- docs/reference/configuration.md | 6 +- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- examples/icp-sync-plugin/README.md | 23 +++- examples/icp-sync-plugin/plugin/src/lib.rs | 17 ++- 15 files changed, 319 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b68d3f5b2..bd140f275 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3785,6 +3785,7 @@ dependencies = [ "console 0.16.3", "hex", "ic-agent", + "ic-management-canister-types 0.8.0", "icp-canister-interfaces", "semver", "snafu", diff --git a/crates/icp-cli/tests/sync_tests.rs b/crates/icp-cli/tests/sync_tests.rs index 888dabdb6..0d9dad12a 100644 --- a/crates/icp-cli/tests/sync_tests.rs +++ b/crates/icp-cli/tests/sync_tests.rs @@ -437,11 +437,18 @@ async fn sync_plugin_registers_seed_data() { clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) .mint_cycles(10 * TRILLION); + // The plugin also reads the canister's candid:service metadata section and + // reports it. No proxy is configured here, so the read is a direct + // read_state; this manifest builds the wasm with a plain `cp`, skipping the + // example's ic-wasm step, so the section genuinely isn't there — proving the + // host performed the round-trip and mapped a proven-absent section to `none` + // rather than to an error. ctx.icp() .current_dir(&project_dir) .args(["deploy", "--environment", "random-environment"]) .assert() - .success(); + .success() + .stderr(contains("candid:service: absent")); // Query the canister to verify all three fruits were registered ctx.icp() @@ -814,6 +821,12 @@ async fn sync_plugin_routes_through_proxy() { // Deploy through proxy so the proxy canister becomes a controller of my-canister. // deploy also runs the sync step: the plugin routes set_uploader through the proxy // (direct: false, proxy is controller), then calls register directly with the user identity. + // + // Its metadata read is proxied too, so it reaches the canister as the + // management canister's `canister_metadata` rather than as a read_state. + // This manifest skips the example's ic-wasm step, so the section really is + // missing — and the host must report the resulting rejection as an absent + // section, the same answer a direct read proves from the certificate. ctx.icp() .current_dir(&project_dir) .args([ @@ -824,7 +837,8 @@ async fn sync_plugin_routes_through_proxy() { "random-environment", ]) .assert() - .success(); + .success() + .stderr(contains("candid:service: absent")); // Query the canister to verify all three fruits were registered ctx.icp() diff --git a/crates/icp-sync-plugin/Cargo.toml b/crates/icp-sync-plugin/Cargo.toml index 74feb682e..a43722968 100644 --- a/crates/icp-sync-plugin/Cargo.toml +++ b/crates/icp-sync-plugin/Cargo.toml @@ -14,6 +14,7 @@ candid.workspace = true console.workspace = true hex.workspace = true ic-agent.workspace = true +ic-management-canister-types.workspace = true icp-canister-interfaces.workspace = true semver.workspace = true snafu.workspace = true diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 27617fc14..de02c264f 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -36,6 +36,13 @@ docs; the *reasons* behind those choices are recorded here. one deployment. (In the earlier `@0.1.0` interface `canister-call` had no target and always reached the canister being synced; see *Interface versioning* below.) +- **`get-metadata-section` mirrors `canister-call`'s targeting and routing** — it + takes the same `call-target` (enforced against `canisters:` the same way) and + the same `direct` flag, so one mental model covers both imports. Its return is + `result>, string>`: a missing section is an ordinary answer for + a plugin probing for an optional section, not a failure it must recognize by + parsing error text. The host pays for that guarantee on the proxied path — see + *Metadata reads* below. - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still @@ -65,7 +72,8 @@ crates/icp-sync-plugin/ path.rs — declared-path safety checks (escapes_base, symlinks) sync-plugin.wit — current WIT interface, v0.2.0 sync-plugin-v1.wit — frozen WIT interface, v0.1.0 - Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, candid, camino, snafu, tokio, semver + Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, ic-management-canister-types, + candid, camino, snafu, tokio, semver ``` Public function: @@ -134,8 +142,8 @@ struct HostState { ``` `HostState` implements `WasiView` so wasmtime_wasi can access the WASI context. -`canister_call` uses `tokio::runtime::Handle::current().block_on(...)` because -the caller already wraps the synchronous `run_plugin` in +Both imports use `tokio::runtime::Handle::current().block_on(...)` because the +caller already wraps the synchronous `run_plugin` in `tokio::task::block_in_place`. For a v0.2.0 plugin the target is resolved from the request's `call-target` by `resolve_call_target`, which enforces the `callable` set; for a v0.1.0 plugin the target is always `host_canister_id`. @@ -143,6 +151,29 @@ When a proxy is configured and the call is a non-`direct` update, it is encoded as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise it goes straight to the resolved target via `ic-agent`. +### Metadata reads (two routes, one answer) + +`get-metadata-section` cannot reuse the call path: `read_state` is not a canister +method, so a proxy canister has nothing to forward. The two routes are therefore +different protocols reaching the same data, chosen by the request's `direct` flag +exactly as `canister-call` chooses one: + +- **Direct** — `Agent::read_state_canister_metadata`, signed by the sync + identity. Absence is *proven* by the certificate, surfacing as + `AgentError::LookupPathAbsent`, which the host maps to `Ok(None)`. +- **Proxied** — `ProxyArgs` aimed at the management canister's + `canister_metadata`, so the controller check runs against the proxy. This is + the same shape the CLI's own management calls take through + `update_or_proxy_raw`; the runtime inlines it rather than depending on the CLI. + +The routes disagree on how absence arrives: the management canister *rejects* +("The canister `` has no metadata section with the name ``.") and the +proxy hands the plugin a reason string with no reject code attached, so the host +matches `NO_SUCH_SECTION_REJECT` against it to produce the same `Ok(None)` a +direct read proves. Matching replica text is the price of one uniform contract; +it fails in the safe direction — a reword upstream turns absence back into an +error rather than into a wrong answer. + ### Interface versioning (parallel v0.1.0 / v0.2.0 support) A component built with wit-bindgen imports the interface it `use`s as a @@ -163,10 +194,11 @@ simply dropped for a v1 plugin; a v1 plugin cannot observe them, so declaring The compute-time limit is enforced with wasmtime's epoch interruption: a background thread calls `Engine::increment_epoch` once per second, and the store -deadline (`set_epoch_deadline`) bounds pure wasm execution. Because canister -calls block the guest while the host awaits the network, `canister_call` records -the elapsed time and the `epoch_deadline_callback` grants it back via -`epoch_extension` — so network latency is *not* charged against the limit. The +deadline (`set_epoch_deadline`) bounds pure wasm execution. Because a host +call blocks the guest while the host awaits the network, both imports record the +elapsed time (`refund_host_call_time`) and the `epoch_deadline_callback` grants +it back via `epoch_extension` — so network latency is *not* charged against the +limit. The ticker thread stops when its RAII guard drops at the end of `run_plugin`. The deadline in seconds is the `compute_limit_secs` parameter. The CLI resolves diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index c7690e7b4..b57120395 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -24,7 +24,9 @@ pub const PLUGIN_COMPUTE_LIMIT_ENV: &str = "ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS"; use bytes::Bytes; use camino::Utf8PathBuf; use candid::{Encode, Principal}; -use ic_agent::Agent; +use ic_agent::{Agent, AgentError}; +use ic_management_canister_types::{CanisterMetadataArgs, CanisterMetadataResult}; +use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; use semver::{Version, VersionReq}; use snafu::prelude::*; use tokio::io::{self, AsyncWrite}; @@ -93,6 +95,16 @@ pub struct CallableCanisters { pub by_name: BTreeMap, } +/// The distinguishing phrase in the management canister's rejection of a +/// metadata read for a section the target does not have ("The canister has +/// no metadata section with the name ."). A proxied read reaches the +/// plugin as reject text, not as a code, so recognizing absence — which +/// [`HostState::do_get_metadata_section`] reports as `Ok(None)`, matching what +/// a direct read proves from the certificate — means matching that text. A +/// reword upstream turns absence back into an error rather than into a wrong +/// answer. +const NO_SUCH_SECTION_REJECT: &str = "no metadata section"; + /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing /// that the plugin listed it in `canisters`. The canister being synced (`host`) /// is always permitted. @@ -119,7 +131,8 @@ struct HostState { /// Canisters the plugin declared in `canisters` and may also call. callable: CallableCanisters, agent: Arc, - /// Proxy canister to route update calls through, if configured. + /// Proxy canister to route update calls and metadata reads through, if + /// configured. proxy: Option, // WASI context. Preopened directories in this context are the only // filesystem locations the plugin can access. @@ -154,8 +167,6 @@ impl HostState { direct: bool, cycles: u64, ) -> Result, String> { - use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; - let agent = Arc::clone(&self.agent); let proxy = if direct { None } else { self.proxy }; @@ -201,12 +212,85 @@ impl HostState { .map_err(|e| format!("canister call failed: {e}")), } }); - // Return the time spent in the host call to the compute budget so - // canister network latency doesn't count against the plugin's limit. + self.refund_host_call_time(start); + result + } + + /// Read a metadata section from an already-resolved target principal. + /// `Ok(None)` is the target reporting it has no such section, kept distinct + /// from a failed read so a plugin can probe for an optional section without + /// inspecting error text (see [`NO_SUCH_SECTION_REJECT`]). + /// + /// A direct read is a certified `read_state` signed by the sync identity — + /// `read_state` is not a canister method, so it cannot be forwarded. A + /// proxied read therefore goes the other way around: the proxy calls the + /// management canister's `canister_metadata` on the plugin's behalf, which + /// checks the *proxy* against the target's controllers and so reaches + /// sections private to it. + fn do_get_metadata_section( + &mut self, + target: Principal, + name: String, + direct: bool, + ) -> Result>, String> { + let agent = Arc::clone(&self.agent); + let proxy = if direct { None } else { self.proxy }; + + let start = Instant::now(); + let result = tokio::runtime::Handle::current().block_on(async move { + let Some(proxy_cid) = proxy else { + return match agent.read_state_canister_metadata(target, &name).await { + Ok(bytes) => Ok(Some(bytes)), + Err(AgentError::LookupPathAbsent(_)) => Ok(None), + Err(err) => Err(format!("metadata read failed: {err}")), + }; + }; + + let metadata_args = Encode!(&CanisterMetadataArgs { + canister_id: target, + name, + }) + .map_err(|e| format!("metadata encode failed: {e}"))?; + let proxy_args = ProxyArgs { + canister_id: Principal::management_canister(), + method: "canister_metadata".to_string(), + args: metadata_args, + cycles: candid::Nat::from(0u8), + }; + let encoded = Encode!(&proxy_args).map_err(|e| format!("proxy encode failed: {e}"))?; + let raw = agent + .update(&proxy_cid, "proxy") + .with_arg(encoded) + .await + .map_err(|e| format!("proxy call failed: {e}"))?; + let (result,): (ProxyResult,) = + candid::decode_args(&raw).map_err(|e| format!("proxy decode failed: {e}"))?; + match result { + ProxyResult::Ok(ok) => { + let (metadata,): (CanisterMetadataResult,) = candid::decode_args(&ok.result) + .map_err(|e| format!("metadata decode failed: {e}"))?; + Ok(Some(metadata.value)) + } + ProxyResult::Err(err) => { + let message = err.format_error(); + if message.contains(NO_SUCH_SECTION_REJECT) { + Ok(None) + } else { + Err(message) + } + } + } + }); + self.refund_host_call_time(start); + result + } + + /// Return the wall-clock time a host call spent off-wasm to the compute + /// budget, so network latency doesn't count against the plugin's limit. + fn refund_host_call_time(&self, start: Instant) { let elapsed_ticks = start.elapsed().as_secs() + 1; self.epoch_extension .fetch_add(elapsed_ticks, Ordering::Relaxed); - result } } @@ -230,6 +314,14 @@ impl v2::SyncPluginImports for HostState { req.cycles, ) } + + fn get_metadata_section( + &mut self, + req: v2::icp::sync_plugin::types::MetadataSectionRequest, + ) -> Result>, String> { + let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; + self.do_get_metadata_section(target, req.name, req.direct) + } } // -- v0.1.0 interface: calls always go to the canister being synced. ----------- @@ -424,7 +516,8 @@ pub struct PluginInvocation { pub host_canister_id: Principal, /// Agent used for canister calls. pub agent: Agent, - /// Proxy canister to route update calls through, if configured. + /// Proxy canister to route update calls and metadata reads through, if + /// configured. pub proxy: Option, /// Signing identity principal, surfaced to the plugin. pub identity_principal: Principal, @@ -1074,6 +1167,25 @@ mod tests { )); } + /// A metadata read names its target the same way a call does, and the host + /// enforces the `canisters` list before going to the network — so an + /// undeclared target is refused without a live canister to read from. + #[test] + fn metadata_read_of_undeclared_canister_is_rejected() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let lines = run_plugin(invocation(wasm_path, "metadata-undeclared")) + .expect("plugin should succeed"); + let [refusal] = &lines[..] else { + panic!("expected one refusal line, got: {lines:?}"); + }; + assert!( + refusal.contains("not permitted") && refusal.contains("undeclared"), + "got: {refusal}" + ); + } + #[test] fn plugin_exceeding_compute_limit_is_trapped() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index ef6fa7587..3ef62decc 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -136,11 +136,32 @@ interface types { /// for query calls. cycles: u64, } + + /// A request to read a canister's metadata section. + record metadata-section-request { + /// Which canister to read from. The same rule as + /// `canister-call-request.target` applies: `host` is always permitted, + /// a `name` must appear in the sync step's `canisters` list. + target: call-target, + /// Name of the metadata section, as spelled in the wasm module's custom + /// section minus the `icp:public `/`icp:private ` prefix — e.g. + /// `candid:service`. + name: string, + /// When true, the section is read straight from the target canister + /// with a certified `read_state` request signed by the sync identity, + /// which reaches a private section only if that identity controls the + /// target. When false (the default), the read is routed through the + /// proxy canister configured via `--proxy` — as a call to the + /// management canister's `canister_metadata` method, so a private + /// section gated on the proxy's control is readable. With no proxy + /// configured the read goes directly either way. + direct: bool, + } } /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, dir-input, file-input, field-input}; + use types.{sync-exec-input, canister-call-request, metadata-section-request, call-target, canister-id-entry, dir-input, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin @@ -154,6 +175,19 @@ world sync-plugin { /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; + /// Read a metadata section from a canister. + /// The `req.target` selects the canister under the same rule as + /// `canister-call`: the canister being synced (`host`), or one listed in + /// the sync step's `canisters` list, by name. + /// A direct read is a certified `read_state` request signed by the sync + /// identity; a proxied read (`direct` false, with `--proxy` configured) is + /// a call to the management canister's `canister_metadata` method made by + /// the proxy, which reaches sections private to the proxy's control. + /// Returns the section's raw bytes on success, `none` when the target + /// reports it has no section by that name, or an error message on failure. + /// The plugin is responsible for interpreting the bytes. + import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + // The plugin's stdout is captured and shown as transient progress in // the rolling step view of icp-cli; it is discarded when the step ends. // diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index c3c849f83..0655b5348 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -63,6 +63,19 @@ impl Guest for TestPlugin { } Ok(()) } + // Ask for a metadata section from a canister the step did not + // declare. The host must reject the target before it touches the + // network, so this needs no live canister; echo the refusal. + "metadata-undeclared" => { + let err = get_metadata_section(&MetadataSectionRequest { + target: CallTarget::Name("undeclared".to_string()), + name: "candid:service".to_string(), + direct: true, + }) + .expect_err("host must reject an undeclared target"); + eprintln!("{err}"); + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 9c7069ecf..0d6b1ffbe 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -249,11 +249,12 @@ pub struct Adapter { #[schemars(with = "Option>")] pub fields: Option>, - /// Canisters this plugin may call in addition to the canister being synced. - /// Each entry is a canister name resolved against the project's canister ID - /// table for the environment being synced (e.g. `backend`, or a namespaced - /// subproject canister such as `services/open-crm:backend`). The plugin - /// picks a target per call via the `call-target` in its `canister-call` + /// Canisters this plugin may call, or read metadata from, in addition to + /// the canister being synced. Each entry is a canister name resolved against + /// the project's canister ID table for the environment being synced (e.g. + /// `backend`, or a namespaced subproject canister such as + /// `services/open-crm:backend`). The plugin picks a target per request via + /// the `call-target` in its `canister-call` or `get-metadata-section` /// request; a target not listed here is rejected by the host. pub canisters: Option>, } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 2c53a347f..fe97d528a 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls, read canister metadata, and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -39,19 +39,23 @@ icp sync │ dirs/files/fields = what you declared in the manifest │ └─ plugin makes canister-call({ target, ... }) (× N) + and get-metadata-section({ target, name }) target = host (the canister being synced), or a canister from `canisters:` by name ``` ## The Plugin Interface -The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides one import (`canister-call`); the plugin provides one export (`exec`): +The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `get-metadata-section`); the plugin provides one export (`exec`): ```wit world sync-plugin { // Host import: call the canister being synced or one listed in `canisters:`. import canister-call: func(req: canister-call-request) -> result, string>; + // Host import: read a metadata section from one of those same canisters. + import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + // Plugin export: run the sync step. export exec: func(input: sync-exec-input) -> result<_, string>; } @@ -93,6 +97,25 @@ The plugin calls methods through the `canister-call` import. It picks a `target` The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. A name is the only way to address another canister — the host owns the name→principal mapping, which differs per environment. +### Reading canister metadata — `get-metadata-section` + +The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canister-metadata) — `candid:service`, for instance — through the `get-metadata-section` import: + +| Request field | Meaning | +|---------------|---------| +| `target` | Which canister to read from: `host`, or a canister declared in `canisters:` addressed by `name` — the same targets, and the same enforcement, as `canister-call` | +| `name` | The section name, without the `icp:public `/`icp:private ` prefix the wasm custom section carries (e.g. `candid:service`) | +| `direct` | When `false` (default), the read is routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, it always goes straight to the target | + +A successful read returns the section's raw bytes, or **absent** when the target reports it has no section by that name — so a plugin can probe for an optional section without matching on error text. Anything else (an unreachable canister, a section the caller may not read) is an error. + +The two routes differ in who the target sees asking, which decides what a **private** section will yield: + +- **Direct** — a certified `read_state` request signed by the sync identity. A private section requires that identity to control the target. +- **Proxied** — a call to the management canister's `canister_metadata` method made by the proxy, because `read_state` is not a canister method and cannot be forwarded. A private section requires the *proxy* to control the target — the same arrangement proxied update calls rely on. + +With no proxy configured, both settings read directly. + ### Logging — stdout and stderr The plugin's stdout and stderr are captured by the host (no logging import is needed — use ordinary `println!` / `eprintln!`): @@ -122,6 +145,7 @@ The plugin runs with a deliberately narrow capability surface. | Clocks, RNG, `wasi:io` | yes | Rust's `HashMap`, `chrono`, etc. work normally | | `process::exit` / panics | yes | abort the guest cleanly; the host surfaces the error | | Canister calls | yes | to the canister being synced, and to canisters declared in `canisters:` | +| Canister metadata reads | yes | the same set of canisters as calls | | Environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | | Network sockets / DNS | blocked | treat the network as unavailable | | Filesystem writes | blocked | no writable preopens | @@ -136,7 +160,7 @@ The plugin runs with a deliberately narrow capability surface. | Linear memory | wasm32 address space (≤ 4 GiB) | | stdout / stderr per stream | 1 MiB | -The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a `canister-call` to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. +The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `get-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. ## Next Steps diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 8e466b585..793e2fc1b 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -37,7 +37,7 @@ wit-bindgen = { version = "0.56", features = ["realloc"] } ## Generate Bindings and Implement `exec` -`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the `canister_call` host function. The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. +`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `get_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. ```rust // src/lib.rs @@ -88,6 +88,26 @@ A few things to note: - **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` — the name matches the entries in `input.canister_ids`. Names are the only way to reach another canister; the host resolves them per environment. The host rejects a target you did not declare. - **`direct` and `cycles` control proxy routing.** With `direct: false`, update calls go through the [proxy canister](proxy-canister.md) when one is configured, and `cycles` can fund the forwarded call. With `direct: true`, the call always goes straight to the target. See [The Plugin Interface](../concepts/sync-plugins.md#the-plugin-interface) for the full semantics. +## Read Canister Metadata + +`get_metadata_section` reads a [metadata section](../reference/cli.md#icp-canister-metadata) off a canister — useful for inspecting what is actually deployed before acting on it, e.g. its `candid:service` interface: + +```rust +let interface = get_metadata_section(&MetadataSectionRequest { + target: CallTarget::Host, // same targets, same rules, as canister_call + name: "candid:service".to_string(), + direct: false, // route through the proxy if one is configured +})?; + +match interface { + Some(bytes) => println!("interface: {}", String::from_utf8_lossy(&bytes)), + // `None` means the canister has no such section — not a failure. + None => println!("canister exposes no Candid interface"), +} +``` + +`direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--get-metadata-section) for the full semantics. + ## Read Declared Files and Directories A plugin can't see the filesystem freely — only what you grant it in the manifest's `dirs:` and `files:`. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b1a46582e..f52ffe0d9 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -154,7 +154,7 @@ sync: fields: # key-value fields passed inline api_url: https://example.com retries: 3 - canisters: # extra canisters the plugin may call + canisters: # extra canisters the plugin may reach - ledger # by name (resolved for the environment) - services/open-crm:backend @@ -172,7 +172,7 @@ sync: | `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is made readable read-only via WASI | | `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | | `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | -| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | +| `canisters` | array of string | No | Canisters the plugin may call, or read metadata from, in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | `dirs:` and `files:` each accept either a plain list of paths or a map. As a map, each key names a single path or a list of paths, and the key is surfaced to the plugin as that entry's `key` (a key mapping to a list produces several entries sharing it). A plain-list entry has no key. For example: @@ -194,7 +194,7 @@ A plugin receives every `fields:` value as a string. Numbers and booleans need n A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a canister defined in a subproject. A name that does not resolve to a known canister for the environment fails the sync step. -The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. +The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`), read those canisters' metadata sections, and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. ## Recipes diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 9c2c60912..effbf286d 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index ef3096767..65ece04f8 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/examples/icp-sync-plugin/README.md b/examples/icp-sync-plugin/README.md index 57518b0ab..058ec7218 100644 --- a/examples/icp-sync-plugin/README.md +++ b/examples/icp-sync-plugin/README.md @@ -27,13 +27,27 @@ A simple Rust canister with three methods: A Rust Wasm component that implements the `sync-plugin` world defined in `crates/icp-sync-plugin/sync-plugin.wit`. The host runtime calls its `exec` -export and provides a `canister-call` import the plugin uses to reach the -canister. +export and provides the `canister-call` and `get-metadata-section` imports the +plugin uses to reach the canister. ## How the plugin system is exercised This example is designed to demonstrate both routing modes of the -`canister-call` import — the `direct` flag — in a single sync run. +`canister-call` import — the `direct` flag — in a single sync run, plus a +metadata read that follows the same routing. + +### Read — `candid:service` via proxy (`direct: false`) + +Before calling anything, the plugin asks for the canister's `candid:service` +metadata section and reports its size. The build embeds that section with +`ic-wasm`, so it is there; had it not been, the read would return "absent" +rather than fail — a missing section is an answer, not an error. + +Routed through the proxy (`direct: false`), the read reaches the canister as the +management canister's `canister_metadata` method called by the proxy, so it is +the proxy's control over the canister that a private section would be checked +against. A direct read (`direct: true`) is a `read_state` signed by the user +identity instead. ### Call 1 — `set_uploader` via proxy (`direct: false`) @@ -65,6 +79,9 @@ icp sync │ identity-principal = │ proxy-canister-id = │ + ├─ get-metadata-section candid:service direct=false → proxy → mgmt canister + │ reports the section's size, or "absent" + │ ├─ canister-call set_uploader() direct=false → proxy → canister │ canister stores uploader = │ diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index c3fdee356..2e19ce4d0 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -17,7 +17,20 @@ impl Guest for Plugin { input.canister_id, input.environment ); - // 1. Set the uploader to the current identity principal. + // 1. Report the canister's Candid interface, read from its metadata. + // Reported rather than required: the section is only there if the + // build embedded it (this project's build does, via ic-wasm). + let interface = get_metadata_section(&MetadataSectionRequest { + target: CallTarget::Host, + name: "candid:service".to_string(), + direct: false, + })?; + match &interface { + Some(bytes) => eprintln!("candid:service: {} bytes", bytes.len()), + None => eprintln!("candid:service: absent"), + } + + // 2. Set the uploader to the current identity principal. // Routed through the proxy (direct: false) so the controller-gated // call is signed by the proxy canister, which is a controller. let uploader = Principal::from_text(&input.identity_principal) @@ -33,7 +46,7 @@ impl Guest for Plugin { })?; println!("set_uploader ({}): ok", input.identity_principal); - // 2. Register every file found by traversing the preopened dirs. + // 3. Register every file found by traversing the preopened dirs. // Direct calls (direct: true) because register is gated on the // uploader principal, which is the current identity — not the proxy. let mut registered = 0u32; From eb5824568879b2f81a74ea8eb0731303a6013ab4 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 24 Aug 2026 08:55:25 -0700 Subject: [PATCH 20/51] rename --- crates/icp-sync-plugin/DESIGN.md | 4 ++-- crates/icp-sync-plugin/src/runtime.rs | 8 ++++---- crates/icp-sync-plugin/sync-plugin.wit | 2 +- .../tests/fixtures/test-plugin/src/lib.rs | 2 +- crates/icp/src/manifest/adapter/plugin.rs | 2 +- docs/concepts/sync-plugins.md | 12 ++++++------ docs/guides/writing-sync-plugins.md | 8 ++++---- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- examples/icp-sync-plugin/README.md | 4 ++-- examples/icp-sync-plugin/plugin/src/lib.rs | 2 +- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index de02c264f..a8e35a778 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -36,7 +36,7 @@ docs; the *reasons* behind those choices are recorded here. one deployment. (In the earlier `@0.1.0` interface `canister-call` had no target and always reached the canister being synced; see *Interface versioning* below.) -- **`get-metadata-section` mirrors `canister-call`'s targeting and routing** — it +- **`canister-metadata-section` mirrors `canister-call`'s targeting and routing** — it takes the same `call-target` (enforced against `canisters:` the same way) and the same `direct` flag, so one mental model covers both imports. Its return is `result>, string>`: a missing section is an ordinary answer for @@ -153,7 +153,7 @@ straight to the resolved target via `ic-agent`. ### Metadata reads (two routes, one answer) -`get-metadata-section` cannot reuse the call path: `read_state` is not a canister +`canister-metadata-section` cannot reuse the call path: `read_state` is not a canister method, so a proxy canister has nothing to forward. The two routes are therefore different protocols reaching the same data, chosen by the request's `direct` flag exactly as `canister-call` chooses one: diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index b57120395..2505cf112 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -99,7 +99,7 @@ pub struct CallableCanisters { /// metadata read for a section the target does not have ("The canister has /// no metadata section with the name ."). A proxied read reaches the /// plugin as reject text, not as a code, so recognizing absence — which -/// [`HostState::do_get_metadata_section`] reports as `Ok(None)`, matching what +/// [`HostState::do_canister_metadata_section`] reports as `Ok(None)`, matching what /// a direct read proves from the certificate — means matching that text. A /// reword upstream turns absence back into an error rather than into a wrong /// answer. @@ -227,7 +227,7 @@ impl HostState { /// management canister's `canister_metadata` on the plugin's behalf, which /// checks the *proxy* against the target's controllers and so reaches /// sections private to it. - fn do_get_metadata_section( + fn do_canister_metadata_section( &mut self, target: Principal, name: String, @@ -315,12 +315,12 @@ impl v2::SyncPluginImports for HostState { ) } - fn get_metadata_section( + fn canister_metadata_section( &mut self, req: v2::icp::sync_plugin::types::MetadataSectionRequest, ) -> Result>, String> { let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; - self.do_get_metadata_section(target, req.name, req.direct) + self.do_canister_metadata_section(target, req.name, req.direct) } } diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 3ef62decc..27066621b 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -186,7 +186,7 @@ world sync-plugin { /// Returns the section's raw bytes on success, `none` when the target /// reports it has no section by that name, or an error message on failure. /// The plugin is responsible for interpreting the bytes. - import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; // The plugin's stdout is captured and shown as transient progress in // the rolling step view of icp-cli; it is discarded when the step ends. diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 0655b5348..116eb0bc8 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -67,7 +67,7 @@ impl Guest for TestPlugin { // declare. The host must reject the target before it touches the // network, so this needs no live canister; echo the refusal. "metadata-undeclared" => { - let err = get_metadata_section(&MetadataSectionRequest { + let err = canister_metadata_section(&MetadataSectionRequest { target: CallTarget::Name("undeclared".to_string()), name: "candid:service".to_string(), direct: true, diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 0d6b1ffbe..e15affd7c 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -254,7 +254,7 @@ pub struct Adapter { /// the project's canister ID table for the environment being synced (e.g. /// `backend`, or a namespaced subproject canister such as /// `services/open-crm:backend`). The plugin picks a target per request via - /// the `call-target` in its `canister-call` or `get-metadata-section` + /// the `call-target` in its `canister-call` or `canister-metadata-section` /// request; a target not listed here is rejected by the host. pub canisters: Option>, } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index fe97d528a..6983dd480 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -39,14 +39,14 @@ icp sync │ dirs/files/fields = what you declared in the manifest │ └─ plugin makes canister-call({ target, ... }) (× N) - and get-metadata-section({ target, name }) + and canister-metadata-section({ target, name }) target = host (the canister being synced), or a canister from `canisters:` by name ``` ## The Plugin Interface -The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `get-metadata-section`); the plugin provides one export (`exec`): +The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `canister-metadata-section`); the plugin provides one export (`exec`): ```wit world sync-plugin { @@ -54,7 +54,7 @@ world sync-plugin { import canister-call: func(req: canister-call-request) -> result, string>; // Host import: read a metadata section from one of those same canisters. - import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; // Plugin export: run the sync step. export exec: func(input: sync-exec-input) -> result<_, string>; @@ -97,9 +97,9 @@ The plugin calls methods through the `canister-call` import. It picks a `target` The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. A name is the only way to address another canister — the host owns the name→principal mapping, which differs per environment. -### Reading canister metadata — `get-metadata-section` +### Reading canister metadata — `canister-metadata-section` -The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canister-metadata) — `candid:service`, for instance — through the `get-metadata-section` import: +The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canister-metadata) — `candid:service`, for instance — through the `canister-metadata-section` import: | Request field | Meaning | |---------------|---------| @@ -160,7 +160,7 @@ The plugin runs with a deliberately narrow capability surface. | Linear memory | wasm32 address space (≤ 4 GiB) | | stdout / stderr per stream | 1 MiB | -The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `get-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. +The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `canister-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. ## Next Steps diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 793e2fc1b..64bf19657 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -37,7 +37,7 @@ wit-bindgen = { version = "0.56", features = ["realloc"] } ## Generate Bindings and Implement `exec` -`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `get_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. +`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `canister_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. ```rust // src/lib.rs @@ -90,10 +90,10 @@ A few things to note: ## Read Canister Metadata -`get_metadata_section` reads a [metadata section](../reference/cli.md#icp-canister-metadata) off a canister — useful for inspecting what is actually deployed before acting on it, e.g. its `candid:service` interface: +`canister_metadata_section` reads a [metadata section](../reference/cli.md#icp-canister-metadata) off a canister — useful for inspecting what is actually deployed before acting on it, e.g. its `candid:service` interface: ```rust -let interface = get_metadata_section(&MetadataSectionRequest { +let interface = canister_metadata_section(&MetadataSectionRequest { target: CallTarget::Host, // same targets, same rules, as canister_call name: "candid:service".to_string(), direct: false, // route through the proxy if one is configured @@ -106,7 +106,7 @@ match interface { } ``` -`direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--get-metadata-section) for the full semantics. +`direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--canister-metadata-section) for the full semantics. ## Read Declared Files and Directories diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index effbf286d..be092c3d0 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `canister-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 65ece04f8..11a3bf1ce 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `canister-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/examples/icp-sync-plugin/README.md b/examples/icp-sync-plugin/README.md index 058ec7218..8f2c8ee2e 100644 --- a/examples/icp-sync-plugin/README.md +++ b/examples/icp-sync-plugin/README.md @@ -27,7 +27,7 @@ A simple Rust canister with three methods: A Rust Wasm component that implements the `sync-plugin` world defined in `crates/icp-sync-plugin/sync-plugin.wit`. The host runtime calls its `exec` -export and provides the `canister-call` and `get-metadata-section` imports the +export and provides the `canister-call` and `canister-metadata-section` imports the plugin uses to reach the canister. ## How the plugin system is exercised @@ -79,7 +79,7 @@ icp sync │ identity-principal = │ proxy-canister-id = │ - ├─ get-metadata-section candid:service direct=false → proxy → mgmt canister + ├─ canister-metadata-section candid:service direct=false → proxy → mgmt canister │ reports the section's size, or "absent" │ ├─ canister-call set_uploader() direct=false → proxy → canister diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index 2e19ce4d0..09595a47d 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -20,7 +20,7 @@ impl Guest for Plugin { // 1. Report the canister's Candid interface, read from its metadata. // Reported rather than required: the section is only there if the // build embedded it (this project's build does, via ic-wasm). - let interface = get_metadata_section(&MetadataSectionRequest { + let interface = canister_metadata_section(&MetadataSectionRequest { target: CallTarget::Host, name: "candid:service".to_string(), direct: false, From 2f6ec2e5737b0c3ad3e95aa54ba1d791b1ef6baf Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 24 Aug 2026 09:42:53 -0700 Subject: [PATCH 21/51] fixes --- crates/icp-sync-plugin/DESIGN.md | 7 ++-- crates/icp-sync-plugin/src/runtime.rs | 50 ++++++++++++++++++++++++--- examples/icp-sync-plugin/icp.yaml | 2 +- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index a8e35a778..cfc2b31fd 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -158,9 +158,10 @@ method, so a proxy canister has nothing to forward. The two routes are therefore different protocols reaching the same data, chosen by the request's `direct` flag exactly as `canister-call` chooses one: -- **Direct** — `Agent::read_state_canister_metadata`, signed by the sync - identity. Absence is *proven* by the certificate, surfacing as - `AgentError::LookupPathAbsent`, which the host maps to `Ok(None)`. +- **Direct** — a `read_state` signed by the sync identity, so absence is + *proven* by the certificate rather than asserted. It requests `controllers` + alongside the metadata path, since only that distinguishes a canister with no + such section from one that was never created. - **Proxied** — `ProxyArgs` aimed at the management canister's `canister_metadata`, so the controller check runs against the proxy. This is the same shape the CLI's own management calls take through diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 2505cf112..2b4d4be7d 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -24,7 +24,8 @@ pub const PLUGIN_COMPUTE_LIMIT_ENV: &str = "ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS"; use bytes::Bytes; use camino::Utf8PathBuf; use candid::{Encode, Principal}; -use ic_agent::{Agent, AgentError}; +use ic_agent::Agent; +use ic_agent::hash_tree::{Label, LookupResult}; use ic_management_canister_types::{CanisterMetadataArgs, CanisterMetadataResult}; use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; use semver::{Version, VersionReq}; @@ -239,10 +240,49 @@ impl HostState { let start = Instant::now(); let result = tokio::runtime::Handle::current().block_on(async move { let Some(proxy_cid) = proxy else { - return match agent.read_state_canister_metadata(target, &name).await { - Ok(bytes) => Ok(Some(bytes)), - Err(AgentError::LookupPathAbsent(_)) => Ok(None), - Err(err) => Err(format!("metadata read failed: {err}")), + // A metadata path proven absent is equally what a canister that + // was never created looks like, and the certificate error names + // only the path asked for, so it cannot tell the two apart. + // Read `controllers` in the same request to disambiguate. + let metadata_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "metadata".into(), + name.as_str().into(), + ]; + let controllers_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "controllers".into(), + ]; + let cert = agent + .read_state_raw( + vec![metadata_path.clone(), controllers_path.clone()], + target, + ) + .await + .map_err(|err| format!("metadata read failed: {err}"))?; + + return match cert.tree.lookup_path(&metadata_path) { + LookupResult::Found(bytes) => Ok(Some(bytes.to_vec())), + // Creation writes `controllers`, so unlike `module_hash` it + // is present for a canister with no module installed — which + // has no sections at all, and so is a genuine `Ok(None)`. + LookupResult::Absent => match cert.tree.lookup_path(&controllers_path) { + LookupResult::Found(_) => Ok(None), + LookupResult::Absent => Err(format!("canister {target} does not exist")), + _ => Err(format!( + "metadata read failed: certificate proves nothing about \ + canister {target}" + )), + }, + // Not proof of absence, just a certificate that says nothing + // about the path — reporting the section missing off this + // would be a guess. + _ => Err(format!( + "metadata read failed: certificate proves nothing about section \ + `{name}` of canister {target}" + )), }; }; diff --git a/examples/icp-sync-plugin/icp.yaml b/examples/icp-sync-plugin/icp.yaml index 134cfd827..73a5e6dba 100644 --- a/examples/icp-sync-plugin/icp.yaml +++ b/examples/icp-sync-plugin/icp.yaml @@ -10,7 +10,7 @@ canisters: - type: script commands: - command -v ic-wasm >/dev/null 2>&1 || { echo >&2 "ic-wasm not found. To install ic-wasm, see https://github.com/dfinity/ic-wasm\n"; exit 1; } - - ic-wasm "$ICP_WASM_OUTPUT_PATH" -o "$ICP_WASM_OUTPUT_PATH" metadata candid:service -f demo.did --keep-name-section + - ic-wasm "$ICP_WASM_OUTPUT_PATH" -o "$ICP_WASM_OUTPUT_PATH" metadata candid:service -f demo.did -v public --keep-name-section sync: steps: From 21d9eb8efbd7f5baafbf49920d0b2a4b38c0d8e5 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 24 Aug 2026 12:51:48 -0700 Subject: [PATCH 22/51] improvements to proxy consistency --- crates/icp-sync-plugin/DESIGN.md | 14 +- crates/icp-sync-plugin/src/runtime.rs | 207 +++++++++++++++++-------- crates/icp-sync-plugin/sync-plugin.wit | 8 +- docs/concepts/sync-plugins.md | 4 +- docs/guides/writing-sync-plugins.md | 3 +- 5 files changed, 162 insertions(+), 74 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index cfc2b31fd..03e638278 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -167,13 +167,13 @@ exactly as `canister-call` chooses one: the same shape the CLI's own management calls take through `update_or_proxy_raw`; the runtime inlines it rather than depending on the CLI. -The routes disagree on how absence arrives: the management canister *rejects* -("The canister `` has no metadata section with the name ``.") and the -proxy hands the plugin a reason string with no reject code attached, so the host -matches `NO_SUCH_SECTION_REJECT` against it to produce the same `Ok(None)` a -direct read proves. Matching replica text is the price of one uniform contract; -it fails in the safe direction — a reword upstream turns absence back into an -error rather than into a wrong answer. +Only a certificate can make a read `none`. The management canister answers a +section that isn't there and one private to someone else with the same +rejection, so the proxied route treats that rejection as a claim to check rather +than an answer, and confirms it with a certified read before reporting absence. +A plugin then sees one answer either way: no section by that name and no module +installed at all are `none`; a private section it may not have, a canister that +does not exist, and any other failure are errors. ### Interface versioning (parallel v0.1.0 / v0.2.0 support) diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 2b4d4be7d..c571ed2c6 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -96,15 +96,86 @@ pub struct CallableCanisters { pub by_name: BTreeMap, } -/// The distinguishing phrase in the management canister's rejection of a -/// metadata read for a section the target does not have ("The canister has -/// no metadata section with the name ."). A proxied read reaches the -/// plugin as reject text, not as a code, so recognizing absence — which -/// [`HostState::do_canister_metadata_section`] reports as `Ok(None)`, matching what -/// a direct read proves from the certificate — means matching that text. A -/// reword upstream turns absence back into an error rather than into a wrong -/// answer. -const NO_SUCH_SECTION_REJECT: &str = "no metadata section"; +/// What a certificate says about a metadata section. A section the reader may +/// not have is neither of these: the state tree will not certify it, so it +/// reaches the caller as an error like any other failed read. +enum CertifiedSection { + Present(Vec), + Absent, +} + +/// Ask the target's subnet to certify a metadata section, reporting only what +/// the certificate proves. +/// +/// The section path is requested together with `controllers`, because a +/// metadata path proven absent is equally what a canister that was never created +/// looks like — `controllers` is written at creation, so its presence is what +/// separates the two. A canister with no module installed has no sections at +/// all, which the certificate reports as an absent path under a canister that +/// exists, and so as [`CertifiedSection::Absent`]. +async fn certified_metadata_section( + agent: &Agent, + target: Principal, + name: &str, +) -> Result { + let metadata_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "metadata".into(), + name.into(), + ]; + let controllers_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "controllers".into(), + ]; + let cert = agent + .read_state_raw( + vec![metadata_path.clone(), controllers_path.clone()], + target, + ) + .await + .map_err(|err| format!("metadata read failed: {err}"))?; + + match cert.tree.lookup_path(&metadata_path) { + LookupResult::Found(bytes) => Ok(CertifiedSection::Present(bytes.to_vec())), + LookupResult::Absent => match cert.tree.lookup_path(&controllers_path) { + LookupResult::Found(_) => Ok(CertifiedSection::Absent), + LookupResult::Absent => Err(format!("canister {target} does not exist")), + _ => Err(format!( + "metadata read failed: certificate proves nothing about canister {target}" + )), + }, + // Not proof of absence, just a certificate that says nothing about the + // path — reporting the section missing off this would be a guess. + _ => Err(format!( + "metadata read failed: certificate proves nothing about section `{name}` \ + of canister {target}" + )), + } +} + +/// Whether the management canister rejected a metadata read by claiming the +/// target has no such section, rather than because the read itself failed. +/// +/// The claim is not proof: the same rejection covers a section private to +/// someone other than the proxy, so the caller confirms it against a +/// certificate. A proxied read reaches the plugin as reject text with no code +/// attached, so recognizing the claim at all means matching the replica's +/// wording. Both sentences name the canister and one names the section, so the +/// match is anchored on the values this call supplied rather than on a loose +/// phrase that text relayed from elsewhere might happen to contain. A reword +/// upstream turns the claim into an error rather than into a wrong answer. +fn rejected_as_no_such_section(message: &str, target: Principal, name: &str) -> bool { + // A canister with no module installed has no sections at all, so it reports + // absence in its own words. The certificate says the same thing about it: + // the metadata path is absent while the canister itself is there. + message.contains(&format!( + "The canister {target} has no Wasm module and hence no metadata is available." + )) || message.contains(&format!( + "The canister {target} has no metadata section with the name {name}." + )) +} /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing /// that the plugin listed it in `canisters`. The canister being synced (`host`) @@ -218,16 +289,19 @@ impl HostState { } /// Read a metadata section from an already-resolved target principal. - /// `Ok(None)` is the target reporting it has no such section, kept distinct - /// from a failed read so a plugin can probe for an optional section without - /// inspecting error text (see [`NO_SUCH_SECTION_REJECT`]). + /// `Ok(None)` means a certificate proved the target has no such section, + /// kept distinct from a failed read so a plugin can probe for an optional + /// section without inspecting error text. A section the reader may not have + /// is a failed read, not an absent one, whichever route asked. /// /// A direct read is a certified `read_state` signed by the sync identity — /// `read_state` is not a canister method, so it cannot be forwarded. A /// proxied read therefore goes the other way around: the proxy calls the /// management canister's `canister_metadata` on the plugin's behalf, which /// checks the *proxy* against the target's controllers and so reaches - /// sections private to it. + /// sections private to it. The management canister does not distinguish + /// absence from privacy, so a proxied read that comes back claiming absence + /// is confirmed against a certificate before it is reported as one. fn do_canister_metadata_section( &mut self, target: Principal, @@ -240,55 +314,17 @@ impl HostState { let start = Instant::now(); let result = tokio::runtime::Handle::current().block_on(async move { let Some(proxy_cid) = proxy else { - // A metadata path proven absent is equally what a canister that - // was never created looks like, and the certificate error names - // only the path asked for, so it cannot tell the two apart. - // Read `controllers` in the same request to disambiguate. - let metadata_path: Vec>> = vec![ - "canister".into(), - Label::from_bytes(target.as_slice()), - "metadata".into(), - name.as_str().into(), - ]; - let controllers_path: Vec>> = vec![ - "canister".into(), - Label::from_bytes(target.as_slice()), - "controllers".into(), - ]; - let cert = agent - .read_state_raw( - vec![metadata_path.clone(), controllers_path.clone()], - target, - ) + return certified_metadata_section(&agent, target, &name) .await - .map_err(|err| format!("metadata read failed: {err}"))?; - - return match cert.tree.lookup_path(&metadata_path) { - LookupResult::Found(bytes) => Ok(Some(bytes.to_vec())), - // Creation writes `controllers`, so unlike `module_hash` it - // is present for a canister with no module installed — which - // has no sections at all, and so is a genuine `Ok(None)`. - LookupResult::Absent => match cert.tree.lookup_path(&controllers_path) { - LookupResult::Found(_) => Ok(None), - LookupResult::Absent => Err(format!("canister {target} does not exist")), - _ => Err(format!( - "metadata read failed: certificate proves nothing about \ - canister {target}" - )), - }, - // Not proof of absence, just a certificate that says nothing - // about the path — reporting the section missing off this - // would be a guess. - _ => Err(format!( - "metadata read failed: certificate proves nothing about section \ - `{name}` of canister {target}" - )), - }; + .map(|section| match section { + CertifiedSection::Present(bytes) => Some(bytes), + CertifiedSection::Absent => None, + }); }; let metadata_args = Encode!(&CanisterMetadataArgs { canister_id: target, - name, + name: name.clone(), }) .map_err(|e| format!("metadata encode failed: {e}"))?; let proxy_args = ProxyArgs { @@ -313,10 +349,19 @@ impl HostState { } ProxyResult::Err(err) => { let message = err.format_error(); - if message.contains(NO_SUCH_SECTION_REJECT) { - Ok(None) - } else { - Err(message) + if !rejected_as_no_such_section(&message, target, &name) { + return Err(format!("metadata read failed: {message}")); + } + // The management canister says the same thing about a + // section that isn't there and one that is private to + // someone else, so its word alone cannot be reported as + // absence. Only a certificate proves the section absent. + match certified_metadata_section(&agent, target, &name).await? { + CertifiedSection::Absent => Ok(None), + CertifiedSection::Present(_) => Err(format!( + "metadata read failed: canister {target} does not let the proxy \ + read section `{name}`" + )), } } } @@ -1226,6 +1271,46 @@ mod tests { ); } + /// The replica's own wording for the two ways a target reports it has no + /// section, copied from `CanisterManagerError` in the IC repo. Both are + /// absence, not failure, so both must reach the plugin as `none`. + #[test] + fn management_canister_absence_rejects_are_recognized() { + let target = Principal::from_text("aaaaa-aa").unwrap(); + let other = Principal::from_text("2vxsx-fae").unwrap(); + + let no_module = format!( + "Proxy call failed: The canister {target} has no Wasm module and hence no metadata is available." + ); + let no_section = format!( + "Proxy call failed: The canister {target} has no metadata section with the name candid:service." + ); + assert!(rejected_as_no_such_section( + &no_module, + target, + "candid:service" + )); + assert!(rejected_as_no_such_section( + &no_section, + target, + "candid:service" + )); + + // A section by another name, a canister other than the one asked about, + // and an unrelated failure are all reads that failed. + assert!(!rejected_as_no_such_section(&no_section, target, "dfx")); + assert!(!rejected_as_no_such_section( + &no_module, + other, + "candid:service" + )); + assert!(!rejected_as_no_such_section( + &format!("Proxy call failed: Canister {target} not found."), + target, + "candid:service" + )); + } + #[test] fn plugin_exceeding_compute_limit_is_trapped() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 27066621b..cd44a694f 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -183,9 +183,11 @@ world sync-plugin { /// identity; a proxied read (`direct` false, with `--proxy` configured) is /// a call to the management canister's `canister_metadata` method made by /// the proxy, which reaches sections private to the proxy's control. - /// Returns the section's raw bytes on success, `none` when the target - /// reports it has no section by that name, or an error message on failure. - /// The plugin is responsible for interpreting the bytes. + /// Returns the section's raw bytes on success, or `none` when the target + /// provably has no section by that name — including when it has no module + /// installed at all, and so no sections. A section the reader may not have + /// is an error, as is a canister that does not exist or any other failed + /// read. The plugin is responsible for interpreting the bytes. import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; // The plugin's stdout is captured and shown as transient progress in diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 6983dd480..14d1c5362 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -107,9 +107,9 @@ The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canist | `name` | The section name, without the `icp:public `/`icp:private ` prefix the wasm custom section carries (e.g. `candid:service`) | | `direct` | When `false` (default), the read is routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, it always goes straight to the target | -A successful read returns the section's raw bytes, or **absent** when the target reports it has no section by that name — so a plugin can probe for an optional section without matching on error text. Anything else (an unreachable canister, a section the caller may not read) is an error. +A successful read returns the section's raw bytes, or **absent** when the target provably has no section by that name — including when it has no module installed at all — so a plugin can probe for an optional section without matching on error text. Everything else is an error: a section the reader may not have, a canister that does not exist, a read that fails. -The two routes differ in who the target sees asking, which decides what a **private** section will yield: +The two routes differ in who the target sees asking, which decides whether a **private** section reads as its bytes or as an error: - **Direct** — a certified `read_state` request signed by the sync identity. A private section requires that identity to control the target. - **Proxied** — a call to the management canister's `canister_metadata` method made by the proxy, because `read_state` is not a canister method and cannot be forwarded. A private section requires the *proxy* to control the target — the same arrangement proxied update calls rely on. diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 64bf19657..5fad2280f 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -101,7 +101,8 @@ let interface = canister_metadata_section(&MetadataSectionRequest { match interface { Some(bytes) => println!("interface: {}", String::from_utf8_lossy(&bytes)), - // `None` means the canister has no such section — not a failure. + // `None` means the canister provably has no such section — not a failure. + // A section you may not read, or a canister that does not exist, is an error. None => println!("canister exposes no Candid interface"), } ``` From 593fafb48490758ec7016bcd79a20dd12e26d1d8 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Tue, 25 Aug 2026 08:52:03 -0700 Subject: [PATCH 23/51] refactor: extract the project model and deploy orchestration into icp-deploy-canister MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the `Project` model, manifest types and consolidation, network config, parsers, and the install/sync/deploy orchestration out of `icp` and into a new `icp-deploy-canister` crate that builds for `wasm32-unknown-unknown`. `icp` keeps the host-only halves and re-exports the model, so the dependency direction is `icp-cli -> icp -> icp-deploy-canister` and the library never names `icp`. Host IO reaches the library through injected traits: `files::FileAccess`, `icp_access::IcpAccess` (a dumb byte transport — the library owns all Candid coding and proxy routing), `ids::IdStore`, `sync_exec::{PluginExecutor, ScriptRunner}`, and `canister::recipe::RemoteResourceResolve`. Streamed step output crosses the boundary through the `sync_exec::StepProgress` sink rather than a channel type, so no `tokio` dependency is needed. Sync-step *derivation* moves with the model. `sync_exec` now dispatches on the step kind and resolves each step into a `PluginInvocation` / `ScriptInvocation` — the key-tagged `dirs`/`files`, the inline `fields`, the exposed canister-id table, the resolved `canisters:` call list, and the `ICP_CLI_*` script environment. The host implementations only perform the irreducible host action: fetch-and-run-the-wasm, or spawn-the-subprocess. Recipe handling splits along the same line: rendering (handlebars) moves into the library, while fetching stays in `icp::canister::recipe::resolver`, which implements `RemoteResourceResolve`. The staged fetch/render/commit protocol is preserved — an unpinned download is held back until the template is known to render, and a checksummed one is cached during the fetch. --- Cargo.lock | 39 +- Cargo.toml | 1 + crates/icp-cli/Cargo.toml | 1 + .../icp-cli/src/commands/canister/install.rs | 23 +- crates/icp-cli/src/commands/deploy.rs | 4 +- crates/icp-cli/src/commands/sync.rs | 25 +- crates/icp-cli/src/operations/access.rs | 131 + .../src/operations/binding_env_vars.rs | 110 +- crates/icp-cli/src/operations/install.rs | 311 +-- crates/icp-cli/src/operations/mod.rs | 1 + .../src/operations/proxy_management.rs | 66 +- crates/icp-cli/src/operations/sync.rs | 179 +- crates/icp-deploy-canister/Cargo.toml | 51 + .../icp-deploy-canister/src/canister/mod.rs | 684 +++++ .../src/canister/recipe/mod.rs} | 104 +- crates/icp-deploy-canister/src/deploy.rs | 1051 ++++++++ crates/icp-deploy-canister/src/files.rs | 50 + crates/icp-deploy-canister/src/icp_access.rs | 60 + crates/icp-deploy-canister/src/ids.rs | 42 + crates/icp-deploy-canister/src/lib.rs | 204 ++ .../src/manifest/adapter/mod.rs | 0 .../src/manifest/adapter/plugin.rs | 0 .../src/manifest/adapter/prebuilt.rs | 0 .../src/manifest/adapter/script.rs | 0 .../src/manifest/canister.rs | 0 .../src/manifest/dependency.rs | 0 .../src/manifest/environment.rs | 0 .../icp-deploy-canister/src/manifest/mod.rs | 132 + .../src/manifest/network.rs | 0 .../src/manifest/project.rs | 0 .../src/manifest/recipe.rs | 0 .../src/manifest/serde_helpers.rs | 0 crates/icp-deploy-canister/src/network/mod.rs | 368 +++ crates/icp-deploy-canister/src/parsers.rs | 643 +++++ crates/icp-deploy-canister/src/prelude.rs | 13 + crates/icp-deploy-canister/src/project.rs | 2340 +++++++++++++++++ crates/icp-deploy-canister/src/sync_exec.rs | 472 ++++ crates/icp-deploy-canister/src/testutil.rs | 63 + crates/icp/Cargo.toml | 7 +- crates/icp/src/canister/build/prebuilt.rs | 10 +- crates/icp/src/canister/mod.rs | 702 +---- crates/icp/src/canister/recipe/mod.rs | 68 +- .../canister/recipe/{fetch.rs => resolver.rs} | 269 +- crates/icp/src/canister/script.rs | 4 +- crates/icp/src/canister/sync/mod.rs | 212 +- crates/icp/src/canister/sync/plugin.rs | 344 +-- crates/icp/src/canister/sync/script.rs | 289 -- crates/icp/src/canister/wasm.rs | 24 +- crates/icp/src/context/init.rs | 6 +- crates/icp/src/context/mod.rs | 14 + crates/icp/src/host_files.rs | 64 + crates/icp/src/lib.rs | 227 +- crates/icp/src/manifest/mod.rs | 115 +- crates/icp/src/network/mod.rs | 370 +-- crates/icp/src/parsers.rs | 646 +---- crates/icp/src/project.rs | 2282 +--------------- 56 files changed, 7219 insertions(+), 5602 deletions(-) create mode 100644 crates/icp-cli/src/operations/access.rs create mode 100644 crates/icp-deploy-canister/Cargo.toml create mode 100644 crates/icp-deploy-canister/src/canister/mod.rs rename crates/{icp/src/canister/recipe/render.rs => icp-deploy-canister/src/canister/recipe/mod.rs} (68%) create mode 100644 crates/icp-deploy-canister/src/deploy.rs create mode 100644 crates/icp-deploy-canister/src/files.rs create mode 100644 crates/icp-deploy-canister/src/icp_access.rs create mode 100644 crates/icp-deploy-canister/src/ids.rs create mode 100644 crates/icp-deploy-canister/src/lib.rs rename crates/{icp => icp-deploy-canister}/src/manifest/adapter/mod.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/adapter/plugin.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/adapter/prebuilt.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/adapter/script.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/canister.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/dependency.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/environment.rs (100%) create mode 100644 crates/icp-deploy-canister/src/manifest/mod.rs rename crates/{icp => icp-deploy-canister}/src/manifest/network.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/project.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/recipe.rs (100%) rename crates/{icp => icp-deploy-canister}/src/manifest/serde_helpers.rs (100%) create mode 100644 crates/icp-deploy-canister/src/network/mod.rs create mode 100644 crates/icp-deploy-canister/src/parsers.rs create mode 100644 crates/icp-deploy-canister/src/prelude.rs create mode 100644 crates/icp-deploy-canister/src/project.rs create mode 100644 crates/icp-deploy-canister/src/sync_exec.rs create mode 100644 crates/icp-deploy-canister/src/testutil.rs rename crates/icp/src/canister/recipe/{fetch.rs => resolver.rs} (69%) delete mode 100644 crates/icp/src/canister/sync/script.rs create mode 100644 crates/icp/src/host_files.rs diff --git a/Cargo.lock b/Cargo.lock index bd140f275..5a93a621b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3625,7 +3625,6 @@ dependencies = [ "flate2", "futures", "glob", - "handlebars", "hex", "hmac 0.13.0", "httptest", @@ -3637,6 +3636,7 @@ dependencies = [ "ic-management-canister-types 0.8.0", "ic-utils", "icp-canister-interfaces", + "icp-deploy-canister", "icp-sync-plugin", "icrc-ledger-types", "indexmap", @@ -3728,6 +3728,7 @@ dependencies = [ "ic-utils", "icp", "icp-canister-interfaces", + "icp-deploy-canister", "icp-sync-plugin", "icrc-ledger-types", "indicatif", @@ -3773,6 +3774,42 @@ dependencies = [ "wslpath2", ] +[[package]] +name = "icp-deploy-canister" +version = "1.3.0" +dependencies = [ + "async-trait", + "bigdecimal", + "camino", + "camino-tempfile", + "candid", + "candid_parser", + "clap", + "futures", + "glob", + "handlebars", + "hex", + "ic-management-canister-types 0.8.0", + "indexmap", + "indoc", + "itertools 0.14.0", + "jsonschema", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "pathdiff", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "sha2 0.11.0", + "snafu", + "strum 0.28.0", + "tokio", + "tracing", + "url", +] + [[package]] name = "icp-sync-plugin" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 60dc41e1b..12840d4f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ ic-management-canister-types = { version = "0.8.0" } ic-utils = { version = "0.49.1" } icp = { path = "crates/icp" } icp-canister-interfaces = { path = "crates/icp-canister-interfaces" } +icp-deploy-canister = { path = "crates/icp-deploy-canister" } icp-sync-plugin = { path = "crates/icp-sync-plugin" } ic-identity-hsm = "0.49.1" icrc-ledger-types = "0.1.10" diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index 7e326084c..8ff4393da 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -44,6 +44,7 @@ ic-management-canister-types.workspace = true ic-utils.workspace = true icp-canister-interfaces.workspace = true icp = { workspace = true, features = ["clap"] } +icp-deploy-canister.workspace = true icp-sync-plugin.workspace = true icrc-ledger-types.workspace = true indicatif.workspace = true diff --git a/crates/icp-cli/src/commands/canister/install.rs b/crates/icp-cli/src/commands/canister/install.rs index a55fe5451..f44ae0246 100644 --- a/crates/icp-cli/src/commands/canister/install.rs +++ b/crates/icp-cli/src/commands/canister/install.rs @@ -10,14 +10,14 @@ use icp::fs; use icp::prelude::*; use tracing::{info, warn}; +use icp_deploy_canister::install_canister_wasm; + use crate::{ commands::args::{self, ArgsOpt}, operations::{ + access::AgentIcpAccess, candid_compat::{CandidCompatibility, check_candid_compatibility}, - install::{ - WasmMemoryPersistenceOpt, install_canister, is_eop_canister, - resolve_install_mode_and_status, - }, + install::{WasmMemoryPersistenceOpt, is_eop_canister, resolve_install_mode_and_status}, }, }; @@ -184,16 +184,21 @@ pub(crate) async fn exec(ctx: &Context, args: &InstallArgs) -> Result<(), anyhow } } - install_canister( - &agent, - args.proxy, - &canister_id, + let icp = AgentIcpAccess::new(agent.clone(), args.proxy); + let wmp = args + .wasm_memory_persistence + .map(WasmMemoryPersistenceOpt::to_ic); + // Install the bytes read above, not a fresh read of the same source, so the + // module installed is the one the Candid check ran against. + install_canister_wasm( &canister_display, + canister_id, &wasm, install_mode, status, init_args_bytes.as_deref(), - args.wasm_memory_persistence, + wmp, + &icp, ) .await?; diff --git a/crates/icp-cli/src/commands/deploy.rs b/crates/icp-cli/src/commands/deploy.rs index b2a1fe4ca..912596e98 100644 --- a/crates/icp-cli/src/commands/deploy.rs +++ b/crates/icp-cli/src/commands/deploy.rs @@ -489,9 +489,10 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: .into_iter() .collect(); - let pkg_cache = ctx.dirs.package_cache()?; + let resolver = ctx.resource_resolver()?; sync_many( ctx.syncer.clone(), + resolver, agent.clone(), sync_canisters, environment_selection.name().to_owned(), @@ -499,7 +500,6 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: canister_ids, args.proxy, ctx.debug, - &pkg_cache, ) .await?; } diff --git a/crates/icp-cli/src/commands/sync.rs b/crates/icp-cli/src/commands/sync.rs index ac2d1e099..567adefbe 100644 --- a/crates/icp-cli/src/commands/sync.rs +++ b/crates/icp-cli/src/commands/sync.rs @@ -9,8 +9,10 @@ use icp::identity::IdentitySelection; use std::collections::BTreeMap; use tracing::info; +use icp::Canister; + use crate::{ - operations::{proxy_management, sync::sync_many}, + operations::{binding_env_vars::set_binding_env_vars_many, proxy_management, sync::sync_many}, options::{EnvironmentOpt, IdentityOpt}, }; @@ -124,9 +126,27 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E .into_iter() .collect(); - let pkg_cache = ctx.dirs.package_cache()?; + // Apply the generated `PUBLIC_CANISTER_ID:*` environment variables before + // syncing. `deploy` does this, but standalone `icp sync` previously did not, + // so a synced canister could run against stale/absent binding ids. + let target_canisters: Vec<(Principal, Canister)> = sync_canisters + .iter() + .map(|(cid, _, info)| (*cid, info.clone())) + .collect(); + set_binding_env_vars_many( + agent.clone(), + args.proxy, + environment_selection.name(), + target_canisters, + canister_ids.clone(), + ctx.debug, + ) + .await?; + + let resolver = ctx.resource_resolver()?; sync_many( ctx.syncer.clone(), + resolver, agent, sync_canisters, environment_selection.name().to_owned(), @@ -134,7 +154,6 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E canister_ids, args.proxy, ctx.debug, - &pkg_cache, ) .await?; diff --git a/crates/icp-cli/src/operations/access.rs b/crates/icp-cli/src/operations/access.rs new file mode 100644 index 000000000..993074caf --- /dev/null +++ b/crates/icp-cli/src/operations/access.rs @@ -0,0 +1,131 @@ +//! Host implementations of the `icp-deploy-canister` IO traits, backing the +//! library's install/sync/deploy core with the CLI's `ic-agent` transport and +//! on-disk stores. + +use std::sync::Arc; + +use async_trait::async_trait; +use candid::Principal; +use icp::prelude::*; +use icp::store_artifact; +use icp_deploy_canister::files::{FileAccess, FileAccessError}; +use icp_deploy_canister::icp_access::{IcpAccess, IcpAccessError}; + +use super::proxy::update_or_proxy_raw; + +/// [`IcpAccess`] over an `ic-agent` `Agent`. Proxy routing is baked in (the +/// impl is constructed with the proxy principal); the library never threads a +/// proxy per call. The caller's principal is captured once at construction. +pub struct AgentIcpAccess { + agent: ic_agent::Agent, + proxy: Option, + caller: Principal, +} + +impl AgentIcpAccess { + pub fn new(agent: ic_agent::Agent, proxy: Option) -> Self { + let caller = agent + .get_principal() + .unwrap_or_else(|_| Principal::anonymous()); + Self { + agent, + proxy, + caller, + } + } +} + +#[async_trait] +impl IcpAccess for AgentIcpAccess { + async fn canister_update( + &self, + canister: Principal, + method: &str, + arg: Vec, + effective_canister_id: Principal, + cycles: u128, + ) -> Result, IcpAccessError> { + update_or_proxy_raw( + &self.agent, + canister, + method, + arg, + self.proxy, + Some(effective_canister_id), + cycles, + ) + .await + .map_err(|e| IcpAccessError::Update { + canister, + method: method.to_owned(), + message: e.to_string(), + }) + } + + async fn read_canister_metadata( + &self, + canister: Principal, + path: &str, + ) -> Result>, IcpAccessError> { + // A read failure is treated as "metadata absent" (matching the previous + // EOP-detection behavior), so a missing custom section never aborts an + // install. + Ok(self + .agent + .read_state_canister_metadata(canister, path) + .await + .ok()) + } + + fn caller_principal(&self) -> Principal { + self.caller + } +} + +/// [`FileAccess`] backed by the canister build-artifact store. The library reads +/// a canister's built wasm via `read_file(artifact_path)`; here the "path" is the +/// canister's store key, resolved through the (locked) artifact store. Only +/// `read_file` is used by the install path; the other methods have benign +/// defaults. +pub struct ArtifactFileAccess(pub Arc); + +#[async_trait] +impl FileAccess for ArtifactFileAccess { + async fn read_file(&self, path: &Path) -> Result, FileAccessError> { + self.0 + .lookup(path.as_str()) + .await + .map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn read_to_string(&self, path: &Path) -> Result { + let bytes = self.read_file(path).await?; + String::from_utf8(bytes).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn exists(&self, path: &Path) -> bool { + self.0.lookup(path.as_str()).await.is_ok() + } + + async fn is_file(&self, path: &Path) -> bool { + self.exists(path).await + } + + async fn is_dir(&self, _path: &Path) -> bool { + false + } + + async fn read_dir(&self, _path: &Path) -> Result, FileAccessError> { + Ok(Vec::new()) + } + + async fn canonicalize(&self, path: &Path) -> Option { + Some(path.to_owned()) + } +} diff --git a/crates/icp-cli/src/operations/binding_env_vars.rs b/crates/icp-cli/src/operations/binding_env_vars.rs index a2118f193..41c21e07a 100644 --- a/crates/icp-cli/src/operations/binding_env_vars.rs +++ b/crates/icp-cli/src/operations/binding_env_vars.rs @@ -1,29 +1,16 @@ use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::{Agent, export::Principal}; -use ic_management_canister_types::{CanisterSettings, EnvironmentVariable, UpdateSettingsArgs}; use icp::Canister; +use icp_deploy_canister::{SyncCanisterError, apply_binding_env_vars}; use snafu::Snafu; use tracing::error; +use crate::operations::access::AgentIcpAccess; use crate::progress::{ProgressManager, ProgressManagerSettings}; -use super::proxy::UpdateOrProxyError; -use super::proxy_management; - -#[derive(Debug, Snafu)] -pub enum BindingEnvVarsOperationError { - #[snafu(display("Could not find canister id(s) for {} in environment '{environment}'. Make sure they are created first", canister_names.join(", ")))] - CanisterNotCreated { - environment: String, - canister_names: Vec, - }, - - #[snafu(transparent)] - UpdateOrProxy { source: UpdateOrProxyError }, -} - #[derive(Debug, Snafu)] #[snafu(display("Canister(s) {names:?} failed to update environment variables."))] pub struct SetBindingEnvVarsManyError { @@ -34,50 +21,15 @@ pub struct SetBindingEnvVarsManyError { struct BindingEnvVarsFailure { canister_name: String, canister_id: Principal, - error: BindingEnvVarsOperationError, -} - -pub(crate) async fn set_env_vars_for_canister( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_info: &Canister, - binding_vars: &[(String, String)], -) -> Result<(), BindingEnvVarsOperationError> { - let mut environment_variables = canister_info - .settings - .environment_variables - .to_owned() - .unwrap_or_default(); - - // inject the ids of the other canisters - for (k, v) in binding_vars.iter() { - environment_variables.insert(k.to_string(), v.to_string()); - } - - let environment_variables = environment_variables - .into_iter() - .map(|(name, value)| EnvironmentVariable { name, value }) - .collect::>(); - - proxy_management::update_settings( - agent, - proxy, - UpdateSettingsArgs { - canister_id: *canister_id, - settings: CanisterSettings { - environment_variables: Some(environment_variables), - ..Default::default() - }, - sender_canister_version: None, - }, - ) - .await?; - - Ok(()) + error: SyncCanisterError, } -/// Orchestrates setting environment variables for multiple canisters with progress tracking +/// Orchestrates setting environment variables for multiple canisters with progress tracking. +/// +/// The per-canister work (computing the generated `PUBLIC_CANISTER_ID:*` +/// bindings, merging with manifest env vars, and applying them) lives in +/// `icp_deploy_canister::apply_binding_env_vars`; this wrapper only adds the +/// missing-id precheck and progress display. pub(crate) async fn set_binding_env_vars_many( agent: Agent, proxy: Option, @@ -86,20 +38,14 @@ pub(crate) async fn set_binding_env_vars_many( canister_list: BTreeMap, debug: bool, ) -> Result<(), SetBindingEnvVarsManyError> { - // Check that all the canisters in this environment have an id - // We need to have all the ids to generate environment variables - // for the bindings + // Check that all the canisters in this environment have an id: we need all + // ids to generate the binding environment variables. let canisters_with_ids: HashSet<&String> = canister_list.keys().collect(); - let all_canister_names: Vec = target_canisters + let missing_canisters: Vec = target_canisters .iter() .map(|(_, info)| info.name.clone()) - .collect(); - - let missing_canisters: Vec = all_canister_names - .iter() - .filter(|c| !canisters_with_ids.contains(*c)) - .map(|c| c.to_string()) + .filter(|c| !canisters_with_ids.contains(c)) .collect(); if !missing_canisters.is_empty() { @@ -116,38 +62,23 @@ pub(crate) async fn set_binding_env_vars_many( .fail(); } + let icp = Arc::new(AgentIcpAccess::new(agent, proxy)); + let canister_list = Arc::new(canister_list); + let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); for (cid, info) in target_canisters { let pb = progress_manager.create_progress_bar(&info.name); let canister_name = info.name.clone(); - - // Each canister receives only the ids it is wired to (its own project's - // canisters by their local names, plus any declared dependencies under - // their aliases), resolved to the ids that exist in this environment. - // A project without dependencies wires every canister to every sibling, - // reproducing the previous flat behavior. - let binding_vars: Vec<(String, String)> = info - .bindings - .iter() - .filter_map(|(env_name, referenced_key)| { - canister_list.get(referenced_key).map(|principal| { - ( - format!("PUBLIC_CANISTER_ID:{env_name}"), - principal.to_text(), - ) - }) - }) - .collect(); + let icp = icp.clone(); + let canister_list = canister_list.clone(); let settings_fn = { - let agent = agent.clone(); let pb = pb.clone(); - async move { pb.set_message("Updating environment variables..."); - set_env_vars_for_canister(&agent, proxy, &cid, &info, &binding_vars).await + apply_binding_env_vars(&info, cid, &canister_list, icp.as_ref()).await } }; @@ -160,7 +91,6 @@ pub(crate) async fn set_binding_env_vars_many( ) .await; - // Map error to include canister context for deferred printing result.map_err(|error| BindingEnvVarsFailure { canister_name, canister_id: cid, diff --git a/crates/icp-cli/src/operations/install.rs b/crates/icp-cli/src/operations/install.rs index 24f90fd4e..f7b5190e7 100644 --- a/crates/icp-cli/src/operations/install.rs +++ b/crates/icp-cli/src/operations/install.rs @@ -1,16 +1,15 @@ -use candid::Encode; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::{Agent, export::Principal}; use ic_management_canister_types::{ - CanisterId, CanisterIdRecord, CanisterInstallMode, CanisterStatusType, ChunkHash, - ClearChunkStoreArgs, InstallChunkedCodeArgs, InstallCodeArgs, UpgradeFlags, UploadChunkArgs, - WasmMemoryPersistence, + CanisterId, CanisterIdRecord, CanisterInstallMode, CanisterStatusType, WasmMemoryPersistence, }; -use sha2::{Digest, Sha256}; +use icp::prelude::*; +use icp_deploy_canister::{InstallCanisterError, install_canister_resolved}; use snafu::{ResultExt, Snafu}; use std::sync::Arc; -use tracing::{debug, error, warn}; +use tracing::error; +use crate::operations::access::{AgentIcpAccess, ArtifactFileAccess}; use crate::progress::{ProgressManager, ProgressManagerSettings}; use super::misc::fetch_canister_metadata; @@ -28,7 +27,7 @@ pub enum WasmMemoryPersistenceOpt { } impl WasmMemoryPersistenceOpt { - fn to_ic(self) -> WasmMemoryPersistence { + pub(crate) fn to_ic(self) -> WasmMemoryPersistence { match self { WasmMemoryPersistenceOpt::Keep => WasmMemoryPersistence::Keep, WasmMemoryPersistenceOpt::Replace => WasmMemoryPersistence::Replace, @@ -44,43 +43,13 @@ pub(crate) async fn is_eop_canister(agent: &Agent, canister_id: &Principal) -> b .is_some() } -#[derive(Debug, Snafu)] -pub enum InstallOperationError { - #[snafu(display("Could not find build artifact for canister '{canister_name}'"))] - ArtifactNotFound { canister_name: String }, - - #[snafu(display("Failed to stop canister '{canister_name}' before upgrade"))] - StopCanister { - canister_name: String, - source: UpdateOrProxyError, - }, - - #[snafu(display("Failed to start canister '{canister_name}' after upgrade"))] - StartCanister { - canister_name: String, - source: UpdateOrProxyError, - }, - - #[snafu(transparent)] - UpdateOrProxy { source: UpdateOrProxyError }, -} - -#[derive(Debug, Snafu)] -#[snafu(display("Canister(s) {names:?} failed to install."))] -pub struct InstallManyError { - names: Vec, -} - -/// Holds error information from a failed canister install operation -struct InstallFailure { - canister_name: String, - canister_id: Principal, - error: InstallOperationError, -} - /// Resolve a mode string ("auto", "install", "reinstall", "upgrade") into /// a [`CanisterInstallMode`]. For "auto", queries `canister_status` to /// determine whether the canister already has code installed. +/// +/// Returns the resolved mode plus the current status; callers (deploy, the +/// candid-compat gate) need the resolved mode before installing, so resolution +/// happens here once and the result is handed to [`install_canister_resolved`]. pub(crate) async fn resolve_install_mode_and_status( agent: &Agent, proxy: Option, @@ -118,235 +87,48 @@ pub(crate) struct ResolveInstallModeError { source: UpdateOrProxyError, } -pub(crate) async fn install_canister( - agent: &Agent, - proxy: Option, +/// Install one canister whose build artifact lives in the store, addressed by +/// its store key `canister_name`. The install-code/chunking/EOP logic lives in +/// `icp_deploy_canister::install_canister_resolved`; this is a thin wrapper over +/// the artifact-backed `FileAccess` and agent-backed `IcpAccess`. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn install_stored_canister( + icp: &AgentIcpAccess, + files: &ArtifactFileAccess, canister_id: &Principal, canister_name: &str, - wasm: &[u8], mode: CanisterInstallMode, status: CanisterStatusType, init_args: Option<&[u8]>, wasm_memory_persistence: Option, -) -> Result<(), InstallOperationError> { - let mode = match mode { - CanisterInstallMode::Upgrade(_) => { - // if this is a motoko canister using EOP we need to set additional options. - // If the caller supplied an explicit override, trust it (the CLI layer has - // already validated that it's an EOP canister); otherwise auto-detect and - // default to Keep. - let persistence = match wasm_memory_persistence { - Some(opt) => Some(opt.to_ic()), - None => is_eop_canister(agent, canister_id) - .await - .then_some(WasmMemoryPersistence::Keep), - }; - if let Some(persistence) = persistence { - CanisterInstallMode::Upgrade(Some(UpgradeFlags { - skip_pre_upgrade: None, - wasm_memory_persistence: Some(persistence), - })) - } else { - mode - } - } - _ => mode, - }; - - debug!( - "Install new canister code for {} with mode `{:?}`", - canister_name, mode - ); - - do_install_operation( - agent, - proxy, - canister_id, +) -> Result<(), InstallCanisterError> { + install_canister_resolved( canister_name, - wasm, + *canister_id, + // The artifact `FileAccess` resolves the store key, so the "path" is the + // canister name. + Path::new(canister_name), mode, status, init_args, + wasm_memory_persistence.map(WasmMemoryPersistenceOpt::to_ic), + files, + icp, ) .await } -async fn do_install_operation( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_name: &str, - wasm: &[u8], - mode: CanisterInstallMode, - status: CanisterStatusType, - init_args: Option<&[u8]>, -) -> Result<(), InstallOperationError> { - // Threshold for chunked installation: 2 MB - // Raw install_code messages are limited to 2 MiB - const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024; - - // Chunk size: 1 MB (spec limit is 1 MiB per chunk) - const CHUNK_SIZE: usize = 1024 * 1024; - - // Generous overhead for encoding, target canister ID, install mode, etc. - const ENCODING_OVERHEAD: usize = 500; - - let cid = CanisterId::from(*canister_id); - let arg = init_args - .map(|a| a.to_vec()) - .unwrap_or_else(|| Encode!().unwrap()); - - // Calculate total install message size - let total_install_size = wasm.len() + arg.len() + ENCODING_OVERHEAD; - - if total_install_size <= CHUNK_THRESHOLD { - // Small wasm: use regular install_code - debug!("Installing wasm for {canister_name} using install_code"); - - let install_args = InstallCodeArgs { - mode, - canister_id: cid, - wasm_module: wasm.to_vec(), - arg, - sender_canister_version: None, - }; - - stop_and_start_if_upgrade( - agent, - proxy, - canister_id, - canister_name, - mode, - status, - async { - proxy_management::install_code(agent, proxy, install_args).await?; - Ok(()) - }, - ) - .await?; - } else { - // Large wasm: use chunked installation - debug!("Installing wasm for {canister_name} using chunked installation"); - - // Clear any existing chunks to ensure a clean state - proxy_management::clear_chunk_store(agent, proxy, ClearChunkStoreArgs { canister_id: cid }) - .await?; - - // Split wasm into chunks and upload them - let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect(); - let mut chunk_hashes: Vec = Vec::new(); - - for (i, chunk) in chunks.iter().enumerate() { - debug!( - "Uploading chunk {}/{} ({} bytes)", - i + 1, - chunks.len(), - chunk.len() - ); - - let upload_args = UploadChunkArgs { - canister_id: cid, - chunk: chunk.to_vec(), - }; - - let chunk_hash = proxy_management::upload_chunk(agent, proxy, upload_args).await?; - - chunk_hashes.push(chunk_hash); - } - - // Compute SHA-256 hash of the entire wasm module - let mut hasher = Sha256::new(); - hasher.update(wasm); - let wasm_module_hash = hasher.finalize().to_vec(); - - debug!("Installing chunked code with {} chunks", chunk_hashes.len()); - - let chunked_args = InstallChunkedCodeArgs { - mode, - target_canister: cid, - store_canister: None, - chunk_hashes_list: chunk_hashes, - wasm_module_hash, - arg, - sender_canister_version: None, - }; - - let install_res = stop_and_start_if_upgrade( - agent, - proxy, - canister_id, - canister_name, - mode, - status, - async { - proxy_management::install_chunked_code(agent, proxy, chunked_args).await?; - Ok(()) - }, - ) - .await; - - // Clear chunk store after successful installation to free up storage - let clear_res = proxy_management::clear_chunk_store( - agent, - proxy, - ClearChunkStoreArgs { canister_id: cid }, - ) - .await - .map_err(InstallOperationError::from); - - if let Err(clear_error) = clear_res { - if let Err(install_error) = install_res { - warn!("Failed to clear chunk store after failed install: {clear_error}"); - return Err(install_error); - } else { - return Err(clear_error); - } - } - install_res?; - } - - Ok(()) +#[derive(Debug, Snafu)] +#[snafu(display("Canister(s) {names:?} failed to install."))] +pub struct InstallManyError { + names: Vec, } -async fn stop_and_start_if_upgrade( - agent: &Agent, - proxy: Option, - canister_id: &Principal, - canister_name: &str, - mode: CanisterInstallMode, - status: CanisterStatusType, - f: impl Future>, -) -> Result<(), InstallOperationError> { - let should_guard = matches!( - mode, - CanisterInstallMode::Upgrade(_) | CanisterInstallMode::Reinstall - ) && matches!(status, CanisterStatusType::Running); - let cid_record = CanisterIdRecord { - canister_id: CanisterId::from(*canister_id), - }; - // Stop the canister before proceeding - if should_guard { - proxy_management::stop_canister(agent, proxy, cid_record.clone()) - .await - .context(StopCanisterSnafu { canister_name })?; - } - // Install the canister - let install_result = f.await; - // Restart the canister whether or not the installation succeeded - if should_guard { - let start_result = proxy_management::start_canister(agent, proxy, cid_record).await; - if let Err(start_error) = start_result { - // If both install and start failed, report the install error since it's more likely to be the root cause - if let Err(install_error) = install_result { - warn!("Failed to start canister after failed upgrade: {start_error}"); - return Err(install_error); - } else { - return Err(start_error).context(StartCanisterSnafu { canister_name }); - } - } - } - - install_result +/// Holds error information from a failed canister install operation +struct InstallFailure { + canister_name: String, + canister_id: Principal, + error: InstallCanisterError, } /// Installs code to multiple canisters and displays progress bars. @@ -365,32 +147,27 @@ pub(crate) async fn install_many( artifacts: Arc, debug: bool, ) -> Result<(), InstallManyError> { + let icp = Arc::new(AgentIcpAccess::new(agent, proxy)); + let files = Arc::new(ArtifactFileAccess(artifacts)); + let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); for (name, cid, mode, status, init_args) in canisters { let pb = progress_manager.create_progress_bar(&name); - let agent = agent.clone(); + let icp = icp.clone(); + let files = files.clone(); let install_fn = { let pb = pb.clone(); - let artifacts = artifacts.clone(); let name = name.clone(); async move { pb.set_message("Installing..."); - - let wasm = artifacts.lookup(&name).await.map_err(|_| { - InstallOperationError::ArtifactNotFound { - canister_name: name.clone(), - } - })?; - - install_canister( - &agent, - proxy, + install_stored_canister( + &icp, + &files, &cid, &name, - &wasm, mode, status, init_args.as_deref(), diff --git a/crates/icp-cli/src/operations/mod.rs b/crates/icp-cli/src/operations/mod.rs index 5ce6546d0..be7d37100 100644 --- a/crates/icp-cli/src/operations/mod.rs +++ b/crates/icp-cli/src/operations/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod access; pub(crate) mod binding_env_vars; pub(crate) mod build; pub(crate) mod bundle; diff --git a/crates/icp-cli/src/operations/proxy_management.rs b/crates/icp-cli/src/operations/proxy_management.rs index ffc7cac70..43ec4aa52 100644 --- a/crates/icp-cli/src/operations/proxy_management.rs +++ b/crates/icp-cli/src/operations/proxy_management.rs @@ -1,15 +1,14 @@ use candid::Principal; use ic_agent::Agent; use ic_management_canister_types::{ - CanisterIdRecord, CanisterStatusResult, ClearChunkStoreArgs, CreateCanisterArgs, - DeleteCanisterArgs, DeleteCanisterSnapshotArgs, FetchCanisterLogsArgs, FetchCanisterLogsResult, - InstallChunkedCodeArgs, InstallCodeArgs, ListCanisterSnapshotsArgs, - ListCanisterSnapshotsResult, LoadCanisterSnapshotArgs, ReadCanisterSnapshotDataArgs, - ReadCanisterSnapshotDataResult, ReadCanisterSnapshotMetadataArgs, + CanisterIdRecord, CanisterStatusResult, CreateCanisterArgs, DeleteCanisterArgs, + DeleteCanisterSnapshotArgs, FetchCanisterLogsArgs, FetchCanisterLogsResult, InstallCodeArgs, + ListCanisterSnapshotsArgs, ListCanisterSnapshotsResult, LoadCanisterSnapshotArgs, + ReadCanisterSnapshotDataArgs, ReadCanisterSnapshotDataResult, ReadCanisterSnapshotMetadataArgs, ReadCanisterSnapshotMetadataResult, StartCanisterArgs, StopCanisterArgs, TakeCanisterSnapshotArgs, TakeCanisterSnapshotResult, UpdateSettingsArgs, UploadCanisterSnapshotDataArgs, UploadCanisterSnapshotMetadataArgs, - UploadCanisterSnapshotMetadataResult, UploadChunkArgs, UploadChunkResult, + UploadCanisterSnapshotMetadataResult, }; use snafu::{ResultExt, Snafu}; @@ -144,61 +143,6 @@ pub async fn install_code( .await } -pub async fn install_chunked_code( - agent: &Agent, - proxy: Option, - args: InstallChunkedCodeArgs, -) -> Result<(), UpdateOrProxyError> { - let effective = args.target_canister; - update_or_proxy::<_, ()>( - agent, - Principal::management_canister(), - "install_chunked_code", - (args,), - proxy, - Some(effective), - 0, - ) - .await -} - -pub async fn upload_chunk( - agent: &Agent, - proxy: Option, - args: UploadChunkArgs, -) -> Result { - let effective = args.canister_id; - let (result,): (UploadChunkResult,) = update_or_proxy( - agent, - Principal::management_canister(), - "upload_chunk", - (args,), - proxy, - Some(effective), - 0, - ) - .await?; - Ok(result) -} - -pub async fn clear_chunk_store( - agent: &Agent, - proxy: Option, - args: ClearChunkStoreArgs, -) -> Result<(), UpdateOrProxyError> { - let effective = args.canister_id; - update_or_proxy::<_, ()>( - agent, - Principal::management_canister(), - "clear_chunk_store", - (args,), - proxy, - Some(effective), - 0, - ) - .await -} - #[derive(Debug, Snafu)] pub enum FetchCanisterLogsError { #[snafu(display("failed to encode call arguments: {source}"))] diff --git a/crates/icp-cli/src/operations/sync.rs b/crates/icp-cli/src/operations/sync.rs index 77ea04174..146fcfde1 100644 --- a/crates/icp-cli/src/operations/sync.rs +++ b/crates/icp-cli/src/operations/sync.rs @@ -1,15 +1,23 @@ +use async_trait::async_trait; use candid::Principal; use futures::{StreamExt, stream::FuturesOrdered}; use ic_agent::Agent; use icp::{ Canister, - canister::sync::{Params, Synchronize, SynchronizeError}, - package::PackageCache, + canister::recipe::RemoteResourceResolve, + canister::sync::{Synchronize, SynchronizeError}, prelude::PathBuf, }; +use icp_deploy_canister::manifest::adapter::prebuilt::SourceField; +use icp_deploy_canister::sync_exec::{ + PluginExecutor, PluginExecutorError, PluginInvocation, ScriptInvocation, ScriptRunError, + ScriptRunner, StepProgress, +}; +use icp_deploy_canister::{SyncCanisterError, SyncStepContext, run_sync_steps}; use snafu::prelude::*; use std::collections::BTreeMap; use std::sync::Arc; +use tokio::sync::Mutex; use tracing::error; use crate::progress::{MultiStepProgressBar, ProgressManager, ProgressManagerSettings}; @@ -24,14 +32,106 @@ pub struct SyncOperationError { struct SyncFailure { canister_name: String, canister_id: Principal, - error: SynchronizeError, + error: SyncCanisterError, progress_output: Vec, } -/// Synchronizes a single canister using its configured sync steps +/// Per-canister mutable state guarded so the `&self` [`PluginExecutor`] can drive +/// the (mutable, sequential) progress bar. +struct SyncStepState<'a> { + pb: &'a mut MultiStepProgressBar, + /// 1-based index of the step about to run, for the progress header. + next: usize, +} + +/// Sync-step executor that runs a resolved step via the host [`Synchronize`] +/// implementation (WASI plugin / subprocess script) and frames it on the +/// canister's multi-step progress bar. The library owns the step loop and all +/// input derivation ([`run_sync_steps`]); this only performs the host action and +/// streams its output. +struct AgentSyncExecutor<'a> { + syncer: Arc, + agent: Agent, + resolver: Arc, + total: usize, + state: Mutex>, +} + +impl AgentSyncExecutor<'_> { + /// Frame a step on the shared progress bar: advance the counter, print the + /// header, run `f` against a fresh line sender, and close the step. Holding + /// the guard across `f` keeps steps framed sequentially on the shared bar. + async fn framed( + &self, + header: impl FnOnce(usize, usize) -> String, + f: F, + ) -> Result, SynchronizeError> + where + F: FnOnce(tokio::sync::mpsc::Sender) -> Fut, + Fut: Future, SynchronizeError>>, + { + let mut st = self.state.lock().await; + st.next += 1; + let header = header(st.next, self.total); + let tx = st.pb.begin_step(header); + let result = f(tx).await; + st.pb.end_step().await; + result + } +} + +#[async_trait] +impl PluginExecutor for AgentSyncExecutor<'_> { + async fn run_plugin( + &self, + invocation: PluginInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, PluginExecutorError> { + let src = match &invocation.source { + SourceField::Local(l) => format!("path: {}", l.path), + SourceField::Remote(r) => format!("url: {}", r.url), + }; + self.framed( + |n, total| format!("\nSyncing: plugin {src} {n} of {total}"), + |tx| async move { + self.syncer + .run_plugin(&invocation, &self.agent, Some(tx), self.resolver.as_ref()) + .await + }, + ) + .await + .map_err(|source| PluginExecutorError { + source: Box::new(source), + }) + } +} + +#[async_trait] +impl ScriptRunner for AgentSyncExecutor<'_> { + async fn run_script( + &self, + invocation: ScriptInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, ScriptRunError> { + let desc = invocation.commands.join("\n"); + self.framed( + |n, total| format!("\nSyncing: script {desc} {n} of {total}"), + |tx| async move { self.syncer.run_script(&invocation, Some(tx)).await }, + ) + .await + .map_err(|source| ScriptRunError { + source: Box::new(source), + }) + } +} + +/// Synchronize a single canister's steps through the library, framing progress +/// on `pb`. Environment variables are applied separately by the caller. +#[allow(clippy::too_many_arguments)] async fn sync_canister( - syncer: &Arc, - agent: &Agent, + syncer: Arc, + resolver: Arc, + agent: Agent, canister_path: PathBuf, canister_id: Principal, canister_info: &Canister, @@ -40,49 +140,31 @@ async fn sync_canister( canister_ids: &BTreeMap, proxy: Option, pb: &mut MultiStepProgressBar, - pkg_cache: &PackageCache, -) -> Result, SynchronizeError> { - let step_count = canister_info.sync.steps.len(); - let mut stderr_lines = Vec::new(); - - for (i, step) in canister_info.sync.steps.iter().enumerate() { - // Indicate to user the current step being executed - let current_step = i + 1; - let pb_hdr = format!("\nSyncing: {step} {current_step} of {step_count}"); - - let tx = pb.begin_step(pb_hdr); - - // Execute step - let sync_result = syncer - .sync( - step, - &Params { - path: canister_path.clone(), - cid: canister_id, - name: canister_info.name.clone(), - environment: environment.to_owned(), - network: network.to_owned(), - canister_ids: canister_ids.clone(), - proxy, - }, - agent, - Some(tx), - pkg_cache, - ) - .await; - - // Ensure background receiver drains all messages - pb.end_step().await; - - stderr_lines.extend(sync_result?); - } - - Ok(stderr_lines) +) -> Result, SyncCanisterError> { + let ctx = SyncStepContext { + canister_path, + canister_id, + canister_name: canister_info.name.clone(), + environment: environment.to_owned(), + network: network.to_owned(), + canister_ids: canister_ids.clone(), + proxy, + }; + let executor = AgentSyncExecutor { + syncer, + agent, + resolver, + total: canister_info.sync.steps.len(), + state: Mutex::new(SyncStepState { pb, next: 0 }), + }; + run_sync_steps(canister_info, &ctx, &executor, &executor, None).await } /// Orchestrates syncing multiple canisters with progress tracking +#[allow(clippy::too_many_arguments)] pub(crate) async fn sync_many( syncer: Arc, + resolver: Arc, agent: Agent, canisters: Vec<(Principal, PathBuf, Canister)>, environment: String, @@ -90,7 +172,6 @@ pub(crate) async fn sync_many( canister_ids: BTreeMap, proxy: Option, debug: bool, - pkg_cache: &PackageCache, ) -> Result<(), SyncOperationError> { let mut futs = FuturesOrdered::new(); let progress_manager = ProgressManager::new(ProgressManagerSettings { hidden: debug }); @@ -101,15 +182,16 @@ pub(crate) async fn sync_many( let fut = { let agent = agent.clone(); let syncer = syncer.clone(); + let resolver = resolver.clone(); let environment = environment.clone(); let network = network.clone(); let canister_ids = canister_ids.clone(); async move { - // Define the sync logic let sync_result = sync_canister( - &syncer, - &agent, + syncer, + resolver, + agent, canister_path, cid, &canister_info, @@ -118,7 +200,6 @@ pub(crate) async fn sync_many( &canister_ids, proxy, &mut pb, - pkg_cache, ) .await; diff --git a/crates/icp-deploy-canister/Cargo.toml b/crates/icp-deploy-canister/Cargo.toml new file mode 100644 index 000000000..bfe5f5792 --- /dev/null +++ b/crates/icp-deploy-canister/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "icp-deploy-canister" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +# This crate is intentionally dependency-light so that its install/sync core and +# project model can be compiled into a canister. All host-only IO (filesystem, +# HTTP, the ICP API, sync-step execution, the canister-id store) is abstracted +# behind trait objects; do NOT add ic-agent, reqwest, tokio, wasmtime, keyring, +# bollard, sysinfo, or std::fs-driven crates here. + +[features] +# Enables `clap::ValueEnum` derives on manifest enums used as CLI value types +# (e.g. `ArgsFormat`). Enabled transitively by `icp/clap`. +clap = ["dep:clap"] + +[dependencies] +async-trait.workspace = true +bigdecimal.workspace = true +camino.workspace = true +candid.workspace = true +clap = { workspace = true, optional = true } +candid_parser.workspace = true +futures.workspace = true +glob.workspace = true +handlebars.workspace = true +hex.workspace = true +ic-management-canister-types.workspace = true +indexmap.workspace = true +itertools.workspace = true +num-bigint.workspace = true +num-integer.workspace = true +num-traits.workspace = true +pathdiff.workspace = true +schemars.workspace = true +serde.workspace = true +sha2.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true +snafu.workspace = true +strum.workspace = true +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +camino-tempfile.workspace = true +indoc.workspace = true +jsonschema.workspace = true +tokio.workspace = true diff --git a/crates/icp-deploy-canister/src/canister/mod.rs b/crates/icp-deploy-canister/src/canister/mod.rs new file mode 100644 index 000000000..e8631dbca --- /dev/null +++ b/crates/icp-deploy-canister/src/canister/mod.rs @@ -0,0 +1,684 @@ +use std::collections::HashMap; + +use candid::{Nat, Principal}; +use ic_management_canister_types::{CanisterSettings, LogVisibility}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + parsers::{CyclesAmount, DurationAmount, MemoryAmount}, + prelude::*, +}; + +pub mod recipe; + +/// Controls who can read canister logs. +/// Supports both string format ("controllers", "public") and object format ({ allowed_viewers: [...] }). +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(untagged)] +pub enum LogVisibilityDef { + /// Simple string variants for controllers or public + Simple(LogVisibilitySimple), + /// Object format with allowed_viewers list + AllowedViewers { allowed_viewers: Vec }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum LogVisibilitySimple { + Controllers, + Public, +} + +impl<'de> Deserialize<'de> for LogVisibilityDef { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::{Error, MapAccess, Visitor}; + use std::fmt; + + struct LogVisibilityVisitor; + + impl<'de> Visitor<'de> for LogVisibilityVisitor { + type Value = LogVisibilityDef; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("'controllers', 'public', or object with 'allowed_viewers'") + } + + fn visit_str(self, value: &str) -> Result { + LogVisibilitySimple::deserialize( + serde::de::value::StrDeserializer::::new(value), + ) + .map(LogVisibilityDef::Simple) + .map_err(|_| { + E::custom(format!( + "unknown log_visibility value: '{}', expected 'controllers' or 'public'", + value + )) + }) + } + + fn visit_map(self, mut map: M) -> Result + where + M: MapAccess<'de>, + { + let mut allowed_viewers: Option> = None; + + while let Some(key) = map.next_key::()? { + match key.as_str() { + "allowed_viewers" => { + if allowed_viewers.is_some() { + return Err(Error::duplicate_field("allowed_viewers")); + } + allowed_viewers = Some(map.next_value()?); + } + _ => { + return Err(Error::unknown_field(&key, &["allowed_viewers"])); + } + } + } + + allowed_viewers + .map(|v| LogVisibilityDef::AllowedViewers { allowed_viewers: v }) + .ok_or_else(|| Error::missing_field("allowed_viewers")) + } + } + + deserializer.deserialize_any(LogVisibilityVisitor) + } +} + +impl JsonSchema for LogVisibilityDef { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::borrow::Cow::Borrowed("LogVisibility") + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "Controls who can read canister logs.", + "oneOf": [ + { + "type": "string", + "enum": ["controllers", "public"], + "description": "Simple log visibility: 'controllers' (only controllers can view) or 'public' (anyone can view)" + }, + { + "type": "object", + "properties": { + "allowed_viewers": { + "type": "array", + "items": { + "type": "string", + "description": "A principal ID that can view logs" + }, + "description": "List of principal IDs that can view canister logs" + } + }, + "required": ["allowed_viewers"], + "additionalProperties": false, + "description": "Specific principals that can view logs" + } + ] + }) + } +} + +impl From for LogVisibility { + fn from(value: LogVisibilityDef) -> Self { + match value { + LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) => { + LogVisibility::Controllers + } + LogVisibilityDef::Simple(LogVisibilitySimple::Public) => LogVisibility::Public, + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + LogVisibility::AllowedViewers(allowed_viewers) + } + } + } +} + +/// A reference to a controller: either an explicit principal or a canister name in this project. +/// +/// During deserialization, principal text format is tried first; strings that don't parse as a +/// principal are treated as canister names. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ControllerRef { + /// An explicitly specified principal (e.g. "2vxsx-fae") + Principal(candid::Principal), + /// A canister name from the same project (e.g. "my_canister") + CanisterName(String), +} + +impl ControllerRef { + /// Resolve to a `Principal` using the provided ID mapping. + /// Returns `None` if this is a `CanisterName` not present in `ids`. + pub fn resolve(&self, ids: &crate::ids::IdMapping) -> Option { + match self { + ControllerRef::Principal(p) => Some(*p), + ControllerRef::CanisterName(name) => ids.get(name).copied(), + } + } + + /// If this is a `CanisterName`, returns the name; otherwise `None`. + pub fn canister_name(&self) -> Option<&str> { + match self { + ControllerRef::CanisterName(n) => Some(n), + ControllerRef::Principal(_) => None, + } + } +} + +/// Partition a slice of controller references into resolved principals and unresolved canister +/// names, using `ids` for name lookup. +pub fn resolve_controllers( + crefs: &[ControllerRef], + ids: &crate::ids::IdMapping, +) -> (Vec, Vec) { + let mut resolved = Vec::new(); + let mut unresolved = Vec::new(); + for cref in crefs { + match cref.resolve(ids) { + Some(p) => resolved.push(p), + None => { + if let Some(name) = cref.canister_name() { + unresolved.push(name.to_owned()); + } + } + } + } + (resolved, unresolved) +} + +impl schemars::JsonSchema for ControllerRef { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::borrow::Cow::Borrowed("ControllerRef") + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "description": "A controller: either a principal text (e.g. '2vxsx-fae') or a canister name in this project (e.g. 'my_canister')" + }) + } +} + +/// An environment variable value as written in a manifest. +/// +/// A plain scalar is the value itself: +/// ```yaml +/// environment_variables: +/// API_ENDPOINT: https://api.example.com +/// ``` +/// +/// The object form reads the value from a file, relative to the canister's own +/// directory — including when an environment overrides the variable, matching how +/// an `init_args` override resolves its path: +/// ```yaml +/// environment_variables: +/// API_KEY: +/// path: ./secrets/api-key +/// ``` +#[derive(Clone, Debug, PartialEq, Deserialize, JsonSchema, Serialize)] +#[serde(untagged, expecting = "a string, or `{ path: }`")] +pub enum ManifestEnvVar { + /// The value, written inline. + Value(String), + /// A file holding the value. Surrounding whitespace is trimmed off the + /// file's contents, so a trailing newline does not become part of the value. + Path { + #[schemars(with = "String")] + path: PathBuf, + }, +} + +impl Default for ManifestEnvVar { + fn default() -> Self { + Self::Value(String::new()) + } +} + +/// Canister settings loaded from a manifest, before file-backed environment +/// variable values have been read. See [`Settings`] for the resolved form. +pub type ManifestSettings = Settings; + +/// Canister settings, such as compute and memory allocation. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Settings { + /// Controls who can read canister logs. + #[serde(skip_serializing_if = "Option::is_none")] + pub log_visibility: Option, + + /// Compute allocation (0 to 100). Represents guaranteed compute capacity. + #[serde(skip_serializing_if = "Option::is_none")] + pub compute_allocation: Option, + + /// Memory allocation in bytes. If unset, memory is allocated dynamically. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_allocation: Option, + + /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. + /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). + #[serde(skip_serializing_if = "Option::is_none")] + pub freezing_threshold: Option, + + /// Upper limit on cycles reserved for future resource payments. + /// Memory allocations that would push the reserved balance above this limit will fail. + /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reserved_cycles_limit: Option, + + /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub wasm_memory_limit: Option, + + /// Wasm memory threshold in bytes. Triggers a callback when exceeded. + /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] + pub wasm_memory_threshold: Option, + + /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. + /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub log_memory_limit: Option, + + /// Environment variables for the canister as key-value pairs. + /// These variables are accessible within the canister and can be used to configure + /// behavior without hardcoding values in the WASM module. + /// A value may also be read from a file with `{ path: }`. + #[serde(skip_serializing_if = "Option::is_none")] + pub environment_variables: Option>, + + /// Controllers for this canister. Each entry is either a principal text + /// (e.g. "2vxsx-fae") or the name of another canister in this project. + /// Named canisters that do not yet exist will be set as controllers once created. + #[serde(default)] + pub controllers: Option>, +} + +impl From for ManifestSettings { + fn from(settings: Settings) -> Self { + let Settings { + log_visibility, + compute_allocation, + memory_allocation, + freezing_threshold, + reserved_cycles_limit, + wasm_memory_limit, + wasm_memory_threshold, + log_memory_limit, + environment_variables, + controllers, + } = settings; + + Self { + log_visibility, + compute_allocation, + memory_allocation, + freezing_threshold, + reserved_cycles_limit, + wasm_memory_limit, + wasm_memory_threshold, + log_memory_limit, + environment_variables: environment_variables.map(|vars| { + vars.into_iter() + .map(|(name, value)| (name, ManifestEnvVar::Value(value))) + .collect() + }), + controllers, + } + } +} + +impl From for CanisterSettings { + fn from(settings: Settings) -> Self { + CanisterSettings { + freezing_threshold: settings.freezing_threshold.map(|d| Nat::from(d.get())), + controllers: None, + reserved_cycles_limit: settings.reserved_cycles_limit.map(|c| Nat::from(c.get())), + log_visibility: settings.log_visibility.map(Into::into), + memory_allocation: settings.memory_allocation.map(|m| Nat::from(m.get())), + compute_allocation: settings.compute_allocation.map(Nat::from), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use indoc::indoc; + + use super::*; + + #[test] + fn log_visibility_deserialize_controllers() { + let yaml = "controllers"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) + ); + } + + #[test] + fn log_visibility_deserialize_public() { + let yaml = "public"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + LogVisibilityDef::Simple(LogVisibilitySimple::Public) + ); + } + + #[test] + fn log_visibility_deserialize_allowed_viewers() { + let yaml = r#" +allowed_viewers: + - "aaaaa-aa" + - "2vxsx-fae" +"#; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + match result { + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + assert_eq!(allowed_viewers.len(), 2); + assert_eq!( + allowed_viewers[0], + Principal::from_text("aaaaa-aa").unwrap() + ); + assert_eq!( + allowed_viewers[1], + Principal::from_text("2vxsx-fae").unwrap() + ); + } + _ => panic!("Expected AllowedViewers variant"), + } + } + + #[test] + fn log_visibility_deserialize_allowed_viewers_empty() { + let yaml = "allowed_viewers: []"; + let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); + match result { + LogVisibilityDef::AllowedViewers { allowed_viewers } => { + assert!(allowed_viewers.is_empty()); + } + _ => panic!("Expected AllowedViewers variant"), + } + } + + #[test] + fn log_visibility_deserialize_invalid_string() { + let yaml = "invalid"; + let result: Result = serde_yaml::from_str(yaml); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("unknown log_visibility value")); + } + + #[test] + fn log_visibility_deserialize_invalid_field() { + let yaml = "unknown_field: []"; + let result: Result = serde_yaml::from_str(yaml); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("unknown field")); + } + + #[test] + fn log_visibility_serialize_controllers() { + let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert_eq!(yaml.trim(), "controllers"); + } + + #[test] + fn log_visibility_serialize_public() { + let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Public); + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert_eq!(yaml.trim(), "public"); + } + + #[test] + fn log_visibility_serialize_allowed_viewers() { + let log_vis = LogVisibilityDef::AllowedViewers { + allowed_viewers: vec![ + Principal::from_text("aaaaa-aa").unwrap(), + Principal::from_text("2vxsx-fae").unwrap(), + ], + }; + let yaml = serde_yaml::to_string(&log_vis).unwrap(); + assert!(yaml.contains("allowed_viewers")); + assert!(yaml.contains("aaaaa-aa")); + assert!(yaml.contains("2vxsx-fae")); + } + + #[test] + fn settings_reserved_cycles_limit_parses_suffix() { + let yaml = "reserved_cycles_limit: 4.3t"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.reserved_cycles_limit.as_ref().map(|c| c.get()), + Some(4_300_000_000_000) + ); + } + + #[test] + fn settings_reserved_cycles_limit_parses_number() { + let yaml = "reserved_cycles_limit: 5000000000000"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.reserved_cycles_limit.as_ref().map(|c| c.get()), + Some(5_000_000_000_000) + ); + } + + #[test] + fn settings_memory_allocation_parses_suffix() { + let yaml = "memory_allocation: 4gib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.memory_allocation.as_ref().map(|m| m.get()), + Some(4 * 1024 * 1024 * 1024) + ); + } + + #[test] + fn settings_memory_allocation_parses_number() { + let yaml = "memory_allocation: 4294967296"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.memory_allocation.as_ref().map(|m| m.get()), + Some(4294967296) + ); + } + + #[test] + fn settings_wasm_memory_limit_parses_suffix() { + let yaml = "wasm_memory_limit: 1.5gib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.wasm_memory_limit.as_ref().map(|m| m.get()), + Some(1610612736) + ); + } + + #[test] + fn settings_log_memory_limit_parses_suffix() { + let yaml = "log_memory_limit: 256kib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.log_memory_limit.as_ref().map(|m| m.get()), + Some(256 * 1024) + ); + } + + #[test] + fn settings_log_memory_limit_parses_mib() { + let yaml = "log_memory_limit: 2mib"; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.log_memory_limit.as_ref().map(|m| m.get()), + Some(2 * 1024 * 1024) + ); + } + + #[test] + fn settings_environment_variables_take_values_or_files() { + let yaml = indoc! {r#" + environment_variables: + API_ENDPOINT: https://api.example.com + API_KEY: + path: ./secrets/api-key + "#}; + let settings: ManifestSettings = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + settings.environment_variables, + Some(HashMap::from([ + ( + "API_ENDPOINT".to_owned(), + ManifestEnvVar::Value("https://api.example.com".to_owned()), + ), + ( + "API_KEY".to_owned(), + ManifestEnvVar::Path { + path: "./secrets/api-key".into(), + }, + ), + ])), + ); + } + + #[test] + fn settings_environment_variable_rejects_unknown_object_form() { + let yaml = indoc! {r#" + environment_variables: + API_KEY: + file: ./secrets/api-key + "#}; + let err = serde_yaml::from_str::(yaml) + .expect_err("only the `path` object form is accepted"); + assert!( + err.to_string().contains("a string, or `{ path: }`"), + "unhelpful error: {err}" + ); + } + + /// A value of the wrong scalar type reports what is accepted, rather than + /// serde's default "did not match any variant" for an untagged enum. + #[test] + fn settings_environment_variable_rejects_non_string_scalar() { + let err = + serde_yaml::from_str::("environment_variables:\n PORT: 8080\n") + .expect_err("a bare integer is not a value"); + assert!( + err.to_string().contains("a string, or `{ path: }`"), + "unhelpful error: {err}" + ); + } + + #[test] + fn resolved_settings_serialize_environment_variables_inline() { + let settings = Settings { + environment_variables: Some(HashMap::from([( + "API_KEY".to_owned(), + "s3cret".to_owned(), + )])), + ..Default::default() + }; + let yaml = serde_yaml::to_string(&ManifestSettings::from(settings)).unwrap(); + assert!( + yaml.contains("environment_variables:\n API_KEY: s3cret\n"), + "unexpected yaml: {yaml}" + ); + } + + #[test] + fn controller_ref_deserializes_principal() { + let yaml = "\"2vxsx-fae\""; + let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + ControllerRef::Principal(Principal::from_text("2vxsx-fae").unwrap()) + ); + } + + #[test] + fn controller_ref_deserializes_canister_name() { + let yaml = "\"my_canister\""; + let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + result, + ControllerRef::CanisterName("my_canister".to_owned()) + ); + } + + #[test] + fn controller_ref_resolve_principal() { + let p = Principal::from_text("aaaaa-aa").unwrap(); + let cref = ControllerRef::Principal(p); + let ids = crate::ids::IdMapping::new(); + assert_eq!(cref.resolve(&ids), Some(p)); + } + + #[test] + fn controller_ref_resolve_canister_name_present() { + let p = Principal::from_text("aaaaa-aa").unwrap(); + let cref = ControllerRef::CanisterName("backend".to_owned()); + let mut ids = crate::ids::IdMapping::new(); + ids.insert("backend".to_owned(), p); + assert_eq!(cref.resolve(&ids), Some(p)); + } + + #[test] + fn controller_ref_resolve_canister_name_absent() { + let cref = ControllerRef::CanisterName("backend".to_owned()); + let ids = crate::ids::IdMapping::new(); + assert_eq!(cref.resolve(&ids), None); + } + + #[test] + fn settings_controllers_parses_mixed() { + let yaml = r#" +controllers: + - "aaaaa-aa" + - "my_other_canister" +"#; + let settings: Settings = serde_yaml::from_str(yaml).unwrap(); + let controllers = settings.controllers.unwrap(); + assert_eq!(controllers.len(), 2); + assert_eq!( + controllers[0], + ControllerRef::Principal(Principal::from_text("aaaaa-aa").unwrap()) + ); + assert_eq!( + controllers[1], + ControllerRef::CanisterName("my_other_canister".to_owned()) + ); + } + + #[test] + fn log_visibility_conversion_to_ic_type() { + let controllers = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); + let ic_controllers: LogVisibility = controllers.into(); + assert!(matches!(ic_controllers, LogVisibility::Controllers)); + + let public = LogVisibilityDef::Simple(LogVisibilitySimple::Public); + let ic_public: LogVisibility = public.into(); + assert!(matches!(ic_public, LogVisibility::Public)); + + let viewers = LogVisibilityDef::AllowedViewers { + allowed_viewers: vec![Principal::from_text("aaaaa-aa").unwrap()], + }; + let ic_viewers: LogVisibility = viewers.into(); + match ic_viewers { + LogVisibility::AllowedViewers(v) => { + assert_eq!(v.len(), 1); + } + _ => panic!("Expected AllowedViewers"), + } + } +} diff --git a/crates/icp/src/canister/recipe/render.rs b/crates/icp-deploy-canister/src/canister/recipe/mod.rs similarity index 68% rename from crates/icp/src/canister/recipe/render.rs rename to crates/icp-deploy-canister/src/canister/recipe/mod.rs index f59941c73..edc150541 100644 --- a/crates/icp/src/canister/recipe/render.rs +++ b/crates/icp-deploy-canister/src/canister/recipe/mod.rs @@ -1,22 +1,19 @@ -//! Stage two of recipe resolution: turn template text into build/sync steps. - use std::collections::HashMap; +use async_trait::async_trait; use handlebars::{Context, Handlebars, Helper, HelperDef, HelperResult, Output, RenderContext}; use serde::Deserialize; use snafu::prelude::*; -use tracing::debug; use crate::manifest::{ + adapter::prebuilt::SourceField, canister::{BuildSteps, SyncSteps}, recipe::{Recipe, RecipeType}, }; +use crate::prelude::*; +use crate::sync_exec::StepProgress; -/// Describes the canister being built, for the render stage. -/// -/// Belongs to rendering alone: [`Resolve::resolve`](super::Resolve::resolve) no -/// longer takes it, since fetching a template does not depend on which canister -/// the template is for. Only [`render_recipe`] consumes it. +/// Context passed to a recipe resolver, describing the canister being built. /// /// Serializes to the shape injected into recipe templates under the `_` namespace: /// @@ -44,14 +41,82 @@ impl RecipeContext { } } +/// Fetches the remote resources a project references — recipe templates and +/// plugin wasms — retrieving them over HTTP and caching as needed. +/// +/// The concrete resolver (which owns the HTTP client and the package cache) +/// lives in the host `icp` crate; this crate defines the interface, and renders +/// fetched recipe templates itself (see [`render_recipe`]), so that +/// consolidation and sync can call an injected resolver. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait RemoteResourceResolve: Sync + Send { + /// Fetch a recipe's Handlebars template, returning its raw source. Callers + /// render it into build/sync steps with [`render_recipe`], then hand the + /// result back to [`commit_recipe`](Self::commit_recipe). + async fn resolve_recipe(&self, recipe: &Recipe) -> Result; + + /// Accept a template from [`resolve_recipe`](Self::resolve_recipe) once it + /// has rendered successfully, letting the resolver commit whatever it held + /// back — for the host resolver, writing a fresh download to the package + /// cache. A resolver that never sets [`FetchedRecipe::deferred`] has nothing + /// to commit and implements this as `Ok(())`. + async fn commit_recipe( + &self, + recipe: &Recipe, + fetched: &FetchedRecipe, + ) -> Result<(), ResolveError>; + + /// Resolve a plugin wasm `source` (relative to `base_dir`) to a location the + /// host's [`PluginExecutor`](crate::sync_exec::PluginExecutor) can load, + /// verifying `sha256` and caching a remote download. `progress` receives + /// status lines. + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + progress: Option<&dyn StepProgress>, + ) -> Result; +} + +/// A recipe template retrieved by a [`RemoteResourceResolve`]. +/// +/// A resolver that caches downloads must not commit one before the template is +/// known to render: for an unpinned URL a single malformed response would +/// otherwise become the cached entry that every later project load reuses. Such +/// a resolver returns the template with `deferred` set and waits for +/// [`RemoteResourceResolve::commit_recipe`]. +pub struct FetchedRecipe { + /// Raw Handlebars template source. + pub template: String, + + /// Whether the resolver is holding work back until the caller confirms the + /// template renders. + pub deferred: bool, +} + +#[derive(Debug, Snafu)] +pub enum ResolveError { + /// The injected resolver failed. The concrete source (e.g. a fetch/cache + /// error from the host resolver) is boxed because this crate does not depend + /// on the resolver's implementation. + #[snafu(display("failed to fetch recipe template"))] + Resolve { + source: Box, + }, + + #[snafu(display("failed to resolve plugin wasm"))] + ResolveWasm { + source: Box, + }, +} + #[derive(Debug, Snafu)] pub enum RenderRecipeError { #[snafu(display("recipe template for '{recipe}' failed to render"))] Render { - // Boxed to keep `Result<_, RenderRecipeError>` small; `RenderError` - // alone is well over a hundred bytes. - #[snafu(source(from(handlebars::RenderError, Box::new)))] - source: Box, + source: handlebars::RenderError, recipe: RecipeType, }, @@ -67,6 +132,7 @@ pub enum RenderRecipeError { /// The template is rendered with the recipe's `configuration` plus the reserved /// `_` namespace (the `_` key always overrides any user-supplied value), then the /// resulting YAML is parsed. A recipe may only produce `build` and `sync`. +#[allow(clippy::result_large_err)] pub fn render_recipe( template: &str, recipe: &Recipe, @@ -84,19 +150,12 @@ pub fn render_recipe( let mut render_context: HashMap = recipe.configuration.clone(); render_context.insert("_".to_string(), recipe_context.to_yaml()); - debug!("Rendering recipe template:\n------\n{template}\n------"); - let out = reg .render_template(template, &render_context) .context(RenderSnafu { recipe: recipe.recipe_type.clone(), })?; - // Logged rather than carried in `Parse` below: a recipe author debugging a - // malformed render needs the whole document, which is too much for an error - // message. - debug!("Rendered recipe template:\n------\n{out}\n------"); - // Recipes can only render `build`/`sync`. #[derive(Deserialize)] struct BuildSyncHelper { @@ -125,14 +184,9 @@ impl HelperDef for ReplaceHelper { _: &mut RenderContext<'reg, 'rc>, out: &mut dyn Output, ) -> HelperResult { - let (from, to) = ( - h.param(0).unwrap().render(), // from - h.param(1).unwrap().render(), // to - ); - + let (from, to) = (h.param(0).unwrap().render(), h.param(1).unwrap().render()); let v = h.param(2).unwrap().render(); out.write(&v.replace(&from, &to))?; - Ok(()) } } diff --git a/crates/icp-deploy-canister/src/deploy.rs b/crates/icp-deploy-canister/src/deploy.rs new file mode 100644 index 000000000..2e860e953 --- /dev/null +++ b/crates/icp-deploy-canister/src/deploy.rs @@ -0,0 +1,1051 @@ +//! Canister installation, environment-variable wiring, syncing, and deploy +//! orchestration, expressed entirely over the injected IO traits so the core +//! can run inside a canister. + +use std::collections::BTreeMap; + +use candid::Principal; +use candid::utils::ArgumentEncoder; +use ic_management_canister_types::{ + CanisterIdRecord, CanisterInstallMode, CanisterSettings, CanisterStatusResult, + CanisterStatusType, ChunkHash, ClearChunkStoreArgs, EnvironmentVariable, + InstallChunkedCodeArgs, InstallCodeArgs, UpdateSettingsArgs, UpgradeFlags, UploadChunkArgs, + WasmMemoryPersistence, +}; +use sha2::{Digest, Sha256}; +use snafu::prelude::*; + +use crate::{ + Canister, Project, + files::{FileAccess, FileAccessError}, + icp_access::{IcpAccess, IcpAccessError}, + ids::IdStore, + manifest::canister::SyncStep, + network::Configuration, + prelude::*, + sync_exec::{ + PluginExecutor, PluginInvocation, ScriptInvocation, ScriptRunner, StepProgress, + SyncStepContext, + }, +}; + +/// EOP custom-section metadata key marking a Motoko enhanced-orthogonal-persistence canister. +const EOP_METADATA: &str = "enhanced-orthogonal-persistence"; + +/// Requested installation mode. `Auto` resolves to `Install` or `Upgrade` by +/// querying the canister's current status. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InstallMode { + Auto, + Install, + Reinstall, + Upgrade, +} + +// --------------------------------------------------------------------------- +// Management-canister transport +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum MgmtCallError { + #[snafu(display("failed to encode arguments for management method '{method}'"))] + Encode { + method: String, + source: candid::Error, + }, + + #[snafu(display("management call '{method}' failed"))] + Call { + method: String, + source: IcpAccessError, + }, + + #[snafu(display("failed to decode reply from management method '{method}'"))] + Decode { + method: String, + source: candid::Error, + }, +} + +/// Encode + issue a management-canister update call and decode its reply. +/// +/// The management canister has no routing of its own, so the effective canister +/// id is the `target`. Candid coding happens here; [`IcpAccess`] is dumb transport. +async fn mgmt_call( + icp: &dyn IcpAccess, + method: &str, + target: Principal, + args: A, + cycles: u128, +) -> Result +where + A: ArgumentEncoder, + R: for<'a> candid::utils::ArgumentDecoder<'a>, +{ + let arg = candid::encode_args(args).context(EncodeSnafu { method })?; + let raw = icp + .canister_update( + Principal::management_canister(), + method, + arg, + target, + cycles, + ) + .await + .context(CallSnafu { method })?; + candid::decode_args(&raw).context(DecodeSnafu { method }) +} + +// --------------------------------------------------------------------------- +// Install +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum InstallCanisterError { + #[snafu(display("failed to read the built artifact for canister '{canister}'"))] + ReadArtifact { + canister: String, + source: FileAccessError, + }, + + #[snafu(display("failed to query status of canister '{canister}'"))] + CanisterStatus { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to detect orthogonal-persistence metadata on canister '{canister}'"))] + DetectEop { + canister: String, + source: IcpAccessError, + }, + + #[snafu(display("failed to stop canister '{canister}' before upgrade"))] + StopCanister { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to start canister '{canister}' after upgrade"))] + StartCanister { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to clear the chunk store for canister '{canister}'"))] + ClearChunkStore { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to upload wasm chunk {index} for canister '{canister}'"))] + UploadChunk { + canister: String, + index: usize, + source: MgmtCallError, + }, + + #[snafu(display("failed to install code on canister '{canister}'"))] + InstallCode { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to install chunked code on canister '{canister}'"))] + InstallChunkedCode { + canister: String, + source: MgmtCallError, + }, +} + +/// Query `canister_status` and pick the concrete install mode for `Auto`. +async fn resolve_mode_and_status( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + mode: InstallMode, +) -> Result<(CanisterInstallMode, CanisterStatusType), InstallCanisterError> { + let (status,): (CanisterStatusResult,) = mgmt_call( + icp, + "canister_status", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(CanisterStatusSnafu { + canister: canister_name, + })?; + let install_mode = match mode { + InstallMode::Auto => { + if status.module_hash.is_some() { + CanisterInstallMode::Upgrade(None) + } else { + CanisterInstallMode::Install + } + } + InstallMode::Install => CanisterInstallMode::Install, + InstallMode::Reinstall => CanisterInstallMode::Reinstall, + InstallMode::Upgrade => CanisterInstallMode::Upgrade(None), + }; + Ok((install_mode, status.status)) +} + +/// Whether the canister exposes the `enhanced-orthogonal-persistence` metadata. +async fn is_eop_canister( + icp: &dyn IcpAccess, + canister_id: Principal, +) -> Result { + Ok(icp + .read_canister_metadata(canister_id, EOP_METADATA) + .await? + .is_some()) +} + +/// Install (or upgrade/reinstall) a single, already-built canister. +/// +/// Reads the wasm from `artifact_path` through `files`; resolves `Auto` mode and +/// EOP-upgrade flags through `icp`. Large wasm is installed via the chunk store. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister( + canister_name: &str, + canister_id: Principal, + artifact_path: &Path, + mode: InstallMode, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + files: &dyn FileAccess, + icp: &dyn IcpAccess, +) -> Result<(), InstallCanisterError> { + let (mode, status) = resolve_mode_and_status(icp, canister_name, canister_id, mode).await?; + install_canister_resolved( + canister_name, + canister_id, + artifact_path, + mode, + status, + init_args, + wasm_memory_persistence, + files, + icp, + ) + .await +} + +/// Like [`install_canister`], but with the install mode and current status +/// already resolved by the caller. Callers that need the resolved mode/status +/// for their own logic first (e.g. a Candid-compatibility gate before install) +/// resolve once via [`resolve_install_mode_and_status`] and pass the result here, +/// avoiding a second `canister_status` call. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister_resolved( + canister_name: &str, + canister_id: Principal, + artifact_path: &Path, + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + files: &dyn FileAccess, + icp: &dyn IcpAccess, +) -> Result<(), InstallCanisterError> { + let wasm = files + .read_file(artifact_path) + .await + .context(ReadArtifactSnafu { + canister: canister_name, + })?; + install_canister_wasm( + canister_name, + canister_id, + &wasm, + mode, + status, + init_args, + wasm_memory_persistence, + icp, + ) + .await +} + +/// Like [`install_canister_resolved`], but for a caller that already holds the +/// wasm bytes. Callers that inspect the module before installing (e.g. the +/// Candid-compatibility gate) pass those same bytes here, so the code that is +/// installed is exactly the code that was checked. +#[allow(clippy::too_many_arguments)] +pub async fn install_canister_wasm( + canister_name: &str, + canister_id: Principal, + wasm: &[u8], + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, + wasm_memory_persistence: Option, + icp: &dyn IcpAccess, +) -> Result<(), InstallCanisterError> { + // For EOP Motoko canisters an upgrade must set `wasm_memory_persistence`. + // Trust an explicit caller override; otherwise auto-detect and default to Keep. + let mode = match mode { + CanisterInstallMode::Upgrade(_) => { + let persistence = match wasm_memory_persistence { + Some(p) => Some(p), + None => is_eop_canister(icp, canister_id) + .await + .context(DetectEopSnafu { + canister: canister_name, + })? + .then_some(WasmMemoryPersistence::Keep), + }; + if let Some(persistence) = persistence { + CanisterInstallMode::Upgrade(Some(UpgradeFlags { + skip_pre_upgrade: None, + wasm_memory_persistence: Some(persistence), + })) + } else { + mode + } + } + other => other, + }; + + do_install( + icp, + canister_name, + canister_id, + wasm, + mode, + status, + init_args, + ) + .await +} + +/// Resolve an [`InstallMode`] and the canister's current status via +/// `canister_status` (the resolution [`install_canister`] performs internally). +/// Exposed for callers that need the resolved mode before installing. +pub async fn resolve_install_mode_and_status( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + mode: InstallMode, +) -> Result<(CanisterInstallMode, CanisterStatusType), InstallCanisterError> { + resolve_mode_and_status(icp, canister_name, canister_id, mode).await +} + +#[allow(clippy::too_many_arguments)] +async fn do_install( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + wasm: &[u8], + mode: CanisterInstallMode, + status: CanisterStatusType, + init_args: Option<&[u8]>, +) -> Result<(), InstallCanisterError> { + // Raw install_code messages are limited to 2 MiB; larger wasm goes through + // the chunk store (spec limit is 1 MiB per chunk). + const CHUNK_THRESHOLD: usize = 2 * 1024 * 1024; + const CHUNK_SIZE: usize = 1024 * 1024; + // Generous overhead for encoding, target canister ID, install mode, etc. + const ENCODING_OVERHEAD: usize = 500; + + let arg = init_args + .map(|a| a.to_vec()) + .unwrap_or_else(|| candid::encode_args(()).expect("encoding empty args is infallible")); + + let total_install_size = wasm.len() + arg.len() + ENCODING_OVERHEAD; + + if total_install_size <= CHUNK_THRESHOLD { + let install_args = InstallCodeArgs { + mode, + canister_id, + wasm_module: wasm.to_vec(), + arg, + sender_canister_version: None, + }; + stop_and_start_if_needed(icp, canister_name, canister_id, mode, status, async { + mgmt_call::<_, ()>(icp, "install_code", canister_id, (install_args,), 0) + .await + .context(InstallCodeSnafu { + canister: canister_name, + }) + }) + .await?; + } else { + // Clear any existing chunks to ensure a clean state. + clear_chunk_store(icp, canister_name, canister_id).await?; + + let chunks: Vec<&[u8]> = wasm.chunks(CHUNK_SIZE).collect(); + let mut chunk_hashes: Vec = Vec::new(); + for (i, chunk) in chunks.iter().enumerate() { + let (hash,): (ChunkHash,) = mgmt_call( + icp, + "upload_chunk", + canister_id, + (UploadChunkArgs { + canister_id, + chunk: chunk.to_vec(), + },), + 0, + ) + .await + .context(UploadChunkSnafu { + canister: canister_name, + index: i, + })?; + chunk_hashes.push(hash); + } + + let wasm_module_hash = Sha256::digest(wasm).to_vec(); + let chunked_args = InstallChunkedCodeArgs { + mode, + target_canister: canister_id, + store_canister: None, + chunk_hashes_list: chunk_hashes, + wasm_module_hash, + arg, + sender_canister_version: None, + }; + + let install_res = + stop_and_start_if_needed(icp, canister_name, canister_id, mode, status, async { + mgmt_call::<_, ()>(icp, "install_chunked_code", canister_id, (chunked_args,), 0) + .await + .context(InstallChunkedCodeSnafu { + canister: canister_name, + }) + }) + .await; + + // Always clear the chunk store afterwards to free storage. If the install + // failed, report that error in preference to a clear-store failure. + let clear_res = clear_chunk_store(icp, canister_name, canister_id).await; + install_res?; + clear_res?; + } + + Ok(()) +} + +async fn clear_chunk_store( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, +) -> Result<(), InstallCanisterError> { + mgmt_call::<_, ()>( + icp, + "clear_chunk_store", + canister_id, + (ClearChunkStoreArgs { canister_id },), + 0, + ) + .await + .context(ClearChunkStoreSnafu { + canister: canister_name, + }) +} + +/// Guard an upgrade/reinstall of a Running canister by stopping it first and +/// restarting it afterwards (whether or not the install succeeded). +async fn stop_and_start_if_needed( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, + mode: CanisterInstallMode, + status: CanisterStatusType, + install: F, +) -> Result<(), InstallCanisterError> +where + F: Future>, +{ + let should_guard = matches!( + mode, + CanisterInstallMode::Upgrade(_) | CanisterInstallMode::Reinstall + ) && matches!(status, CanisterStatusType::Running); + + if should_guard { + mgmt_call::<_, ()>( + icp, + "stop_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StopCanisterSnafu { + canister: canister_name, + })?; + } + + let install_result = install.await; + + if !should_guard { + return install_result; + } + + let start_result = mgmt_call::<_, ()>( + icp, + "start_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StartCanisterSnafu { + canister: canister_name, + }); + + // Restart whether or not the install succeeded. If both failed, the install + // error is the more likely root cause. + match (install_result, start_result) { + (Err(install_err), _) => Err(install_err), + (Ok(()), Err(start_err)) => Err(start_err), + (Ok(()), Ok(())) => Ok(()), + } +} + +/// Start a canister (idempotent; a no-op if already Running). +pub async fn start_canister( + icp: &dyn IcpAccess, + canister_name: &str, + canister_id: Principal, +) -> Result<(), InstallCanisterError> { + mgmt_call::<_, ()>( + icp, + "start_canister", + canister_id, + (CanisterIdRecord { canister_id },), + 0, + ) + .await + .context(StartCanisterSnafu { + canister: canister_name, + }) +} + +// --------------------------------------------------------------------------- +// Environment variables + sync +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum SyncCanisterError { + #[snafu(display("failed to apply environment variables to canister '{canister}'"))] + ApplyEnvVars { + canister: String, + source: MgmtCallError, + }, + + #[snafu(display("failed to run sync step for canister '{canister}'"))] + RunStep { + canister: String, + source: SyncStepError, + }, +} + +#[derive(Debug, Snafu)] +pub enum SyncStepError { + #[snafu(transparent)] + ResolvePlugin { + source: crate::sync_exec::UnknownCallableCanisterError, + }, + + #[snafu(transparent)] + Plugin { + source: crate::sync_exec::PluginExecutorError, + }, + + #[snafu(transparent)] + Script { + source: crate::sync_exec::ScriptRunError, + }, +} + +/// Compute the environment variables a canister should run with: its manifest +/// `settings` variables merged with the generated `PUBLIC_CANISTER_ID:` +/// variables, resolved against `canister_ids`. +/// +/// Each canister is wired only to the ids it declares in `bindings`, resolved to +/// the ids that exist in this environment; unresolved bindings are skipped. +pub fn binding_env_vars( + canister: &Canister, + canister_ids: &BTreeMap, +) -> Vec { + let mut env_vars = canister + .settings + .environment_variables + .clone() + .unwrap_or_default(); + + for (env_name, referenced_key) in &canister.bindings { + if let Some(principal) = canister_ids.get(referenced_key) { + env_vars.insert( + format!("PUBLIC_CANISTER_ID:{env_name}"), + principal.to_text(), + ); + } + } + + env_vars + .into_iter() + .map(|(name, value)| EnvironmentVariable { name, value }) + .collect() +} + +/// Apply the canister's environment variables (see [`binding_env_vars`]) via +/// `update_settings`. +/// +/// This is the piece that standalone `icp sync` previously skipped: the binding +/// ids must be (re)written whenever a canister is synced, not only on deploy. +pub async fn apply_binding_env_vars( + canister: &Canister, + canister_id: Principal, + canister_ids: &BTreeMap, + icp: &dyn IcpAccess, +) -> Result<(), SyncCanisterError> { + let environment_variables = binding_env_vars(canister, canister_ids); + + mgmt_call::<_, ()>( + icp, + "update_settings", + canister_id, + (UpdateSettingsArgs { + canister_id, + settings: CanisterSettings { + environment_variables: Some(environment_variables), + ..Default::default() + }, + sender_canister_version: None, + },), + 0, + ) + .await + .context(ApplyEnvVarsSnafu { + canister: canister.name.clone(), + }) +} + +/// Run a canister's configured sync steps through the injected executors, +/// collecting any retained stderr lines. Does not apply environment variables +/// (see [`apply_binding_env_vars`] / [`sync_canister`]). +pub async fn run_sync_steps( + canister: &Canister, + ctx: &SyncStepContext, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, + progress: Option<&dyn StepProgress>, +) -> Result, SyncCanisterError> { + let mut lines = Vec::new(); + for step in &canister.sync.steps { + // This crate owns dispatch and input derivation; the executors only run + // the fully-resolved invocation. + let step_lines = match step { + SyncStep::Plugin(adapter) => match PluginInvocation::new(adapter, ctx) { + Ok(invocation) => plugin_exec + .run_plugin(invocation, progress) + .await + .map_err(SyncStepError::from), + Err(err) => Err(SyncStepError::from(err)), + }, + SyncStep::Script(adapter) => script_runner + .run_script(ScriptInvocation::new(adapter, ctx), progress) + .await + .map_err(SyncStepError::from), + } + .context(RunStepSnafu { + canister: canister.name.clone(), + })?; + lines.extend(step_lines); + } + Ok(lines) +} + +/// Sync a single canister: (re)apply its binding environment variables, then run +/// its sync steps. Applying env vars here is what makes standalone `icp sync` +/// include the generated `PUBLIC_CANISTER_ID:*` variables. +#[allow(clippy::too_many_arguments)] +pub async fn sync_canister( + canister: &Canister, + canister_id: Principal, + ctx: &SyncStepContext, + icp: &dyn IcpAccess, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, + progress: Option<&dyn StepProgress>, +) -> Result, SyncCanisterError> { + apply_binding_env_vars(canister, canister_id, &ctx.canister_ids, icp).await?; + run_sync_steps(canister, ctx, plugin_exec, script_runner, progress).await +} + +// --------------------------------------------------------------------------- +// Deploy +// --------------------------------------------------------------------------- + +#[derive(Debug, Snafu)] +pub enum DeployCanisterError { + #[snafu(display("could not find canister '{canister}' in environment '{environment}'"))] + UnknownCanister { + canister: String, + environment: String, + }, + + #[snafu(display("could not find an id for canister '{canister}'; create it first"))] + LookupId { + canister: String, + source: crate::ids::IdStoreError, + }, + + #[snafu(display("failed to encode init args for canister '{canister}'"))] + InitArgs { + canister: String, + source: crate::InitArgsToBytesError, + }, + + #[snafu(transparent)] + Install { source: InstallCanisterError }, + + #[snafu(transparent)] + Sync { source: SyncCanisterError }, +} + +/// Deploy (install then sync) a single already-built canister in `environment`. +/// +/// Assumes the canister already exists (its id is read from `ids`); creating +/// canisters is the caller's responsibility. +#[allow(clippy::too_many_arguments)] +pub async fn deploy_canister( + project: &Project, + canister_name: &str, + environment: &str, + artifact_path: &Path, + mode: InstallMode, + proxy: Option, + files: &dyn FileAccess, + icp: &dyn IcpAccess, + ids: &dyn IdStore, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, + progress: Option<&dyn StepProgress>, +) -> Result, DeployCanisterError> { + let env = project + .environments + .get(environment) + .context(UnknownCanisterSnafu { + canister: canister_name, + environment, + })?; + let is_cache = matches!(env.network.configuration, Configuration::Managed { .. }); + let network = env.network.name.clone(); + + let (canister_path, canister) = + env.canisters + .get(canister_name) + .context(UnknownCanisterSnafu { + canister: canister_name, + environment, + })?; + + let canister_id = ids + .lookup(is_cache, environment, canister_name) + .context(LookupIdSnafu { + canister: canister_name, + })?; + let canister_ids = ids + .lookup_by_environment(is_cache, environment) + .unwrap_or_default(); + + let init_args = canister + .init_args + .as_ref() + .map(|ia| ia.to_bytes()) + .transpose() + .context(InitArgsSnafu { + canister: canister_name, + })?; + + // Environment variables first, then install, then sync steps. + apply_binding_env_vars(canister, canister_id, &canister_ids, icp).await?; + install_canister( + canister_name, + canister_id, + artifact_path, + mode, + init_args.as_deref(), + None, + files, + icp, + ) + .await?; + // Asset sync requires a Running canister; install_code is status-preserving. + start_canister(icp, canister_name, canister_id).await?; + + let ctx = SyncStepContext { + canister_path: canister_path.clone(), + canister_id, + canister_name: canister.name.clone(), + environment: environment.to_owned(), + network, + canister_ids, + proxy, + }; + let lines = run_sync_steps(canister, &ctx, plugin_exec, script_runner, progress).await?; + Ok(lines) +} + +#[derive(Debug, Snafu)] +#[snafu(display("failed to deploy canister(s): {}", names.join(", ")))] +pub struct DeployError { + pub names: Vec, + pub failures: Vec<(String, DeployCanisterError)>, +} + +/// Deploy the `selected` already-built canisters in `environment`, each through +/// [`deploy_canister`]. `artifact_paths` maps canister name → its built wasm +/// path. Per-canister failures are aggregated. This is a batch entry point with +/// no progress reporting; the CLI drives its own per-canister fan-out instead. +#[allow(clippy::too_many_arguments)] +pub async fn deploy( + project: &Project, + selected: &[String], + environment: &str, + mode: InstallMode, + proxy: Option, + artifact_paths: &BTreeMap, + files: &dyn FileAccess, + icp: &dyn IcpAccess, + ids: &dyn IdStore, + plugin_exec: &dyn PluginExecutor, + script_runner: &dyn ScriptRunner, +) -> Result<(), DeployError> { + let mut failures = Vec::new(); + for name in selected { + let Some(artifact_path) = artifact_paths.get(name) else { + failures.push(( + name.clone(), + DeployCanisterError::UnknownCanister { + canister: name.clone(), + environment: environment.to_owned(), + }, + )); + continue; + }; + if let Err(e) = deploy_canister( + project, + name, + environment, + artifact_path, + mode, + proxy, + files, + icp, + ids, + plugin_exec, + script_runner, + None, + ) + .await + { + failures.push((name.clone(), e)); + } + } + + if failures.is_empty() { + Ok(()) + } else { + let names = failures.iter().map(|(n, _)| n.clone()).collect(); + Err(DeployError { names, failures }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::canister::Settings; + use crate::manifest::adapter::script::{self, CommandField}; + use crate::manifest::canister::{BuildSteps, SyncStep, SyncSteps}; + use async_trait::async_trait; + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + /// Test principal `2vxsx-fae` (the anonymous principal), used as a stand-in. + fn principal() -> Principal { + Principal::anonymous() + } + + /// Records the ordered sequence of interactions across the mock `IcpAccess` + /// and mock sync executors, plus the raw args of each management call. + #[derive(Default)] + struct Log { + events: Vec, + calls: Vec<(String, Vec)>, + } + + struct MockIcp { + log: Arc>, + } + + #[cfg_attr(target_family = "wasm", async_trait(?Send))] + #[cfg_attr(not(target_family = "wasm"), async_trait)] + impl IcpAccess for MockIcp { + async fn canister_update( + &self, + _canister: Principal, + method: &str, + arg: Vec, + _effective_canister_id: Principal, + _cycles: u128, + ) -> Result, IcpAccessError> { + let mut log = self.log.lock().unwrap(); + log.events.push(method.to_owned()); + log.calls.push((method.to_owned(), arg)); + // Every management method exercised here replies with unit. + Ok(candid::encode_args(()).unwrap()) + } + + async fn read_canister_metadata( + &self, + _canister: Principal, + _path: &str, + ) -> Result>, IcpAccessError> { + Ok(None) + } + + fn caller_principal(&self) -> Principal { + principal() + } + } + + struct MockExec { + log: Arc>, + } + + #[cfg_attr(target_family = "wasm", async_trait(?Send))] + #[cfg_attr(not(target_family = "wasm"), async_trait)] + impl PluginExecutor for MockExec { + async fn run_plugin( + &self, + _invocation: PluginInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, crate::sync_exec::PluginExecutorError> { + self.log + .lock() + .unwrap() + .events + .push("run_plugin".to_owned()); + Ok(vec![]) + } + } + + #[cfg_attr(target_family = "wasm", async_trait(?Send))] + #[cfg_attr(not(target_family = "wasm"), async_trait)] + impl ScriptRunner for MockExec { + async fn run_script( + &self, + _invocation: ScriptInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, crate::sync_exec::ScriptRunError> { + self.log + .lock() + .unwrap() + .events + .push("run_script".to_owned()); + Ok(vec![]) + } + } + + fn canister_with_binding() -> Canister { + Canister { + name: "backend".to_owned(), + settings: Settings { + environment_variables: Some(HashMap::from([("FOO".to_owned(), "bar".to_owned())])), + ..Default::default() + }, + build: BuildSteps { steps: vec![] }, + sync: SyncSteps { + steps: vec![SyncStep::Script(script::Adapter { + command: CommandField::Command("noop".to_owned()), + })], + }, + init_args: None, + registry_recipe: None, + bindings: BTreeMap::from([("backend".to_owned(), "backend".to_owned())]), + friendly_names: vec![], + environment_variable_files: BTreeMap::new(), + } + } + + fn ctx_for(canister_id: Principal) -> SyncStepContext { + SyncStepContext { + canister_path: PathBuf::from("/project"), + canister_id, + canister_name: "backend".to_owned(), + environment: "local".to_owned(), + network: "local".to_owned(), + canister_ids: BTreeMap::from([("backend".to_owned(), canister_id)]), + proxy: None, + } + } + + /// The bug fix: `sync_canister` must (re)apply the binding environment + /// variables *before* running any sync step, so standalone `icp sync` + /// includes the generated `PUBLIC_CANISTER_ID:*` variables. + #[tokio::test] + async fn sync_canister_applies_env_vars_before_steps() { + let log = Arc::new(Mutex::new(Log::default())); + let icp = MockIcp { log: log.clone() }; + let exec = MockExec { log: log.clone() }; + let canister = canister_with_binding(); + let cid = principal(); + let ctx = ctx_for(cid); + + sync_canister(&canister, cid, &ctx, &icp, &exec, &exec, None) + .await + .unwrap(); + + let events = &log.lock().unwrap().events; + assert_eq!( + events.as_slice(), + &["update_settings".to_owned(), "run_script".to_owned()], + "env vars must be applied before sync steps run" + ); + } + + /// `apply_binding_env_vars` merges manifest env vars with the generated + /// `PUBLIC_CANISTER_ID:` ids. + #[tokio::test] + async fn apply_binding_env_vars_merges_manifest_and_bindings() { + let log = Arc::new(Mutex::new(Log::default())); + let icp = MockIcp { log: log.clone() }; + let canister = canister_with_binding(); + let cid = principal(); + let canister_ids = BTreeMap::from([("backend".to_owned(), cid)]); + + apply_binding_env_vars(&canister, cid, &canister_ids, &icp) + .await + .unwrap(); + + let calls = &log.lock().unwrap().calls; + assert_eq!(calls.len(), 1); + let (method, arg) = &calls[0]; + assert_eq!(method, "update_settings"); + let (args,): (UpdateSettingsArgs,) = candid::decode_args(arg).unwrap(); + let vars: HashMap = args + .settings + .environment_variables + .unwrap() + .into_iter() + .map(|v| (v.name, v.value)) + .collect(); + assert_eq!(vars.get("FOO"), Some(&"bar".to_owned())); + assert_eq!(vars.get("PUBLIC_CANISTER_ID:backend"), Some(&cid.to_text())); + } + + /// A binding whose referenced canister has no id in this environment is + /// skipped rather than emitted with an empty value. + #[test] + fn unresolved_binding_is_skipped() { + let canister = canister_with_binding(); + let vars = binding_env_vars(&canister, &BTreeMap::new()); + let names: Vec<&str> = vars.iter().map(|v| v.name.as_str()).collect(); + assert_eq!(names, ["FOO"], "unresolved bindings must not be emitted"); + } +} diff --git a/crates/icp-deploy-canister/src/files.rs b/crates/icp-deploy-canister/src/files.rs new file mode 100644 index 000000000..546a11347 --- /dev/null +++ b/crates/icp-deploy-canister/src/files.rs @@ -0,0 +1,50 @@ +//! Abstracted filesystem access. +//! +//! Project loading, manifest consolidation, and reading built wasm artifacts all +//! go through [`FileAccess`] rather than touching the real filesystem, so the +//! same logic can run inside a canister (backed by, e.g., stable-memory blobs). +//! Paths are `camino` UTF-8 paths. + +use async_trait::async_trait; +use snafu::Snafu; + +use crate::prelude::*; + +#[derive(Debug, Snafu)] +pub enum FileAccessError { + #[snafu(display("failed to read file at '{path}': {message}"))] + Read { path: PathBuf, message: String }, + + #[snafu(display("failed to list directory at '{path}': {message}"))] + ReadDir { path: PathBuf, message: String }, +} + +/// Read-oriented filesystem access, rooted at the project directory. +/// +/// Predicate methods (`exists`/`is_file`/`is_dir`) return `false` on any error, +/// matching the `std::path` inherent methods they replace. `canonicalize` +/// returns `None` when the path cannot be resolved or is not valid UTF-8. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait FileAccess: Send + Sync { + /// Read the raw bytes of a file. + async fn read_file(&self, path: &Path) -> Result, FileAccessError>; + + /// Read a file as a UTF-8 string. + async fn read_to_string(&self, path: &Path) -> Result; + + async fn exists(&self, path: &Path) -> bool; + + async fn is_file(&self, path: &Path) -> bool; + + async fn is_dir(&self, path: &Path) -> bool; + + /// Non-recursive directory listing. Entries are returned as absolute paths + /// (the directory joined with each entry name). + async fn read_dir(&self, path: &Path) -> Result, FileAccessError>; + + /// Canonicalize a path (resolve `..` and symlinks). Returns `None` if the + /// path does not exist or does not resolve to valid UTF-8; callers treat + /// that as "cannot establish identity", which is safe for de-duplication. + async fn canonicalize(&self, path: &Path) -> Option; +} diff --git a/crates/icp-deploy-canister/src/icp_access.rs b/crates/icp-deploy-canister/src/icp_access.rs new file mode 100644 index 000000000..0cad5c597 --- /dev/null +++ b/crates/icp-deploy-canister/src/icp_access.rs @@ -0,0 +1,60 @@ +//! Abstracted access to the ICP API. +//! +//! [`IcpAccess`] is a dumb transport: this crate encodes/decodes Candid and +//! decides *what* to call (including all management-canister calls), while the +//! implementation only routes bytes. Proxy routing is owned by the impl +//! (constructed with the proxy principal); the caller never threads a proxy +//! through per call. + +use async_trait::async_trait; +use candid::Principal; +use snafu::Snafu; + +#[derive(Debug, Snafu)] +pub enum IcpAccessError { + #[snafu(display("update call to '{method}' on canister '{canister}' failed: {message}"))] + Update { + canister: Principal, + method: String, + message: String, + }, + + #[snafu(display("failed to read metadata '{path}' from canister '{canister}': {message}"))] + ReadMetadata { + canister: Principal, + path: String, + message: String, + }, +} + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait IcpAccess: Send + Sync { + /// Perform an update call and return the raw reply bytes. + /// + /// `effective_canister_id` is the canister used for request routing. For a + /// normal application call it equals `canister`; for a management-canister + /// call (`canister == aaaaa-aa`) it must be the *target* canister, since the + /// management canister has no routing of its own. `cycles` is attached to + /// the call (only meaningful for proxied/funded calls such as + /// `create_canister`). + async fn canister_update( + &self, + canister: Principal, + method: &str, + arg: Vec, + effective_canister_id: Principal, + cycles: u128, + ) -> Result, IcpAccessError>; + + /// Read a canister's custom-section metadata (via `read_state`). Returns + /// `None` when the section is absent. Used for EOP-upgrade detection. + async fn read_canister_metadata( + &self, + canister: Principal, + path: &str, + ) -> Result>, IcpAccessError>; + + /// The caller's (identity's) principal. + fn caller_principal(&self) -> Principal; +} diff --git a/crates/icp-deploy-canister/src/ids.rs b/crates/icp-deploy-canister/src/ids.rs new file mode 100644 index 000000000..4665f445e --- /dev/null +++ b/crates/icp-deploy-canister/src/ids.rs @@ -0,0 +1,42 @@ +//! Canister-id store: per-environment `name → principal` mappings. + +use std::collections::BTreeMap; + +use candid::Principal; +use snafu::Snafu; + +/// Mapping of canister names to their principals within an environment. +pub type IdMapping = BTreeMap; + +#[derive(Debug, Snafu)] +pub enum IdStoreError { + #[snafu(display("could not find id for canister '{canister_name}' in environment '{env}'"))] + NotFound { env: String, canister_name: String }, + + #[snafu(display("failed to access canister id store for environment '{env}': {message}"))] + Access { env: String, message: String }, +} + +/// Read/write access to canister-id mappings. +/// +/// The `is_cache` flag lets an implementation that keeps two stores — a +/// managed-network cache and a connected-network data store — pick the right +/// one. `register` mutates through `&self`, so the store is interior-mutable. +pub trait IdStore: Send + Sync { + fn lookup( + &self, + is_cache: bool, + env: &str, + canister_name: &str, + ) -> Result; + + fn lookup_by_environment(&self, is_cache: bool, env: &str) -> Result; + + fn register( + &self, + is_cache: bool, + env: &str, + canister_name: &str, + canister_id: Principal, + ) -> Result<(), IdStoreError>; +} diff --git a/crates/icp-deploy-canister/src/lib.rs b/crates/icp-deploy-canister/src/lib.rs new file mode 100644 index 000000000..2445e37b1 --- /dev/null +++ b/crates/icp-deploy-canister/src/lib.rs @@ -0,0 +1,204 @@ +//! Canister installation, syncing, and the project model, with all host IO +//! abstracted behind trait objects so the core can run inside a canister. +//! +//! See the module-level docs on the IO traits ([`files`], [`icp_access`], +//! [`ids`]) for the abstraction boundary. + +use std::collections::{BTreeMap, HashMap}; + +use indexmap::IndexMap; +use serde::Serialize; +use snafu::prelude::*; + +use candid_parser::parse_idl_args; + +use crate::{ + canister::Settings, + manifest::{ + ArgsFormat, + canister::{BuildSteps, SyncSteps}, + }, + network::Configuration, + prelude::*, +}; + +pub mod canister; +pub mod deploy; +pub mod files; +pub mod icp_access; +pub mod ids; +pub mod manifest; +pub mod network; +pub mod parsers; +pub mod prelude; +pub mod project; +pub mod sync_exec; + +#[cfg(test)] +mod testutil; + +pub use deploy::{ + DeployCanisterError, DeployError, InstallCanisterError, InstallMode, SyncCanisterError, + SyncStepError, apply_binding_env_vars, binding_env_vars, deploy, deploy_canister, + install_canister, install_canister_resolved, install_canister_wasm, + resolve_install_mode_and_status, run_sync_steps, start_canister, sync_canister, +}; +pub use files::{FileAccess, FileAccessError}; +pub use icp_access::{IcpAccess, IcpAccessError}; +pub use ids::{IdMapping, IdStore, IdStoreError}; +pub use project::{consolidate_manifest, load_project, verify_sandbox}; +pub use sync_exec::{ + PluginExecutor, PluginExecutorError, PluginInvocation, ScriptInvocation, ScriptRunError, + ScriptRunner, StepProgress, SyncStepContext, system_env_vars, +}; + +/// Resolved initialization arguments, with any file references already loaded. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub enum InitArgs { + /// Text content (inline or loaded from file). Format is always known. + Text { content: String, format: ArgsFormat }, + /// Raw binary bytes (from a file with `format: bin`). Used directly. + Binary(Vec), +} + +#[derive(Debug, Snafu)] +pub enum InitArgsToBytesError { + #[snafu(display("failed to decode hex init args"))] + HexDecode { source: hex::FromHexError }, + + #[snafu(display("failed to parse Candid init args"))] + CandidParse { source: candid_parser::Error }, + + #[snafu(display("failed to encode Candid init args to bytes"))] + CandidEncode { source: candid::Error }, +} + +impl InitArgs { + /// Resolve to raw bytes according to the format. + pub fn to_bytes(&self) -> Result, InitArgsToBytesError> { + match self { + InitArgs::Binary(bytes) => Ok(bytes.clone()), + InitArgs::Text { content, format } => match format { + ArgsFormat::Hex => hex::decode(content.trim()).context(HexDecodeSnafu), + ArgsFormat::Candid => { + let args = parse_idl_args(content.trim()).context(CandidParseSnafu)?; + args.to_bytes().context(CandidEncodeSnafu) + } + ArgsFormat::Bin => { + unreachable!("binary format cannot appear in InitArgs::Text") + } + }, + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Canister { + pub name: String, + + /// Canister settings, such as memory constaints, etc. + pub settings: Settings, + + /// The build configuration specifying how to compile the canister's source + /// code into a WebAssembly module, including the adapter to use. + pub build: BuildSteps, + + /// The configuration specifying how to sync the canister + pub sync: SyncSteps, + + /// Initialization arguments passed to the canister during installation. + /// Resolved from the manifest — file contents are already loaded. + pub init_args: Option, + + /// If the canister was defined via a recipe reference, this holds the + /// original recipe specifier string (e.g. `@dfinity/motoko@v4.0.0`). + /// `None` when the canister uses explicit build/sync instructions. + pub registry_recipe: Option, + + /// Canister-discovery wiring. Maps the name this canister reads in a + /// `PUBLIC_CANISTER_ID:` environment variable to the store key of the + /// referenced canister. Computed during consolidation so each canister sees + /// the view its owning project expects: its own project's canisters under + /// their local names, plus any declared dependencies under their aliases + /// (`:`). For a project with no dependencies this maps every + /// canister's local name to itself, reproducing the flat "every canister sees + /// every sibling" behavior. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub bindings: BTreeMap, + + /// Subdomain prefixes for the canister's friendly URLs, most-specific label + /// first, e.g. `["backend"]` for an own canister or `["backend.openemail"]` + /// for a dependency canister (dot-nested by alias chain). A de-duplicated + /// shared dependency canister carries one entry per alias chain that reaches + /// it. Consumed only at deploy time to build `custom-domains.txt` entries and + /// the printed URLs; a runtime display aid that is always recomputed during + /// consolidation, so it is never serialized. + #[serde(skip)] + pub friendly_names: Vec, + + /// For each environment variable whose value came from a file, the file it + /// was read from. `settings.environment_variables` already holds the + /// contents; the paths are kept so `icp project bundle` can hold a file + /// backing a variable to the same containment rule it applies to every other + /// file a manifest points at. Bookkeeping for that check rather than part of + /// the resolved configuration, so it is never serialized. + #[serde(skip)] + pub environment_variable_files: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Network { + pub name: String, + pub configuration: Configuration, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Environment { + pub name: String, + pub network: Network, + pub canisters: IndexMap, +} + +impl Environment { + pub fn get_canister_names(&self) -> Vec { + self.canisters.keys().cloned().collect() + } + + pub fn contains_canister(&self, canister_name: &str) -> bool { + self.canisters.contains_key(canister_name) + } + + pub fn get_canister_info(&self, canister: &str) -> Result<(PathBuf, Canister), String> { + self.canisters + .get(canister) + .ok_or_else(|| { + format!( + "canister '{}' not declared in environment '{}'", + canister, self.name + ) + }) + .cloned() + } +} + +/// Consolidated project definition +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Project { + pub dir: PathBuf, + pub canisters: IndexMap, + pub networks: HashMap, + pub environments: HashMap, + + /// Environments the workspace defines that some vendored member does *not* + /// declare, keyed by environment name → the missing members' store-key + /// prefixes. Enforced when the environment is selected (strict rule). + /// Empty for standalone projects and workspaces whose members are complete. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub member_missing_envs: HashMap>, +} + +impl Project { + pub fn get_canister(&self, canister_name: &str) -> Option<&(PathBuf, Canister)> { + self.canisters.get(canister_name) + } +} diff --git a/crates/icp/src/manifest/adapter/mod.rs b/crates/icp-deploy-canister/src/manifest/adapter/mod.rs similarity index 100% rename from crates/icp/src/manifest/adapter/mod.rs rename to crates/icp-deploy-canister/src/manifest/adapter/mod.rs diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp-deploy-canister/src/manifest/adapter/plugin.rs similarity index 100% rename from crates/icp/src/manifest/adapter/plugin.rs rename to crates/icp-deploy-canister/src/manifest/adapter/plugin.rs diff --git a/crates/icp/src/manifest/adapter/prebuilt.rs b/crates/icp-deploy-canister/src/manifest/adapter/prebuilt.rs similarity index 100% rename from crates/icp/src/manifest/adapter/prebuilt.rs rename to crates/icp-deploy-canister/src/manifest/adapter/prebuilt.rs diff --git a/crates/icp/src/manifest/adapter/script.rs b/crates/icp-deploy-canister/src/manifest/adapter/script.rs similarity index 100% rename from crates/icp/src/manifest/adapter/script.rs rename to crates/icp-deploy-canister/src/manifest/adapter/script.rs diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp-deploy-canister/src/manifest/canister.rs similarity index 100% rename from crates/icp/src/manifest/canister.rs rename to crates/icp-deploy-canister/src/manifest/canister.rs diff --git a/crates/icp/src/manifest/dependency.rs b/crates/icp-deploy-canister/src/manifest/dependency.rs similarity index 100% rename from crates/icp/src/manifest/dependency.rs rename to crates/icp-deploy-canister/src/manifest/dependency.rs diff --git a/crates/icp/src/manifest/environment.rs b/crates/icp-deploy-canister/src/manifest/environment.rs similarity index 100% rename from crates/icp/src/manifest/environment.rs rename to crates/icp-deploy-canister/src/manifest/environment.rs diff --git a/crates/icp-deploy-canister/src/manifest/mod.rs b/crates/icp-deploy-canister/src/manifest/mod.rs new file mode 100644 index 000000000..99bc680a7 --- /dev/null +++ b/crates/icp-deploy-canister/src/manifest/mod.rs @@ -0,0 +1,132 @@ +use std::marker::PhantomData; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use snafu::prelude::*; + +use crate::files::{FileAccess, FileAccessError}; +use crate::prelude::*; + +pub mod adapter; +pub mod canister; +pub mod dependency; +pub mod environment; +pub mod network; +pub mod project; +pub mod recipe; +pub mod serde_helpers; + +pub use { + adapter::plugin, + adapter::prebuilt, + canister::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, + SyncStep, SyncSteps, + }, + dependency::DependencyManifest, + environment::EnvironmentManifest, + network::{ManagedMode, Mode, NetworkManifest}, + project::ProjectManifest, +}; + +pub const PROJECT_MANIFEST: &str = "icp.yaml"; +pub const CANISTER_MANIFEST: &str = "canister.yaml"; + +#[derive(Debug, Snafu)] +pub enum LoadManifestError { + #[snafu(transparent)] + Read { source: FileAccessError }, + + #[snafu(display("failed to parse manifest at '{path}'"))] + Parse { + source: serde_yaml::Error, + path: PathBuf, + }, +} + +/// Load and parse a YAML manifest of type `T` through the injected [`FileAccess`]. +pub async fn load_manifest(files: &dyn FileAccess, path: &Path) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let content = files.read_file(path).await?; + let m = serde_yaml::from_slice::(&content).context(ParseSnafu { + path: path.to_path_buf(), + })?; + Ok(m) +} + +// A manifest item that can either be a path to another manifest file or the manifest itself. +// +// The valid path specifications are: +// - CanisterManifest: path or glob pattern to the directory containing "canister.yaml" +// - NetworkManifest: path to network manifest +// - EnvironmentManifest: path to environment manifest +#[derive(Clone, Debug, PartialEq, JsonSchema)] +#[serde(untagged)] +pub enum Item { + /// Path to a manifest + Path(String), + + /// The manifest + Manifest(T), +} + +/// Items in path form serialize back to a bare path string, *not* to the contents of the +/// referenced file. Callers that need a self-contained YAML output (e.g. `icp project bundle`) +/// must convert any `Item::Path` to `Item::Manifest` themselves by loading the referenced +/// manifest first. +impl Serialize for Item { + fn serialize(&self, serializer: S) -> Result { + match self { + Item::Path(p) => p.serialize(serializer), + Item::Manifest(m) => m.serialize(serializer), + } + } +} + +impl<'de, T> Deserialize<'de> for Item +where + T: Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::{MapAccess, Visitor, value::MapAccessDeserializer}; + use std::fmt; + + struct ItemVisitor(PhantomData); + + impl<'de, T: Deserialize<'de>> Visitor<'de> for ItemVisitor { + type Value = Item; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string path or a manifest object") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(Item::Path(v.to_owned())) + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + Ok(Item::Path(v)) + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + T::deserialize(MapAccessDeserializer::new(map)).map(Item::Manifest) + } + } + + deserializer.deserialize_any(ItemVisitor(PhantomData)) + } +} diff --git a/crates/icp/src/manifest/network.rs b/crates/icp-deploy-canister/src/manifest/network.rs similarity index 100% rename from crates/icp/src/manifest/network.rs rename to crates/icp-deploy-canister/src/manifest/network.rs diff --git a/crates/icp/src/manifest/project.rs b/crates/icp-deploy-canister/src/manifest/project.rs similarity index 100% rename from crates/icp/src/manifest/project.rs rename to crates/icp-deploy-canister/src/manifest/project.rs diff --git a/crates/icp/src/manifest/recipe.rs b/crates/icp-deploy-canister/src/manifest/recipe.rs similarity index 100% rename from crates/icp/src/manifest/recipe.rs rename to crates/icp-deploy-canister/src/manifest/recipe.rs diff --git a/crates/icp/src/manifest/serde_helpers.rs b/crates/icp-deploy-canister/src/manifest/serde_helpers.rs similarity index 100% rename from crates/icp/src/manifest/serde_helpers.rs rename to crates/icp-deploy-canister/src/manifest/serde_helpers.rs diff --git a/crates/icp-deploy-canister/src/network/mod.rs b/crates/icp-deploy-canister/src/network/mod.rs new file mode 100644 index 000000000..4f21d3e7f --- /dev/null +++ b/crates/icp-deploy-canister/src/network/mod.rs @@ -0,0 +1,368 @@ +//! Network *configuration* model (the manifest-derived view of a network). +//! +//! Runtime concerns — launching/stopping managed networks, network descriptors, +//! agent access — live in the host `icp` crate. + +use schemars::JsonSchema; +use serde::{Deserialize, Deserializer, Serialize}; +use strum::EnumString; +use url::Url; + +pub use crate::manifest::network::RootKeySpec; +use crate::manifest::network::{ + Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode, +}; + +pub const DEFAULT_LOCAL_NETWORK_BIND: &str = "127.0.0.1"; +pub const DEFAULT_LOCAL_NETWORK_PORT: u16 = 8000; + +#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] +pub enum Port { + Fixed(u16), + Random, +} + +impl Default for Port { + fn default() -> Self { + Port::Fixed(8000) + } +} + +impl<'de> Deserialize<'de> for Port { + fn deserialize>(d: D) -> Result { + Ok(match u16::deserialize(d)? { + 0 => Port::Random, + p => Port::Fixed(p), + }) + } +} + +fn default_bind() -> String { + "127.0.0.1".to_string() +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Gateway { + #[serde(default = "default_bind")] + pub bind: String, + + #[serde(default)] + pub port: Port, + + #[serde(default)] + pub domains: Vec, +} + +impl Default for Gateway { + fn default() -> Self { + Self { + bind: default_bind(), + port: Default::default(), + domains: Default::default(), + } + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct Managed { + #[serde(flatten)] + pub mode: ManagedMode, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +#[serde(untagged)] +pub enum ManagedMode { + Image(Box), + Launcher(Box), +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct ManagedLauncherConfig { + pub gateway: Gateway, + pub artificial_delay_ms: Option, + pub ii: bool, + pub nns: bool, + pub subnets: Option>, + pub bitcoind_addr: Option>, + pub dogecoind_addr: Option>, + pub version: Option, +} + +#[derive( + Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize, EnumString, strum::Display, +)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum SubnetKind { + Application, + System, + VerifiedApplication, + Bitcoin, + Fiduciary, + Nns, + Sns, +} + +impl Default for ManagedMode { + fn default() -> Self { + Self::default_for_port(DEFAULT_LOCAL_NETWORK_PORT) + } +} + +impl ManagedMode { + pub fn default_for_port(port: u16) -> Self { + ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway: Gateway { + bind: default_bind(), + port: if port == 0 { + Port::Random + } else { + Port::Fixed(port) + }, + domains: vec![], + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + version: None, + })) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +pub struct ManagedImageConfig { + pub image: String, + pub port_mapping: Vec, + pub rm_on_exit: bool, + pub args: Vec, + pub entrypoint: Option>, + pub environment: Vec, + pub volumes: Vec, + pub platform: Option, + pub user: Option, + pub shm_size: Option, + pub status_dir: String, + pub mounts: Vec, + pub extra_hosts: Vec, +} + +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Connected { + /// The URL this network's API can be reached at. + pub api_url: Url, + + /// The URL this network's HTTP gateway can be reached at. + pub http_gateway_url: Option, + + /// How to obtain the root key used to verify responses from this network. + pub root_key: RootKeySpec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] +#[serde(tag = "mode", rename_all = "lowercase")] +pub enum Configuration { + // Note: we must use struct variants to be able to flatten + // and make schemars generate the proper schema + /// A managed network is one which can be controlled and manipulated. + Managed { + #[serde(flatten)] + managed: Managed, + }, + + /// A connected network is one which can be interacted with + /// but cannot be controlled or manipulated. + Connected { + #[serde(flatten)] + connected: Connected, + }, +} + +impl Default for Configuration { + fn default() -> Self { + Configuration::Managed { + managed: Managed::default(), + } + } +} + +impl From for Gateway { + fn from(value: ManifestGateway) -> Self { + let ManifestGateway { + bind, + domains, + port, + } = value; + let bind = bind.unwrap_or("127.0.0.1".to_string()); + let port = match port { + Some(0) => Port::Random, + Some(p) => Port::Fixed(p), + None => Port::default(), + }; + let mut domains = domains.unwrap_or_default(); + if bind == "127.0.0.1" || bind == "0.0.0.0" || bind == "::1" || bind == "::" { + domains.insert(0, "localhost".to_string()); + } + Gateway { + bind, + port, + domains, + } + } +} + +impl From for Connected { + fn from(value: ManifestConnected) -> Self { + let root_key = value.root_key; + match value.endpoints { + Endpoints::Implicit { url } => Connected { + api_url: url.clone(), + http_gateway_url: Some(url), + root_key, + }, + Endpoints::Explicit { + api_url, + http_gateway_url, + } => Connected { + api_url, + http_gateway_url, + root_key, + }, + } + } +} + +impl From for Configuration { + fn from(value: Mode) -> Self { + match value { + Mode::Managed(managed) => match *managed.mode { + crate::manifest::network::ManagedMode::Launcher { + gateway, + artificial_delay_ms, + ii, + nns, + subnets, + bitcoind_addr, + dogecoind_addr, + version, + } => { + let gateway: Gateway = match gateway { + Some(g) => g.into(), + None => Gateway::default(), + }; + let version = match version { + Some(v) => { + if v.starts_with('v') { + Some(v) + } else { + Some(format!("v{v}")) + } + } + None => None, + }; + Configuration::Managed { + managed: Managed { + mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway, + artificial_delay_ms, + ii: ii.unwrap_or(false), + nns: nns.unwrap_or(false), + subnets, + bitcoind_addr, + dogecoind_addr, + version, + })), + }, + } + } + crate::manifest::network::ManagedMode::Image { + image, + port_mapping, + rm_on_exit, + args, + entrypoint, + environment, + volumes, + platform, + user, + shm_size, + status_dir, + mounts: mount, + extra_hosts, + } => Configuration::Managed { + managed: Managed { + mode: ManagedMode::Image(Box::new(ManagedImageConfig { + image, + port_mapping, + rm_on_exit: rm_on_exit.unwrap_or(false), + args: args.unwrap_or_default(), + entrypoint, + environment: environment.unwrap_or_default(), + volumes: volumes.unwrap_or_default(), + platform, + user, + shm_size, + status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), + mounts: mount.unwrap_or_default(), + extra_hosts: extra_hosts.unwrap_or_default(), + })), + }, + }, + }, + Mode::Connected(connected) => Configuration::Connected { + connected: connected.into(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::network::{ + Gateway as ManifestGateway, Managed as ManifestManaged, ManagedMode as ManifestManagedMode, + Mode, + }; + + #[test] + fn from_mode_launcher_with_bitcoind_addr() { + let mode = Mode::Managed(ManifestManaged { + mode: Box::new(ManifestManagedMode::Launcher { + gateway: Some(ManifestGateway { + bind: None, + port: Some(8000), + domains: None, + }), + artificial_delay_ms: None, + ii: None, + nns: None, + subnets: None, + bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), + dogecoind_addr: None, + version: None, + }), + }); + + let config: Configuration = mode.into(); + match config { + Configuration::Managed { + managed: + Managed { + mode: ManagedMode::Launcher(launcher_config), + }, + } => { + assert_eq!( + launcher_config.bitcoind_addr, + Some(vec!["127.0.0.1:18444".to_string()]) + ); + assert_eq!(launcher_config.dogecoind_addr, None); + assert!(!launcher_config.ii); + assert!(!launcher_config.nns); + } + _ => panic!("expected ManagedMode::Launcher"), + } + } +} diff --git a/crates/icp-deploy-canister/src/parsers.rs b/crates/icp-deploy-canister/src/parsers.rs new file mode 100644 index 000000000..b0a74730f --- /dev/null +++ b/crates/icp-deploy-canister/src/parsers.rs @@ -0,0 +1,643 @@ +//! Parsing of token, cycle, memory, and duration amounts with support for suffixes and underscores. + +use bigdecimal::{BigDecimal, Signed}; +use num_bigint::BigUint; +use num_integer::Integer; +use num_traits::{ToPrimitive, Zero}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::str::FromStr; + +/// Parse a token amount with support for suffixes (k, m, b, t) and underscores. +/// +/// Examples: +/// - `1` -> 1 +/// - `1_000` -> 1000 +/// - `1k` or `1K` -> 1000 +/// - `1t` or `1T` -> 1000000000000 +/// - `0.5` -> 0.5 +/// - `0.5k` -> 500 +pub fn parse_token_amount(input: &str) -> Result { + let input = input.trim(); + + if input.is_empty() { + return Err("Token amount cannot be empty".to_string()); + } + + let (number_part, multiplier) = if let Some(last_char) = input.chars().last() { + match last_char { + 'k' | 'K' => (&input[..input.len() - 1], 1_000u128), + 'm' | 'M' => (&input[..input.len() - 1], 1_000_000u128), + 'b' | 'B' => (&input[..input.len() - 1], 1_000_000_000u128), + 't' | 'T' => (&input[..input.len() - 1], 1_000_000_000_000u128), + _ => (input, 1u128), + } + } else { + (input, 1u128) + }; + + let cleaned = number_part.replace('_', ""); + let base = + BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid token amount: '{}'", input))?; + + if base.is_negative() { + return Err(format!("Token amount cannot be negative: '{}'", input)); + } + + let multiplier_decimal = BigDecimal::from(multiplier); + Ok(base * multiplier_decimal) +} + +/// Convert a token amount to the smallest unit by multiplying by 10^token_decimals. +/// E.g. 1.5 with 8 decimals -> 150000000. Fails if the result would be fractional. +pub fn to_token_unit_amount( + token_amount: BigDecimal, + token_decimals: u8, +) -> Result { + use num_bigint::BigInt; + use num_traits::pow::Pow; + + let (mantissa, exponent) = token_amount.into_bigint_and_exponent(); + let scale_adjustment = token_decimals as i64 - exponent; + let ten = BigInt::from(10); + + let result = if scale_adjustment >= 0 { + let multiplier = ten.pow(scale_adjustment as u32); + mantissa * multiplier + } else { + let divisor = ten.pow((-scale_adjustment) as u32); + let (quotient, remainder) = mantissa.div_rem(&divisor); + if !remainder.is_zero() { + return Err(format!( + "Token amount cannot be represented with {} decimals (would result in fractional units)", + token_decimals + )); + } + quotient + }; + + result + .try_into() + .map_err(|_| "Token amount cannot be negative".to_string()) +} + +fn parse_cycles_str(s: &str) -> Result { + let token_amount = parse_token_amount(s)?; + let unit_amount = to_token_unit_amount(token_amount, 0)?; + unit_amount + .to_u128() + .ok_or_else(|| format!("Cycles amount too large: '{}'", s)) +} + +/// An amount of cycles. +/// +/// Deserializes from a number or a string with suffixes (k, m, b, t) and optional underscore separators. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum CyclesAmount { + Number(u64), // yaml only supports up to u64 + Str(String), +} + +impl CyclesAmount { + pub fn get(&self) -> u128 { + match self { + CyclesAmount::Number(n) => *n as u128, + CyclesAmount::Str(s) => parse_cycles_str(s) + .unwrap_or_else(|e| panic!("invalid cycles amount '{}': {}", s, e)), + } + } +} + +impl<'de> Deserialize<'de> for CyclesAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + // Identical enum to CyclesAmount. Needed to avoid a circular dependency. + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom("cycles amount must be a number or a string with optional suffix (k, m, b, t), e.g. 1000 or \"4t\"") + })?; + let c = match v { + Raw::Number(n) => CyclesAmount::Number(n), + Raw::Str(ref s) => { + parse_cycles_str(s).map_err(serde::de::Error::custom)?; // validate the string is a valid cycles amount + CyclesAmount::Str(s.clone()) + } + }; + Ok(c) + } +} + +impl Serialize for CyclesAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + CyclesAmount::Number(n) => serializer.serialize_u64(*n), + CyclesAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for CyclesAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_cycles_str(s)?; // validate the string is a valid cycles amount + Ok(CyclesAmount::Str(s.to_string())) + } +} + +impl fmt::Display for CyclesAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u128 { + fn from(c: CyclesAmount) -> Self { + c.get() + } +} + +impl From for CyclesAmount { + fn from(n: u128) -> Self { + if let Ok(n64) = u64::try_from(n) { + CyclesAmount::Number(n64) + } else { + CyclesAmount::Str(n.to_string()) + } + } +} + +const KB: u64 = 1000; +const KIB: u64 = 1024; +const MB: u64 = 1_000_000; +const MIB: u64 = 1024 * 1024; +const GB: u64 = 1_000_000_000; +const GIB: u64 = 1024 * 1024 * 1024; + +fn parse_memory_str(s: &str) -> Result { + let s = s.trim(); + if s.is_empty() { + return Err("Memory amount cannot be empty".to_string()); + } + let lower = s.to_lowercase(); + let (number_part, factor) = if lower.ends_with("gib") { + (&s[..s.len() - 3], GIB) + } else if lower.ends_with("gb") { + (&s[..s.len() - 2], GB) + } else if lower.ends_with("mib") { + (&s[..s.len() - 3], MIB) + } else if lower.ends_with("mb") { + (&s[..s.len() - 2], MB) + } else if lower.ends_with("kib") { + (&s[..s.len() - 3], KIB) + } else if lower.ends_with("kb") { + (&s[..s.len() - 2], KB) + } else { + (s, 1u64) + }; + let cleaned = number_part.trim().replace('_', ""); + let amount = + BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid memory amount: '{}'", s))?; + if amount.is_negative() { + return Err(format!("Memory amount cannot be negative: '{}'", s)); + } + let product = amount * BigDecimal::from(factor); + if !product.is_integer() { + return Err( + "Memory amount must be a whole number of bytes (fractional bytes not allowed)" + .to_string(), + ); + } + product + .to_u64() + .ok_or_else(|| format!("Memory amount too large: '{}'", s)) +} + +/// An amount of memory in bytes. +/// +/// Deserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib), +/// optional decimals, and optional underscore separators. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum MemoryAmount { + Number(u64), + Str(String), +} + +impl MemoryAmount { + pub fn get(&self) -> u64 { + match self { + MemoryAmount::Number(n) => *n, + MemoryAmount::Str(s) => parse_memory_str(s) + .unwrap_or_else(|e| panic!("invalid memory amount '{}': {}", s, e)), + } + } +} + +impl<'de> Deserialize<'de> for MemoryAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom( + "memory amount must be a number or a string with optional suffix (kb, kib, mb, mib, gb, gib), e.g. 1024 or \"2.5gib\"", + ) + })?; + let m = match v { + Raw::Number(n) => MemoryAmount::Number(n), + Raw::Str(ref s) => { + parse_memory_str(s).map_err(serde::de::Error::custom)?; + MemoryAmount::Str(s.clone()) + } + }; + Ok(m) + } +} + +impl Serialize for MemoryAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + MemoryAmount::Number(n) => serializer.serialize_u64(*n), + MemoryAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for MemoryAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_memory_str(s)?; + Ok(MemoryAmount::Str(s.to_string())) + } +} + +impl fmt::Display for MemoryAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u64 { + fn from(m: MemoryAmount) -> Self { + m.get() + } +} + +impl From for MemoryAmount { + fn from(n: u64) -> Self { + MemoryAmount::Number(n) + } +} + +const SECONDS_PER_MINUTE: u64 = 60; +const SECONDS_PER_HOUR: u64 = 3600; +const SECONDS_PER_DAY: u64 = 86400; +const SECONDS_PER_WEEK: u64 = 604800; + +fn parse_duration_str(s: &str) -> Result { + let s = s.trim(); + if s.is_empty() { + return Err("Duration cannot be empty".to_string()); + } + let lower = s.to_lowercase(); + let (number_part, factor) = if lower.ends_with('w') { + (&s[..s.len() - 1], SECONDS_PER_WEEK) + } else if lower.ends_with('d') { + (&s[..s.len() - 1], SECONDS_PER_DAY) + } else if lower.ends_with('h') { + (&s[..s.len() - 1], SECONDS_PER_HOUR) + } else if lower.ends_with('m') { + (&s[..s.len() - 1], SECONDS_PER_MINUTE) + } else if lower.ends_with('s') { + (&s[..s.len() - 1], 1u64) + } else { + (s, 1u64) + }; + let cleaned = number_part.trim().replace('_', ""); + if cleaned.is_empty() { + return Err(format!("Invalid duration: '{s}'")); + } + let value: u64 = cleaned + .parse() + .map_err(|_| format!("Invalid duration: '{s}'"))?; + value + .checked_mul(factor) + .ok_or_else(|| format!("Duration too large: '{s}'")) +} + +/// A duration in seconds. +/// +/// Deserializes from a number (seconds) or a string with duration suffix (s, m, h, d, w) +/// and optional underscore separators. +/// +/// Suffixes (case-insensitive): +/// - `s` — seconds +/// - `m` — minutes (×60) +/// - `h` — hours (×3600) +/// - `d` — days (×86400) +/// - `w` — weeks (×604800) +/// +/// A bare number without suffix is treated as seconds. +#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] +#[schemars(untagged)] +pub enum DurationAmount { + Number(u64), + Str(String), +} + +impl DurationAmount { + pub fn get(&self) -> u64 { + match self { + DurationAmount::Number(n) => *n, + DurationAmount::Str(s) => { + parse_duration_str(s).unwrap_or_else(|e| panic!("invalid duration '{}': {}", s, e)) + } + } + } +} + +impl<'de> Deserialize<'de> for DurationAmount { + fn deserialize(d: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Number(u64), + Str(String), + } + let v = Raw::deserialize(d).map_err(|_| { + serde::de::Error::custom( + "duration must be a number (seconds) or a string with optional suffix (s, m, h, d, w), e.g. 2592000 or \"30d\"", + ) + })?; + let c = match v { + Raw::Number(n) => DurationAmount::Number(n), + Raw::Str(ref s) => { + parse_duration_str(s).map_err(serde::de::Error::custom)?; + DurationAmount::Str(s.clone()) + } + }; + Ok(c) + } +} + +impl Serialize for DurationAmount { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + DurationAmount::Number(n) => serializer.serialize_u64(*n), + DurationAmount::Str(s) => serializer.serialize_str(s), + } + } +} + +impl FromStr for DurationAmount { + type Err = String; + + fn from_str(s: &str) -> Result { + parse_duration_str(s)?; + Ok(DurationAmount::Str(s.to_string())) + } +} + +impl fmt::Display for DurationAmount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.get().fmt(f) + } +} + +impl From for u64 { + fn from(d: DurationAmount) -> Self { + d.get() + } +} + +impl From for DurationAmount { + fn from(n: u64) -> Self { + DurationAmount::Number(n) + } +} + +impl PartialEq for DurationAmount { + fn eq(&self, other: &u64) -> bool { + self.get() == *other + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cycles_amount_from_str_plain() { + assert_eq!("1".parse::().unwrap().get(), 1); + assert_eq!("1000".parse::().unwrap().get(), 1000); + } + + #[test] + fn cycles_amount_from_str_suffixes() { + assert_eq!("1k".parse::().unwrap().get(), 1000); + assert_eq!( + "1t".parse::().unwrap().get(), + 1_000_000_000_000 + ); + assert_eq!( + "4t".parse::().unwrap().get(), + 4_000_000_000_000 + ); + assert_eq!( + "0.5t".parse::().unwrap().get(), + 500_000_000_000 + ); + } + + #[test] + fn cycles_amount_from_str_underscores() { + assert_eq!("1_000".parse::().unwrap().get(), 1000); + } + + #[test] + fn cycles_amount_from_str_fractional_rejected() { + assert!("1.5".parse::().is_err()); + } + + #[test] + fn cycles_amount_deserialize() { + let yaml = "4t"; + let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(c.get(), 4_000_000_000_000); + + let yaml = "5000000000000"; + let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(c.get(), 5_000_000_000_000); + } + + #[test] + fn parse_token_amount_plain_and_suffixes() { + use std::str::FromStr; + assert_eq!( + parse_token_amount("1").unwrap(), + BigDecimal::from_str("1").unwrap() + ); + assert_eq!( + parse_token_amount("1k").unwrap(), + BigDecimal::from_str("1000").unwrap() + ); + assert_eq!( + parse_token_amount("0.5t").unwrap(), + BigDecimal::from_str("500000000000").unwrap() + ); + } + + #[test] + fn memory_amount_from_str_plain() { + assert_eq!("1".parse::().unwrap().get(), 1); + assert_eq!("1024".parse::().unwrap().get(), 1024); + } + + #[test] + fn memory_amount_from_str_suffixes() { + assert_eq!("1kb".parse::().unwrap().get(), 1000); + assert_eq!("1kib".parse::().unwrap().get(), 1024); + assert_eq!("1mb".parse::().unwrap().get(), 1_000_000); + assert_eq!("1mib".parse::().unwrap().get(), 1024 * 1024); + assert_eq!("1gb".parse::().unwrap().get(), 1_000_000_000); + assert_eq!( + "1gib".parse::().unwrap().get(), + 1024 * 1024 * 1024 + ); + assert_eq!( + "2 GiB".parse::().unwrap().get(), + 2 * 1024 * 1024 * 1024 + ); + } + + #[test] + fn memory_amount_from_str_decimals() { + assert_eq!("0.5kib".parse::().unwrap().get(), 512); + assert_eq!("1.5gib".parse::().unwrap().get(), 1610612736); + } + + #[test] + fn memory_amount_fractional_bytes_rejected() { + assert!("1.5".parse::().is_err()); // 1.5 bytes + assert!("0.3kib".parse::().is_err()); // 307.2 bytes + } + + #[test] + fn memory_amount_from_str_underscores() { + assert_eq!("1_024".parse::().unwrap().get(), 1024); + } + + #[test] + fn memory_amount_deserialize() { + let yaml = "2gib"; + let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(m.get(), 2 * 1024 * 1024 * 1024); + + let yaml = "4294967296"; + let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(m.get(), 4294967296); + } + + #[test] + fn duration_amount_from_str_plain() { + assert_eq!("60".parse::().unwrap().get(), 60); + assert_eq!("2592000".parse::().unwrap().get(), 2592000); + } + + #[test] + fn duration_amount_from_str_underscores() { + assert_eq!( + "2_592_000".parse::().unwrap().get(), + 2592000 + ); + } + + #[test] + fn duration_amount_from_str_suffixes() { + assert_eq!("60s".parse::().unwrap().get(), 60); + assert_eq!("90m".parse::().unwrap().get(), 5400); + assert_eq!("24h".parse::().unwrap().get(), 86400); + assert_eq!("30d".parse::().unwrap().get(), 2592000); + assert_eq!("4w".parse::().unwrap().get(), 2419200); + } + + #[test] + fn duration_amount_from_str_case_insensitive() { + assert_eq!("30D".parse::().unwrap().get(), 2592000); + assert_eq!("1W".parse::().unwrap().get(), 604800); + assert_eq!("24H".parse::().unwrap().get(), 86400); + assert_eq!("60S".parse::().unwrap().get(), 60); + assert_eq!("90M".parse::().unwrap().get(), 5400); + } + + #[test] + fn duration_amount_from_str_underscores_with_suffix() { + assert_eq!( + "2_592_000s".parse::().unwrap().get(), + 2592000 + ); + } + + #[test] + fn duration_amount_from_str_errors() { + assert!("abc".parse::().is_err()); + assert!("".parse::().is_err()); + assert!("1x".parse::().is_err()); + assert!("1.5d".parse::().is_err()); + assert!("-1d".parse::().is_err()); + } + + #[test] + fn duration_amount_deserialize() { + let yaml = "30d"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + + let yaml = "2592000"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + + let yaml = "2_592_000"; + let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(d.get(), 2592000); + } + + #[test] + fn duration_amount_partial_eq_u64() { + let d = DurationAmount::Number(2592000); + assert!(d == 2592000); + assert!(d != 0); + + let d = DurationAmount::Str("30d".to_string()); + assert!(d == 2592000); + } +} diff --git a/crates/icp-deploy-canister/src/prelude.rs b/crates/icp-deploy-canister/src/prelude.rs new file mode 100644 index 000000000..3cf6e17e0 --- /dev/null +++ b/crates/icp-deploy-canister/src/prelude.rs @@ -0,0 +1,13 @@ +pub use camino::{FromPathBufError, Utf8Path as Path, Utf8PathBuf as PathBuf}; + +pub const TRILLION: u128 = 1_000_000_000_000; + +pub const SECOND: u64 = 1; +pub const MINUTE: u64 = 60 * SECOND; + +pub const IC_MAINNET_NETWORK_API_URL: &str = "https://icp-api.io"; +pub const IC_MAINNET_NETWORK_GATEWAY_URL: &str = "https://icp.net"; +/// Name of the implicit IC mainnet network and its implicit environment +pub const IC: &str = "ic"; +/// Name of the implicit local managed network and its implicit environment +pub const LOCAL: &str = "local"; diff --git a/crates/icp-deploy-canister/src/project.rs b/crates/icp-deploy-canister/src/project.rs new file mode 100644 index 000000000..02a53f0ac --- /dev/null +++ b/crates/icp-deploy-canister/src/project.rs @@ -0,0 +1,2340 @@ +use std::collections::{BTreeMap, HashMap, HashSet, hash_map::Entry}; + +use indexmap::{IndexMap, map::Entry as IndexEntry}; + +use snafu::prelude::*; + +use crate::{ + Canister, Environment, InitArgs, Network, Project, + canister::{ControllerRef, ManifestEnvVar, ManifestSettings, Settings, recipe}, + files::{FileAccess, FileAccessError}, + manifest::{ + ArgsFormat, CANISTER_MANIFEST, CanisterManifest, DependencyManifest, EnvironmentManifest, + Item, LoadManifestError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, + ProjectManifest, + canister::{Instructions, SyncSteps}, + environment::CanisterSelection, + load_manifest, + network::RootKeySpec, + recipe::RecipeType, + }, + network::{ + Configuration, Connected, DEFAULT_LOCAL_NETWORK_BIND, DEFAULT_LOCAL_NETWORK_PORT, Gateway, + Managed, ManagedLauncherConfig, ManagedMode, Port, + }, + prelude::*, +}; + +#[derive(Debug, Snafu)] +pub enum EnvironmentError { + #[snafu(display("environment '{environment}' points to invalid network '{network}'"))] + InvalidNetwork { + environment: String, + network: String, + }, + + #[snafu(display("environment '{environment}' points to invalid canister '{canister}'"))] + InvalidCanister { + environment: String, + canister: String, + }, +} + +#[derive(Debug, Snafu)] +pub enum ConsolidateManifestError { + #[snafu(display("failed to parse glob pattern"))] + GlobParse { source: glob::PatternError }, + + #[snafu(display("failed to list directory while expanding a glob"))] + ListDir { source: FileAccessError }, + + #[snafu(display("failed to load canister manifest"))] + LoadCanister { source: LoadManifestError }, + + #[snafu(display("failed to load network manifest"))] + LoadNetwork { source: LoadManifestError }, + + #[snafu(display("failed to load environment manifest"))] + LoadEnvironment { source: LoadManifestError }, + + #[snafu(display("failed to load {kind} manifest at: {path}"))] + Failed { kind: String, path: String }, + + #[snafu(display("failed to fetch canister recipe: {recipe_type:?}"))] + FetchRecipe { + #[snafu(source(from(recipe::ResolveError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("failed to render canister recipe: {recipe_type:?}"))] + RenderRecipe { + #[snafu(source(from(recipe::RenderRecipeError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("failed to cache canister recipe: {recipe_type:?}"))] + CacheRecipe { + #[snafu(source(from(recipe::ResolveError, Box::new)))] + source: Box, + recipe_type: RecipeType, + }, + + #[snafu(display("project contains two similarly named {kind}s: '{name}'"))] + Duplicate { kind: String, name: String }, + + #[snafu(display("`{name}` is a reserved {kind} name."))] + Reserved { kind: String, name: String }, + + #[snafu(display("could not locate a {kind} manifest at: '{path}'"))] + NotFound { kind: String, path: String }, + + #[snafu(display("failed to read init_args file for canister '{canister}'"))] + ReadInitArgs { + source: FileAccessError, + canister: String, + }, + + #[snafu(display( + "failed to read the file backing environment variable '{variable}' of canister '{canister}'" + ))] + ReadEnvironmentVariable { + source: FileAccessError, + canister: String, + variable: String, + }, + + #[snafu(display( + "init_args for canister '{canister}' uses format 'bin' with inline content; \ + binary format requires a file path" + ))] + BinFormatInlineContent { canister: String }, + + #[snafu(display( + "canister '{canister}' lists controller '{controller}', but no canister with that \ + name is declared in the project" + ))] + UnknownControllerCanister { + canister: String, + controller: String, + }, + + #[snafu(display( + "canister name '{name}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ + (':' is reserved as the dependency namespace separator)" + ))] + InvalidCanisterName { name: String }, + + #[snafu(display( + "dependency alias '{alias}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ + (':' is reserved as the dependency namespace separator)" + ))] + InvalidDependencyAlias { alias: String }, + + #[snafu(display("project declares two dependencies with the same alias '{alias}'"))] + DuplicateDependencyAlias { alias: String }, + + #[snafu(display( + "dependency alias '{alias}' collides with a canister of the same name in the same project" + ))] + DependencyAliasCollision { alias: String }, + + #[snafu(display("could not find a project manifest for dependency '{alias}' at: '{path}'"))] + DependencyNotFound { alias: String, path: String }, + + #[snafu(display("failed to canonicalize path for dependency '{alias}' at: '{path}'"))] + DependencyCanonicalize { alias: String, path: String }, + + #[snafu(display("failed to load project manifest for dependency '{alias}'"))] + LoadDependencyManifest { + source: LoadManifestError, + alias: String, + }, + + #[snafu(display( + "dependency '{alias}' selects canister '{canister}', which the dependency does not declare" + ))] + UnknownDependencyCanister { alias: String, canister: String }, + + #[snafu(display("dependency cycle detected: {chain}"))] + CircularDependency { chain: String }, + + #[snafu(transparent)] + Environment { source: EnvironmentError }, +} + +/// Resolve a [`ManifestInitArgs`] into a canonical [`InitArgs`] by reading +/// any file references relative to `base_path`. +async fn resolve_manifest_init_args( + files: &dyn FileAccess, + manifest_init_args: &ManifestInitArgs, + base_path: &Path, + canister: &str, +) -> Result { + match manifest_init_args { + ManifestInitArgs::String(content) => Ok(InitArgs::Text { + content: content.trim().to_owned(), + format: ArgsFormat::Candid, + }), + ManifestInitArgs::Path { path, format } => { + let file_path = base_path.join(path); + match format { + ArgsFormat::Bin => { + let bytes = files + .read_file(&file_path) + .await + .context(ReadInitArgsSnafu { canister })?; + Ok(InitArgs::Binary(bytes)) + } + fmt => { + let content = files + .read_to_string(&file_path) + .await + .context(ReadInitArgsSnafu { canister })?; + Ok(InitArgs::Text { + content: content.trim().to_owned(), + format: fmt.clone(), + }) + } + } + } + ManifestInitArgs::Value { value, format } => match format { + ArgsFormat::Bin => BinFormatInlineContentSnafu { canister }.fail(), + fmt => Ok(InitArgs::Text { + content: value.trim().to_owned(), + format: fmt.clone(), + }), + }, + } +} + +/// Resolve a manifest's [`ManifestSettings`] into the model's [`Settings`] by +/// reading any file-backed environment variable values relative to `base_path`. +/// Also returns the file each such value came from, for +/// [`Canister::environment_variable_files`]. +async fn resolve_manifest_settings( + files: &dyn FileAccess, + manifest_settings: &ManifestSettings, + base_path: &Path, + canister: &str, +) -> Result<(Settings, BTreeMap), ConsolidateManifestError> { + let ManifestSettings { + log_visibility, + compute_allocation, + memory_allocation, + freezing_threshold, + reserved_cycles_limit, + wasm_memory_limit, + wasm_memory_threshold, + log_memory_limit, + environment_variables, + controllers, + } = manifest_settings; + + let mut env_files = BTreeMap::new(); + let mut resolved_vars = None; + if let Some(vars) = environment_variables { + let mut resolved = HashMap::with_capacity(vars.len()); + for (name, var) in vars { + let value = match var { + ManifestEnvVar::Value(value) => value.to_owned(), + ManifestEnvVar::Path { path } => { + let file = base_path.join(path); + let contents = files.read_to_string(&file).await.context( + ReadEnvironmentVariableSnafu { + canister, + variable: name, + }, + )?; + env_files.insert(name.to_owned(), file); + contents.trim().to_owned() + } + }; + resolved.insert(name.to_owned(), value); + } + resolved_vars = Some(resolved); + } + + let settings = Settings { + log_visibility: log_visibility.clone(), + compute_allocation: *compute_allocation, + memory_allocation: memory_allocation.clone(), + freezing_threshold: freezing_threshold.clone(), + reserved_cycles_limit: reserved_cycles_limit.clone(), + wasm_memory_limit: wasm_memory_limit.clone(), + wasm_memory_threshold: wasm_memory_threshold.clone(), + log_memory_limit: log_memory_limit.clone(), + environment_variables: resolved_vars, + controllers: controllers.clone(), + }; + Ok((settings, env_files)) +} + +fn is_glob(s: &str) -> bool { + s.contains('*') || s.contains('?') || s.contains('[') || s.contains('{') +} + +/// Collect `dir` and all of its descendant directories (recursively), used to +/// expand a `**` glob segment through the injected [`FileAccess`]. +async fn collect_descendant_dirs( + files: &dyn FileAccess, + dir: &Path, + out: &mut Vec, +) -> Result<(), ConsolidateManifestError> { + // Iterative BFS to avoid boxing a recursive async fn. + let mut queue = vec![dir.to_owned()]; + while let Some(d) = queue.pop() { + let entries = files.read_dir(&d).await.context(ListDirSnafu)?; + for entry in entries { + if files.is_dir(&entry).await { + out.push(entry.clone()); + queue.push(entry); + } + } + } + Ok(()) +} + +/// Expand a glob `pattern` (relative to `base`) into concrete paths, using the +/// injected [`FileAccess`] instead of the real filesystem. Supports literal +/// segments, single-segment wildcards (`*`, `?`, `[...]`, `{...}` via +/// [`glob::Pattern`]), and the `**` recursive segment. +async fn expand_glob( + files: &dyn FileAccess, + base: &Path, + pattern: &str, +) -> Result, ConsolidateManifestError> { + let mut current = vec![base.to_owned()]; + for seg in pattern.split('/') { + if seg.is_empty() { + continue; + } + let mut next = Vec::new(); + if seg == "**" { + for dir in ¤t { + next.push(dir.clone()); + collect_descendant_dirs(files, dir, &mut next).await?; + } + } else if is_glob(seg) { + let pat = glob::Pattern::new(seg).context(GlobParseSnafu)?; + for dir in ¤t { + if !files.is_dir(dir).await { + continue; + } + for entry in files.read_dir(dir).await.context(ListDirSnafu)? { + if let Some(name) = entry.file_name() + && pat.matches(name) + { + next.push(entry); + } + } + } + } else { + for dir in ¤t { + next.push(dir.join(seg)); + } + } + current = next; + } + Ok(current) +} + +/// Whether `name` is a valid canister name or dependency alias: non-empty and +/// containing only ASCII letters, digits, `_`, or `-`. +/// +/// A single strict rule keeps names safe for every purpose they are reused for — +/// store-key segments, `PUBLIC_CANISTER_ID:` env vars, DNS subdomains, and +/// archive paths — so no per-site sanitizing is needed. In particular `:` is the +/// dependency namespace separator, and `.` / `/` would be ambiguous in +/// subdomains and paths. +fn is_valid_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') +} + +/// Builds the canonical canisters declared directly in one project manifest, +/// resolving glob/path/inline entries, recipes, and init-args relative to +/// `pdir`. Returns `(local name, canister dir, canister)` with empty bindings; +/// callers assign store keys and bindings. Does not check for duplicate names +/// across projects — that is the caller's responsibility (via the global map). +async fn build_manifest_canisters( + files: &dyn FileAccess, + pdir: &Path, + manifest_canisters: &[Item], + recipe_resolver: &dyn recipe::RemoteResourceResolve, +) -> Result, ConsolidateManifestError> { + let mut result: Vec<(String, PathBuf, Canister)> = Vec::new(); + + for i in manifest_canisters { + let ms = match i { + Item::Path(pattern) => { + let is_glob_pattern = is_glob(pattern); + let paths = if is_glob_pattern { + expand_glob(files, pdir, pattern).await? + } else { + vec![pdir.join(pattern)] + }; + + let paths = if is_glob_pattern { + // For glob patterns, filter out non-directories and non-canister directories + let mut kept = Vec::new(); + for p in paths { + if files.is_dir(&p).await && files.exists(&p.join(CANISTER_MANIFEST)).await + { + kept.push(p); + } + } + kept + } else { + // For explicit paths, validate that they exist and contain canister.yaml + let mut validated_paths = vec![]; + for p in paths { + if !files.is_file(&p.join(CANISTER_MANIFEST)).await { + return NotFoundSnafu { + kind: "canister".to_string(), + path: pattern.to_string(), + } + .fail(); + } + validated_paths.push(p); + } + validated_paths + }; + + let mut ms = vec![]; + for p in paths { + ms.push(( + p.to_owned(), + load_manifest::(files, &p.join(CANISTER_MANIFEST)) + .await + .context(LoadCanisterSnafu)?, + )); + } + ms + } + + Item::Manifest(m) => vec![(pdir.to_owned(), m.to_owned())], + }; + + for (cdir, m) in ms { + if !is_valid_name(&m.name) { + return InvalidCanisterNameSnafu { + name: m.name.clone(), + } + .fail(); + } + + let registry_recipe = match &m.instructions { + Instructions::BuildSync { .. } => None, + Instructions::Recipe { recipe } => match &recipe.recipe_type { + RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), + _ => None, + }, + }; + + let (build, sync) = + match &m.instructions { + // Build/Sync + Instructions::BuildSync { build, sync } => ( + build.to_owned(), + match sync { + Some(sync) => sync.to_owned(), + None => SyncSteps::default(), + }, + ), + + // Recipe: fetch the template through the resolver, then render + // and parse it into concrete steps. + Instructions::Recipe { recipe } => { + let ctx = recipe::RecipeContext { + canister_name: m.name.clone(), + }; + let fetched = recipe_resolver.resolve_recipe(recipe).await.context( + FetchRecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + }, + )?; + let steps = recipe::render_recipe(&fetched.template, recipe, &ctx) + .context(RenderRecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + })?; + // The template rendered, so an unpinned download is now + // known good and safe to cache. Committing only here is what + // keeps a bad remote response from becoming sticky. + recipe_resolver + .commit_recipe(recipe, &fetched) + .await + .context(CacheRecipeSnafu { + recipe_type: recipe.recipe_type.clone(), + })?; + steps + } + }; + + let (settings, environment_variable_files) = + resolve_manifest_settings(files, &m.settings, &cdir, &m.name).await?; + + let init_args = match m.init_args.as_ref() { + Some(mia) => Some(resolve_manifest_init_args(files, mia, &cdir, &m.name).await?), + None => None, + }; + + result.push(( + m.name.clone(), + cdir, + Canister { + name: m.name.clone(), + settings, + build, + sync, + init_args, + registry_recipe, + bindings: BTreeMap::new(), + // Default to the bare local name; overwritten with the + // dot-nested alias form when the canister is imported as a + // dependency (see `import_dependency`). + friendly_names: vec![m.name.clone()], + environment_variable_files, + }, + )); + } + } + + Ok(result) +} + +/// A dependency instance imported into the workspace. Returned by +/// [`import_dependency`] and cached per canonical path so diamond dependencies +/// reuse the same instance. +#[derive(Clone)] +struct ImportedInstance { + /// This instance's own canisters, as `(local name, full store key)` — the + /// set exposable to the parent via `canisters:` selection. + own: Vec<(String, String)>, + /// Every canister in this instance's subtree (its own canisters plus all + /// transitively imported ones), as `(store key, local name, alias chain + /// from this instance down to the canister's owning project)`. Used to + /// register a friendly URL per alias chain when the instance is reached + /// again via de-duplication (a diamond), including for its descendants. + subtree: Vec<(String, String, Vec)>, +} + +/// A member environment's per-canister config, to be folded into the root's +/// same-named environment beneath any root overrides. +#[derive(Default, Clone)] +struct MemberCanisterOverride { + settings: Option, + init_args: Option, +} + +/// Per-environment member overrides: env name → store key → override. +type MemberEnvOverrides = HashMap>; + +/// A member's identity (store-key prefix) and the environment names it defines, +/// used to enforce that a member declares every environment the root targets +/// (strict rule). +struct MemberEnvInfo { + prefix: String, + defined: HashSet, +} + +/// Canonicalize a dependency root (resolving symlinks and `..`) for use as a +/// de-dup / cycle-detection identity. +async fn canonicalize_dep( + files: &dyn FileAccess, + alias: &str, + dep_root: &Path, +) -> Result { + files.canonicalize(dep_root).await.ok_or_else(|| { + DependencyCanonicalizeSnafu { + alias: alias.to_owned(), + path: dep_root.to_string(), + } + .build() + }) +} + +/// Store-key prefix for a dependency instance: its canonical directory relative +/// to the canonical app root, forward-slash separated so keys are stable across +/// platforms and independent of how each edge spells the path. +pub fn relative_prefix(app_root_canonical: &Path, dep_canonical: &Path) -> String { + let rel = pathdiff::diff_utf8_paths(dep_canonical, app_root_canonical) + .unwrap_or_else(|| dep_canonical.to_owned()); + rel.as_str().replace('\\', "/") +} + +/// Build a dependency canister's friendly-URL subdomain prefix: the canister's +/// local name as the most-specific label, followed by its alias chain reversed +/// (root-most alias last). E.g. local `backend` reached via `[service-a, +/// openemail]` → `backend.openemail.service-a`. Dot-nested so it stays a valid, +/// collision-free multi-label host; see DESIGN §17.2. +fn friendly_name_for(local: &str, alias_chain: &[String]) -> String { + let mut labels = Vec::with_capacity(alias_chain.len() + 1); + labels.push(local.to_string()); + labels.extend(alias_chain.iter().rev().cloned()); + labels.join(".") +} + +/// Rewrite `CanisterName` controller references from a dependency's local +/// canister names to their store keys, so global controller validation and +/// deploy-time id lookup operate uniformly on store keys. +fn translate_controllers(canister: &mut Canister, local_to_key: &BTreeMap) { + translate_settings_controllers(&mut canister.settings, local_to_key); +} + +/// Rewrite `CanisterName` controller references in a `Settings` from a +/// dependency's local canister names to their store keys. +fn translate_settings_controllers( + settings: &mut Settings, + local_to_key: &BTreeMap, +) { + if let Some(controllers) = &mut settings.controllers { + for cref in controllers.iter_mut() { + if let ControllerRef::CanisterName(name) = cref + && let Some(key) = local_to_key.get(name) + { + *name = key.clone(); + } + } + } +} + +/// Compute the `PUBLIC_CANISTER_ID` env-var wiring for canisters in one project +/// scope: its own canisters by local name, plus each dependency's exposed +/// canisters under `:`. +fn compute_bindings( + own: &[(String, String)], + edges: &[(String, Vec<(String, String)>)], +) -> BTreeMap { + let mut bindings = BTreeMap::new(); + for (local, key) in own { + bindings.insert(local.clone(), key.clone()); + } + for (alias, exposed) in edges { + for (dep_local, key) in exposed { + bindings.insert(format!("{alias}:{dep_local}"), key.clone()); + } + } + bindings +} + +/// Select which of a dependency instance's own canisters are exposed to the +/// parent, per the dependency's `canisters` selection. +fn select_exposed( + own: &[(String, String)], + selection: &CanisterSelection, + alias: &str, +) -> Result, ConsolidateManifestError> { + match selection { + CanisterSelection::Everything => Ok(own.to_vec()), + CanisterSelection::None => Ok(vec![]), + CanisterSelection::Named(names) => { + let mut out = Vec::new(); + for name in names { + match own.iter().find(|(local, _)| local == name) { + Some(pair) => out.push(pair.clone()), + None => { + return UnknownDependencyCanisterSnafu { + alias: alias.to_owned(), + canister: name.clone(), + } + .fail(); + } + } + } + Ok(out) + } + } +} + +/// Validate the dependency aliases declared in one project scope: no `:`, no +/// collision with a local canister name, and no duplicate alias. +fn validate_dependency_aliases( + deps: &[DependencyManifest], + own_canister_names: &HashSet, +) -> Result<(), ConsolidateManifestError> { + let mut seen: HashSet<&str> = HashSet::new(); + for d in deps { + if !is_valid_name(&d.name) { + return InvalidDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + if own_canister_names.contains(&d.name) { + return DependencyAliasCollisionSnafu { + alias: d.name.clone(), + } + .fail(); + } + if !seen.insert(&d.name) { + return DuplicateDependencyAliasSnafu { + alias: d.name.clone(), + } + .fail(); + } + } + Ok(()) +} + +/// Recursively import a dependency's canisters into `canisters`, keyed by their +/// app-root-relative store keys. De-duplicates instances by canonical path +/// (diamond dependencies deploy once) and detects cycles. Returns the imported +/// instance's prefix and its own canisters. +#[allow(clippy::too_many_arguments)] +async fn import_dependency( + files: &dyn FileAccess, + app_root_canonical: &Path, + parent_dir: &Path, + dep: &DependencyManifest, + recipe_resolver: &dyn recipe::RemoteResourceResolve, + canisters: &mut IndexMap, + registry: &mut HashMap, + stack: &mut Vec, + member_env_overrides: &mut MemberEnvOverrides, + members: &mut Vec, + // Alias chain from the workspace root to and including this dependency, + // used to build friendly-URL subdomains (§17.2). + alias_chain: &[String], +) -> Result { + let dep_root = parent_dir.join(&dep.path); + let manifest_path = dep_root.join(PROJECT_MANIFEST); + if !files.is_file(&manifest_path).await { + return DependencyNotFoundSnafu { + alias: dep.name.clone(), + path: dep_root.to_string(), + } + .fail(); + } + + let canonical = canonicalize_dep(files, &dep.name, &dep_root).await?; + + // Cycle detection. + if stack.contains(&canonical) { + let mut chain: Vec = stack.iter().map(|p| p.to_string()).collect(); + chain.push(canonical.to_string()); + return CircularDependencySnafu { + chain: chain.join(" -> "), + } + .fail(); + } + + // Diamond de-dup: same resolved directory means the same instance, deployed + // once. It is still reachable via this new alias chain, so register an + // additional friendly URL per chain (§17.3) rather than picking one — for + // the whole subtree (its own canisters *and* its transitive dependencies), + // each named by this chain extended with the canister's alias path below the + // instance. + if let Some(inst) = registry.get(&canonical) { + let inst = inst.clone(); + for (key, local, rel_chain) in &inst.subtree { + let mut chain = alias_chain.to_vec(); + chain.extend(rel_chain.iter().cloned()); + let fname = friendly_name_for(local, &chain); + if let Some((_, canister)) = canisters.get_mut(key) + && !canister.friendly_names.contains(&fname) + { + canister.friendly_names.push(fname); + } + } + return Ok(inst); + } + + stack.push(canonical.clone()); + + let prefix = relative_prefix(app_root_canonical, &canonical); + + let dep_manifest: ProjectManifest = + load_manifest(files, &manifest_path) + .await + .context(LoadDependencyManifestSnafu { + alias: dep.name.clone(), + })?; + + // Build the dependency's own canisters and key them under the prefix. All of + // them are imported (deploy-all); the `canisters` exposure subset is applied + // by the caller when wiring env vars. + let built = + build_manifest_canisters(files, &dep_root, &dep_manifest.canisters, recipe_resolver) + .await?; + + let mut own: Vec<(String, String)> = Vec::new(); + let mut local_to_key: BTreeMap = BTreeMap::new(); + for (local, cdir, mut canister) in built { + let store_key = format!("{prefix}:{local}"); + canister.name = store_key.clone(); + // Friendly URL from the alias chain, not the path-based store key. + canister.friendly_names = vec![friendly_name_for(&local, alias_chain)]; + own.push((local.clone(), store_key.clone())); + local_to_key.insert(local.clone(), store_key.clone()); + match canisters.entry(store_key.clone()) { + IndexEntry::Occupied(_) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: store_key, + } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); + } + } + } + + // Now that every sibling's store key is known, translate the dependency's + // controller references (local sibling name -> store key). + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + translate_controllers(canister, &local_to_key); + } + } + + // Capture the member's own environments so the parent can honor its + // per-canister settings/init_args for the same-named environment + // (standalone-equivalence). The network binding and canister selection are + // ignored; only overrides on the member's *own* canisters are + // folded in — keys naming its dependencies are left to those dependencies. + let mut defined_envs: HashSet = HashSet::new(); + for env_item in &dep_manifest.environments { + let em: EnvironmentManifest = match env_item { + Item::Manifest(m) => m.clone(), + Item::Path(path) => { + let p = dep_root.join(path); + if !files.is_file(&p).await { + return NotFoundSnafu { + kind: "environment".to_string(), + path: p.to_string(), + } + .fail(); + } + load_manifest::(files, &p) + .await + .context(LoadEnvironmentSnafu)? + } + }; + defined_envs.insert(em.name.clone()); + if let Some(settings) = &em.settings { + for (local, s) in settings { + if let Some(key) = local_to_key.get(local) { + // Translate the override's own controller references from the + // member's local names to store keys, so name-based controllers + // resolve against the workspace id map just like base settings. + let mut s = s.clone(); + translate_settings_controllers(&mut s, &local_to_key); + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .settings = Some(s); + } + } + } + if let Some(init_args) = &em.init_args { + for (local, ia) in init_args { + if let Some(key) = local_to_key.get(local) { + member_env_overrides + .entry(em.name.clone()) + .or_default() + .entry(key.clone()) + .or_default() + .init_args = Some(ia.clone()); + } + } + } + } + members.push(MemberEnvInfo { + prefix: prefix.clone(), + defined: defined_envs, + }); + + // Recurse into the dependency's own dependencies. + let own_names: HashSet = own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&dep_manifest.dependencies, &own_names)?; + + // The instance's subtree, for diamond-hit friendly-URL propagation: its own + // canisters sit at the instance root (empty relative alias chain); each + // nested dependency contributes its subtree prefixed with the nested alias. + let mut subtree: Vec<(String, String, Vec)> = own + .iter() + .map(|(local, key)| (key.clone(), local.clone(), Vec::new())) + .collect(); + + let mut edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for nested in &dep_manifest.dependencies { + let mut nested_chain = alias_chain.to_vec(); + nested_chain.push(nested.name.clone()); + let inst = Box::pin(import_dependency( + files, + app_root_canonical, + &dep_root, + nested, + recipe_resolver, + canisters, + registry, + stack, + member_env_overrides, + members, + &nested_chain, + )) + .await?; + for (key, local, rel) in &inst.subtree { + let mut r = Vec::with_capacity(rel.len() + 1); + r.push(nested.name.clone()); + r.extend(rel.iter().cloned()); + subtree.push((key.clone(), local.clone(), r)); + } + let exposed = select_exposed(&inst.own, &nested.canisters, &nested.name)?; + edges.push((nested.name.clone(), exposed)); + } + + // Assign env-var bindings for this instance's own canisters. + let bindings = compute_bindings(&own, &edges); + for (_, key) in &own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = bindings.clone(); + } + } + + stack.pop(); + let instance = ImportedInstance { own, subtree }; + registry.insert(canonical, instance.clone()); + Ok(instance) +} + +/// Build one environment's canister map: select from `canisters`, then apply the +/// member overrides for this environment (standalone-equivalence), then +/// the root's own overrides (highest precedence). Precedence is therefore +/// root-explicit > member-env > canister-base. +async fn build_environment_canisters( + files: &dyn FileAccess, + canisters: &IndexMap, + env_name: &str, + selection: &CanisterSelection, + member_overrides: Option<&HashMap>, + root_settings: Option<&HashMap>, + root_init_args: Option<&HashMap>, +) -> Result, ConsolidateManifestError> { + let mut cs = match selection { + CanisterSelection::None => IndexMap::new(), + CanisterSelection::Everything => canisters.clone(), + CanisterSelection::Named(names) => { + let mut cs: IndexMap = IndexMap::new(); + for name in names { + let v = canisters.get(name).ok_or( + InvalidCanisterSnafu { + environment: env_name.to_owned(), + canister: name.to_owned(), + } + .build(), + )?; + cs.insert(name.to_owned(), v.to_owned()); + } + cs + } + }; + + // Member overrides first (lower precedence than the root's own overrides). + if let Some(overrides) = member_overrides { + for (key, ov) in overrides { + if let Some((cpath, canister)) = cs.get_mut(key) { + if let Some(s) = &ov.settings { + (canister.settings, canister.environment_variable_files) = + resolve_manifest_settings(files, s, cpath, key).await?; + } + if let Some(ia) = &ov.init_args { + canister.init_args = + Some(resolve_manifest_init_args(files, ia, cpath, key).await?); + } + } + } + } + + // Root overrides last (highest precedence). + if let Some(settings) = root_settings { + for (name, s) in settings { + if let Some((cpath, canister)) = cs.get_mut(name) { + (canister.settings, canister.environment_variable_files) = + resolve_manifest_settings(files, s, cpath, name).await?; + } + } + } + if let Some(init_args) = root_init_args { + for (name, ia) in init_args { + if let Some((cpath, canister)) = cs.get_mut(name) { + canister.init_args = + Some(resolve_manifest_init_args(files, ia, cpath, name).await?); + } + } + } + + Ok(cs) +} + +/// Turns the ProjectManifest into a Project struct +/// - Adds the default Networks +/// - Adds the default Environment +/// - Imports any dependency projects' canisters +/// - Validates the manifest to make sure that: +/// - There are no duplicates +/// - All the environments have networks +/// - All the referenced canisters exist +/// - All the recipes have been resolved +pub async fn consolidate_manifest( + files: &dyn FileAccess, + pdir: &Path, + recipe_resolver: &dyn recipe::RemoteResourceResolve, + m: &ProjectManifest, +) -> Result { + // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved + // through to consumers like `icp project bundle`, which needs reproducible output. + let mut canisters: IndexMap = IndexMap::new(); + + // Canonical app root, used to derive stable, order-independent store-key + // prefixes for imported dependency canisters. + let app_root_canonical = files + .canonicalize(pdir) + .await + .unwrap_or_else(|| pdir.to_owned()); + + // This project's own canisters, keyed by their bare local names. + let app_built = build_manifest_canisters(files, pdir, &m.canisters, recipe_resolver).await?; + let mut app_own: Vec<(String, String)> = Vec::new(); + for (local, cdir, canister) in app_built { + app_own.push((local.clone(), local.clone())); + match canisters.entry(local.clone()) { + IndexEntry::Occupied(e) => { + return DuplicateSnafu { + kind: "canister".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + IndexEntry::Vacant(e) => { + e.insert((cdir, canister)); + } + } + } + + // Import dependency projects. Each dependency is deployed in full and keyed + // under its app-root-relative path; diamonds (the same directory reached via + // multiple edges) resolve to a single instance. + let mut registry: HashMap = HashMap::new(); + let mut stack: Vec = Vec::new(); + // Member environment config folded into the root's same-named environments, + // and the per-member set of declared environment names for the strict rule. + let mut member_env_overrides: MemberEnvOverrides = HashMap::new(); + let mut members: Vec = Vec::new(); + let app_own_names: HashSet = app_own.iter().map(|(l, _)| l.clone()).collect(); + validate_dependency_aliases(&m.dependencies, &app_own_names)?; + + let mut app_edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); + for dep in &m.dependencies { + let inst = import_dependency( + files, + &app_root_canonical, + pdir, + dep, + recipe_resolver, + &mut canisters, + &mut registry, + &mut stack, + &mut member_env_overrides, + &mut members, + std::slice::from_ref(&dep.name), + ) + .await?; + let exposed = select_exposed(&inst.own, &dep.canisters, &dep.name)?; + app_edges.push((dep.name.clone(), exposed)); + } + + // Assign env-var bindings for this project's own canisters (own canisters by + // local name plus each dependency's exposed canisters under `:`). + let app_bindings = compute_bindings(&app_own, &app_edges); + for (_, key) in &app_own { + if let Some((_, canister)) = canisters.get_mut(key) { + canister.bindings = app_bindings.clone(); + } + } + + // Friendly URLs need no de-collision pass: the strict name rule (no '.') makes + // own canisters single-label and dependency canisters multi-label (dot-nested + // by alias chain), so their hostnames are disjoint by construction (§17.2). + + // Validate that every canister-name controller reference points to a declared canister. + // Catching typos here turns "perpetual warning" into a clear load-time error. + for (canister_name, (_, canister)) in &canisters { + let Some(crefs) = &canister.settings.controllers else { + continue; + }; + for cref in crefs { + if let Some(ref_name) = cref.canister_name() + && !canisters.contains_key(ref_name) + { + return UnknownControllerCanisterSnafu { + canister: canister_name.to_owned(), + controller: ref_name.to_owned(), + } + .fail(); + } + } + } + + // Networks + let mut networks: HashMap = HashMap::new(); + + // Add IC network first - this is always protected and non-overridable + networks.insert( + IC.to_string(), + Network { + name: IC.to_string(), + configuration: Configuration::Connected { + connected: Connected { + api_url: IC_MAINNET_NETWORK_API_URL.parse().unwrap(), + http_gateway_url: Some(IC_MAINNET_NETWORK_GATEWAY_URL.parse().unwrap()), + root_key: RootKeySpec::Mainnet, + }, + }, + }, + ); + + // Track which network names are protected (only IC network) + let protected_network_names: HashSet = [IC.to_string()].into_iter().collect(); + + // Resolve NetworkManifests and add them (including user-defined "local" if provided) + for i in &m.networks { + let m = match i { + Item::Path(path) => { + let path = pdir.join(path); + if !files.is_file(&path).await { + return NotFoundSnafu { + kind: "network".to_string(), + path: path.to_string(), + } + .fail(); + } + load_manifest::(files, &path) + .await + .context(LoadNetworkSnafu)? + } + Item::Manifest(ms) => ms.clone(), + }; + + match networks.entry(m.name.to_owned()) { + // Duplicate + Entry::Occupied(e) => { + // Only error if trying to override a protected network + if protected_network_names.contains(&m.name) { + return ReservedSnafu { + kind: "network".to_string(), + name: m.name.to_string(), + } + .fail(); + } + + // For non-protected duplicates, this is a user error (defining same network twice) + return DuplicateSnafu { + kind: "network".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + + // Ok + Entry::Vacant(e) => { + e.insert(Network { + name: m.name.to_owned(), + configuration: m.configuration.into(), // Convert manifest to config struct + }); + } + } + } + + // After processing user networks, add default "local" if not already defined + // This provides backward compatibility for projects that don't define their own "local" network + if !networks.contains_key(LOCAL) { + networks.insert( + LOCAL.to_string(), + Network { + name: LOCAL.to_string(), + configuration: Configuration::Managed { + managed: Managed { + mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { + gateway: Gateway { + bind: DEFAULT_LOCAL_NETWORK_BIND.to_string(), + port: Port::Fixed(DEFAULT_LOCAL_NETWORK_PORT), + domains: vec![], + }, + artificial_delay_ms: None, + ii: false, + nns: false, + subnets: None, + bitcoind_addr: None, + dogecoind_addr: None, + version: None, + })), + }, + }, + }, + ); + } + + // Environments + let mut environments: HashMap = HashMap::new(); + + for i in &m.environments { + let m = match i { + Item::Path(path) => { + let path = pdir.join(path); + if !files.is_file(&path).await { + return NotFoundSnafu { + kind: "environment".to_string(), + path: path.to_string(), + } + .fail(); + } + load_manifest::(files, &path) + .await + .context(LoadEnvironmentSnafu)? + } + Item::Manifest(ms) => ms.clone(), + }; + + match environments.entry(m.name.to_owned()) { + // Duplicate + Entry::Occupied(e) => { + return DuplicateSnafu { + kind: "environment".to_string(), + name: e.key().to_owned(), + } + .fail(); + } + + // Ok + Entry::Vacant(e) => { + e.insert(Environment { + name: m.name.to_owned(), + + // Embed network in environment + network: { + let v = networks.get(&m.network).ok_or( + InvalidNetworkSnafu { + environment: m.name.to_owned(), + network: m.network.to_owned(), + } + .build(), + )?; + + v.to_owned() + }, + + // Embed canisters in environment, folding member overrides + // beneath the root's own settings/init_args overrides. + canisters: build_environment_canisters( + files, + &canisters, + &m.name, + &m.canisters, + member_env_overrides.get(&m.name), + m.settings.as_ref(), + m.init_args.as_ref(), + ) + .await?, + }); + } + } + } + + // We're done adding all the user environments + // Now we add the implicit `local` and `ic` environment if the user hasn't overriden it + if let Entry::Vacant(vacant_entry) = environments.entry(LOCAL.to_string()) { + let network = networks + .get(LOCAL) + .ok_or( + InvalidNetworkSnafu { + environment: LOCAL.to_owned(), + network: LOCAL.to_owned(), + } + .build(), + )? + .to_owned(); + vacant_entry.insert(Environment { + name: LOCAL.to_string(), + network, + canisters: build_environment_canisters( + files, + &canisters, + LOCAL, + &CanisterSelection::Everything, + member_env_overrides.get(LOCAL), + None, + None, + ) + .await?, + }); + } + if let Entry::Vacant(vacant_entry) = environments.entry(IC.to_string()) { + let network = networks + .get(IC) + .ok_or( + InvalidNetworkSnafu { + environment: IC.to_owned(), + network: IC.to_owned(), + } + .build(), + )? + .to_owned(); + vacant_entry.insert(Environment { + name: IC.to_string(), + network, + canisters: build_environment_canisters( + files, + &canisters, + IC, + &CanisterSelection::Everything, + member_env_overrides.get(IC), + None, + None, + ) + .await?, + }); + } + + // Strict rule: every member must declare each environment the root targets. + // `local`/`ic` are implicit for every project, so they never count + // as missing; other environments must be declared explicitly by the member. + // Recorded per-environment and enforced lazily when that environment is + // selected (so a missing `staging` never blocks `deploy -e local`). + let mut member_missing_envs: HashMap> = HashMap::new(); + for env_name in environments.keys() { + if env_name == LOCAL || env_name == IC { + continue; + } + for member in &members { + if !member.defined.contains(env_name) { + member_missing_envs + .entry(env_name.clone()) + .or_default() + .push(member.prefix.clone()); + } + } + } + + Ok(Project { + dir: pdir.into(), + canisters, + networks, + environments, + member_missing_envs, + }) +} + +#[derive(Debug, Snafu)] +pub enum LoadProjectError { + #[snafu(display("failed to load project manifest"))] + ProjectManifest { source: LoadManifestError }, + + #[snafu(transparent)] + Consolidate { source: ConsolidateManifestError }, +} + +/// Load and consolidate the project rooted at `project_dir` (already located by +/// the caller), reading all files through `files` and resolving recipes through +/// `recipe`. +pub async fn load_project( + files: &dyn FileAccess, + recipe: &dyn recipe::RemoteResourceResolve, + project_dir: &Path, +) -> Result { + let m: ProjectManifest = load_manifest(files, &project_dir.join(PROJECT_MANIFEST)) + .await + .context(ProjectManifestSnafu)?; + let p = consolidate_manifest(files, project_dir, recipe, &m).await?; + Ok(p) +} + +#[derive(Debug, Snafu)] +pub enum VerifySandboxError { + #[snafu(display( + "canister '{canister}' uses a script {phase} step, which cannot run in the sandbox; \ + only pre-built builds and plugin syncs are permitted" + ))] + ScriptStep { canister: String, phase: String }, +} + +/// Verify that a fully-resolved project (recipes already resolved into concrete +/// steps) contains no script steps. Script build/sync steps spawn host +/// subprocesses and therefore cannot run inside the sandbox; only pre-built +/// builds and plugin syncs are permitted. +pub fn verify_sandbox(project: &Project) -> Result<(), VerifySandboxError> { + use crate::manifest::canister::{BuildStep, SyncStep}; + + for (name, (_, canister)) in &project.canisters { + if canister + .build + .steps + .iter() + .any(|s| matches!(s, BuildStep::Script(_))) + { + return ScriptStepSnafu { + canister: name.clone(), + phase: "build", + } + .fail(); + } + if canister + .sync + .steps + .iter() + .any(|s| matches!(s, SyncStep::Script(_))) + { + return ScriptStepSnafu { + canister: name.clone(), + phase: "sync", + } + .fail(); + } + } + Ok(()) +} + +#[cfg(test)] +mod dependency_tests { + use super::*; + use crate::canister::recipe::{FetchedRecipe, RemoteResourceResolve, ResolveError}; + use crate::manifest::adapter::prebuilt::SourceField; + use crate::manifest::recipe::Recipe; + use crate::sync_exec::StepProgress; + use crate::testutil::HostFiles; + use camino_tempfile::Utf8TempDir; + + /// Recipes and plugins are never used in these tests; every canister is pre-built. + struct PanicResolver; + + #[async_trait::async_trait] + impl RemoteResourceResolve for PanicResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + panic!("recipe resolver should not be called in dependency tests"); + } + + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + panic!("recipe resolver should not be called in dependency tests"); + } + + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _progress: Option<&dyn StepProgress>, + ) -> Result { + panic!("wasm resolver should not be called in dependency tests"); + } + } + + fn write(dir: &Path, rel: &str, contents: &str) { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, contents).unwrap(); + } + + /// A minimal `icp.yaml` body declaring the given pre-built canisters, + /// followed by a raw `dependencies:` block (may be empty). + fn manifest(canisters: &[&str], deps: &str) -> String { + let mut s = String::new(); + if canisters.is_empty() { + s.push_str("canisters: []\n"); + } else { + s.push_str("canisters:\n"); + for c in canisters { + s.push_str(&format!( + " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" + )); + } + } + s.push_str(deps); + s + } + + async fn consolidate(pdir: &Path) -> Result { + let files = HostFiles; + let m: ProjectManifest = load_manifest(&files, &pdir.join(PROJECT_MANIFEST)) + .await + .expect("failed to parse project manifest"); + consolidate_manifest(&files, pdir, &PanicResolver, &m).await + } + + fn bindings_of<'a>(p: &'a Project, key: &str) -> &'a BTreeMap { + &p.canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + p.canisters.keys().collect::>() + ) + }) + .1 + .bindings + } + + fn friendly_names_of<'a>(p: &'a Project, key: &str) -> &'a [String] { + &p.canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + p.canisters.keys().collect::>() + ) + }) + .1 + .friendly_names + } + + #[tokio::test] + async fn single_project_bindings_are_self_and_siblings() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Flat behavior preserved: every canister maps every sibling (incl. self) + // to itself. + let expected = BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ("frontend".to_string(), "frontend".to_string()), + ]); + assert_eq!(bindings_of(&p, "backend"), &expected); + assert_eq!(bindings_of(&p, "frontend"), &expected); + } + + #[tokio::test] + async fn dependency_import_and_exposure_subset() { + let tmp = Utf8TempDir::new().unwrap(); + // Dependency nested inside the app (mirrors a submodule under the app). + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [backend]\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // The whole dependency is deployed (both canisters imported), keyed by path. + assert!(p.canisters.contains_key("backend")); + assert!(p.canisters.contains_key("openemail:backend")); + assert!(p.canisters.contains_key("openemail:frontend")); + + // App's own canister sees itself and only the *exposed* dependency canister. + assert_eq!( + bindings_of(&p, "backend"), + &BTreeMap::from([ + ("backend".to_string(), "backend".to_string()), + ( + "openemail:backend".to_string(), + "openemail:backend".to_string() + ), + ]) + ); + + // The dependency's own canisters keep their standalone view (bare names). + assert_eq!( + bindings_of(&p, "openemail:backend"), + &BTreeMap::from([ + ("backend".to_string(), "openemail:backend".to_string()), + ("frontend".to_string(), "openemail:frontend".to_string()), + ]) + ); + } + + #[tokio::test] + async fn member_env_config_folds_in_with_root_override_winning() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail defines `staging` with per-canister settings for its own + // canisters. + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - name: backend + build: + steps: + - type: pre-built + path: backend.wasm + - name: frontend + build: + steps: + - type: pre-built + path: frontend.wasm +environments: + - name: staging + settings: + backend: + compute_allocation: 5 + frontend: + compute_allocation: 7 +"#, + ); + // The app declares openemail and also defines `staging`, overriding the + // imported backend's settings (the root override must win). + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging + settings: + "openemail:backend": + compute_allocation: 99 +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + // Root override wins over the member's config. + assert_eq!( + staging + .canisters + .get("openemail:backend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(99), + ); + // No root override → the member's own config applies (standalone-equivalence). + assert_eq!( + staging + .canisters + .get("openemail:frontend") + .unwrap() + .1 + .settings + .compute_allocation, + Some(7), + ); + // Both projects declared staging, so nothing is recorded as missing. + assert!(p.member_missing_envs.is_empty()); + } + + #[tokio::test] + async fn missing_member_environment_is_recorded() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + // openemail does not declare `staging`, so it is recorded as missing. + assert_eq!( + p.member_missing_envs.get("staging"), + Some(&vec!["openemail".to_string()]), + ); + // Implicit environments are never recorded as missing. + assert!(!p.member_missing_envs.contains_key("local")); + assert!(!p.member_missing_envs.contains_key("ic")); + } + + #[tokio::test] + async fn diamond_dedups_to_single_instance() { + let tmp = Utf8TempDir::new().unwrap(); + // umbrella layout: service-a and service-b both depend on ../openemail. + write( + tmp.path(), + "umbrella/openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "umbrella/service-a/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "umbrella/service-b/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // openemail is imported exactly once despite two edges reaching it. + let openemail_keys: Vec<_> = p + .canisters + .keys() + .filter(|k| k.contains("openemail")) + .collect(); + assert_eq!( + openemail_keys, + vec![&"umbrella/openemail:backend".to_string()], + "expected a single shared openemail instance" + ); + + // Both services' code reads `openemail:backend`, resolving to the one instance. + assert_eq!( + bindings_of(&p, "umbrella/service-a:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + assert_eq!( + bindings_of(&p, "umbrella/service-b:backend").get("openemail:backend"), + Some(&"umbrella/openemail:backend".to_string()) + ); + + // The single shared instance is reachable at one friendly URL per alias + // chain (§17.3) — the store-key path (`umbrella/`) never appears. + assert_eq!( + friendly_names_of(&p, "umbrella/openemail:backend"), + &["backend.openemail.service-a", "backend.openemail.service-b"] + ); + // Each service's own canister is named by its own alias chain. + assert_eq!( + friendly_names_of(&p, "umbrella/service-a:backend"), + &["backend.service-a"] + ); + assert_eq!( + friendly_names_of(&p, "umbrella/service-b:backend"), + &["backend.service-b"] + ); + } + + #[tokio::test] + async fn diamond_transitive_dependency_gets_url_per_chain() { + let tmp = Utf8TempDir::new().unwrap(); + // The shared openemail itself depends on libfoo, and is reached via both + // service-a and service-b. + write( + tmp.path(), + "umbrella/openemail/libfoo/icp.yaml", + &manifest(&["bar"], ""), + ); + write( + tmp.path(), + "umbrella/openemail/icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: libfoo\n path: ./libfoo\n", + ), + ); + write( + tmp.path(), + "umbrella/service-a/icp.yaml", + &manifest( + &["service"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "umbrella/service-b/icp.yaml", + &manifest( + &["service"], + "dependencies:\n - name: openemail\n path: ../openemail\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // The shared instance's own canister gets one URL per chain... + assert_eq!( + friendly_names_of(&p, "umbrella/openemail:backend"), + &["backend.openemail.service-a", "backend.openemail.service-b"] + ); + // ...and so does its *transitive* dependency (the subtree is revisited on + // the diamond hit, not just the instance's own canisters). + assert_eq!( + friendly_names_of(&p, "umbrella/openemail/libfoo:bar"), + &[ + "bar.libfoo.openemail.service-a", + "bar.libfoo.openemail.service-b" + ] + ); + } + + #[tokio::test] + async fn dot_in_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + // '.' is banned: it would be ambiguous in a dot-nested friendly subdomain + // (an own canister named `frontend.openemail` could collide with dependency + // `openemail`'s `frontend`). The strict name rule rejects it up front. + write( + tmp.path(), + "icp.yaml", + &manifest(&["frontend.openemail"], ""), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn invalid_dependency_alias_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["app"], + "dependencies:\n - name: open.email\n path: ./openemail\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidDependencyAlias { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn member_override_controllers_are_translated_to_store_keys() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail's `staging` override names a controller by its local name. + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - name: backend + build: + steps: + - type: pre-built + path: backend.wasm + - name: frontend + build: + steps: + - type: pre-built + path: frontend.wasm +environments: + - name: staging + settings: + backend: + controllers: ["frontend"] +"#, + ); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: app + build: + steps: + - type: pre-built + path: app.wasm +dependencies: + - name: openemail + path: ./openemail +environments: + - name: staging +"#, + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + let controllers = staging + .canisters + .get("openemail:backend") + .unwrap() + .1 + .settings + .controllers + .clone() + .expect("controllers set by the member override"); + + // The member-local `frontend` must be translated to its store key, so it + // resolves against the workspace id map at deploy time. + assert_eq!( + controllers, + vec![ControllerRef::CanisterName( + "openemail:frontend".to_string() + )] + ); + } + + fn env_vars_of<'a>( + canisters: &'a IndexMap, + key: &str, + ) -> &'a HashMap { + canisters + .get(key) + .unwrap_or_else(|| { + panic!( + "canister '{key}' not found; have {:?}", + canisters.keys().collect::>() + ) + }) + .1 + .settings + .environment_variables + .as_ref() + .expect("environment variables set") + } + + /// A canister manifest's file-backed environment variable resolves against + /// the canister's own directory, and the file's trailing newline is not part + /// of the value. + #[tokio::test] + async fn env_var_file_resolves_against_canister_dir() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "canisters/backend/canister.yaml", + r#" +name: backend +settings: + environment_variables: + API_KEY: + path: secrets/api-key +build: + steps: + - type: pre-built + path: backend.wasm +"#, + ); + write(tmp.path(), "canisters/backend/secrets/api-key", "s3cret\n"); + write( + tmp.path(), + "icp.yaml", + "canisters:\n - ./canisters/backend\n", + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!( + env_vars_of(&p.canisters, "backend"), + &HashMap::from([("API_KEY".to_string(), "s3cret".to_string())]), + ); + } + + /// A canister declared in its own directory, for the override tests below: + /// its directory is neither the project's nor an environment manifest's, so + /// the base a path resolves against is unambiguous. + fn write_backend_canister(dir: &Path, at: &str) { + write( + dir, + &format!("{at}/canister.yaml"), + r#" +name: backend +build: + steps: + - type: pre-built + path: backend.wasm +"#, + ); + } + + /// An environment override resolves a path against the *canister's* directory + /// — the same base an `init_args` override uses — not against the manifest + /// declaring the override, even when that is an environment manifest of its + /// own. + #[tokio::test] + async fn env_var_file_in_environment_override_resolves_against_canister_dir() { + let tmp = Utf8TempDir::new().unwrap(); + write_backend_canister(tmp.path(), "canisters/backend"); + write( + tmp.path(), + "icp.yaml", + "canisters:\n - ./canisters/backend\nenvironments:\n - ./environments/staging.yaml\n", + ); + write( + tmp.path(), + "environments/staging.yaml", + r#" +name: staging +settings: + backend: + environment_variables: + API_KEY: + path: secrets/api-key +"#, + ); + write( + tmp.path(), + "canisters/backend/secrets/api-key", + "staging-key\n", + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + assert_eq!( + env_vars_of(&staging.canisters, "backend"), + &HashMap::from([("API_KEY".to_string(), "staging-key".to_string())]), + ); + // The override applies to the environment only; the canister's own + // settings are untouched. + assert_eq!( + p.canisters + .get("backend") + .unwrap() + .1 + .settings + .environment_variables, + None, + ); + } + + /// A member's own environment override resolves against the member's + /// canister, not against the member's or the root's project directory. + #[tokio::test] + async fn env_var_file_in_member_environment_resolves_against_member_canister_dir() { + let tmp = Utf8TempDir::new().unwrap(); + write_backend_canister(tmp.path(), "openemail/canisters/backend"); + write( + tmp.path(), + "openemail/icp.yaml", + r#" +canisters: + - ./canisters/backend +environments: + - name: staging + settings: + backend: + environment_variables: + API_KEY: + path: secrets/api-key +"#, + ); + write( + tmp.path(), + "openemail/canisters/backend/secrets/api-key", + "member-key\n", + ); + write( + tmp.path(), + "icp.yaml", + &format!( + "{}environments:\n - name: staging\n", + manifest( + &["app"], + "dependencies:\n - name: openemail\n path: ./openemail\n" + ) + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + let staging = p.environments.get("staging").expect("staging environment"); + + assert_eq!( + env_vars_of(&staging.canisters, "openemail:backend"), + &HashMap::from([("API_KEY".to_string(), "member-key".to_string())]), + ); + } + + #[tokio::test] + async fn missing_env_var_file_is_reported_with_the_variable_and_canister() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + r#" +canisters: + - name: backend + settings: + environment_variables: + API_KEY: + path: secrets/api-key + build: + steps: + - type: pre-built + path: backend.wasm +"#, + ); + + let err = consolidate(tmp.path()) + .await + .expect_err("the environment variable's file does not exist"); + + assert!( + matches!( + &err, + ConsolidateManifestError::ReadEnvironmentVariable { canister, variable, .. } + if canister == "backend" && variable == "API_KEY" + ), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn friendly_names_are_bare_for_own_and_dotted_for_dependencies() { + let tmp = Utf8TempDir::new().unwrap(); + // openemail (with a transitive dep libfoo) vendored under the app. + write( + tmp.path(), + "openemail/libfoo/icp.yaml", + &manifest(&["bar"], ""), + ); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest( + &["backend", "frontend"], + "dependencies:\n - name: libfoo\n path: ./libfoo\n", + ), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Own canister: bare name (unchanged from single-project behavior). + assert_eq!(friendly_names_of(&p, "backend"), &["backend"]); + // Direct dependency: dot-nested by alias (no `vendor/` path noise). + assert_eq!( + friendly_names_of(&p, "openemail:backend"), + &["backend.openemail"] + ); + assert_eq!( + friendly_names_of(&p, "openemail:frontend"), + &["frontend.openemail"] + ); + // Transitive dependency: full alias chain, canister-most-specific first. + assert_eq!( + friendly_names_of(&p, "openemail/libfoo:bar"), + &["bar.libfoo.openemail"] + ); + } + + #[tokio::test] + async fn cycle_is_detected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest(&[], "dependencies:\n - name: a\n path: ./a\n"), + ); + write( + tmp.path(), + "a/icp.yaml", + &manifest(&["x"], "dependencies:\n - name: b\n path: ../b\n"), + ); + write( + tmp.path(), + "b/icp.yaml", + &manifest(&["y"], "dependencies:\n - name: a\n path: ../a\n"), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::CircularDependency { .. }), + "expected CircularDependency, got {err:?}" + ); + } + + #[tokio::test] + async fn alias_colliding_with_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["openemail"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DependencyAliasCollision { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn duplicate_alias_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "one/icp.yaml", &manifest(&["backend"], "")); + write(tmp.path(), "two/icp.yaml", &manifest(&["backend"], "")); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: dup\n path: ./one\n - name: dup\n path: ./two\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::DuplicateDependencyAlias { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn colon_in_canister_name_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write(tmp.path(), "icp.yaml", &manifest(&["foo:bar"], "")); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn unknown_exposed_canister_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [nope]\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!( + err, + ConsolidateManifestError::UnknownDependencyCanister { .. } + ), + "got {err:?}" + ); + } + + #[tokio::test] + async fn missing_dependency_path_is_rejected() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "icp.yaml", + &manifest( + &[], + "dependencies:\n - name: openemail\n path: ./does-not-exist\n", + ), + ); + + let err = consolidate(tmp.path()).await.unwrap_err(); + assert!( + matches!(err, ConsolidateManifestError::DependencyNotFound { .. }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn imported_canisters_appear_in_implicit_environments() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); + + let p = consolidate(tmp.path()).await.unwrap(); + + // Deploy-all: the implicit `local` environment includes the dependency. + let local = p.environments.get("local").unwrap(); + assert!(local.canisters.contains_key("backend")); + assert!(local.canisters.contains_key("openemail:backend")); + } +} diff --git a/crates/icp-deploy-canister/src/sync_exec.rs b/crates/icp-deploy-canister/src/sync_exec.rs new file mode 100644 index 000000000..7047459b8 --- /dev/null +++ b/crates/icp-deploy-canister/src/sync_exec.rs @@ -0,0 +1,472 @@ +//! Injected sync-step execution. +//! +//! A canister's sync steps run either a WASI plugin (wasmtime) or a subprocess +//! script — neither can run inside a canister — so their execution is provided +//! by the host through [`PluginExecutor`] and [`ScriptRunner`]. This crate keeps +//! *all* of the derivation, though: it dispatches on the step kind, resolves the +//! plugin inputs, and assembles the `ICP_CLI_*` system environment variables +//! scripts run with. The host implementations only perform the irreducible host +//! action — fetch-and-run-the-wasm, or spawn-the-subprocess — against a +//! fully-resolved [`PluginInvocation`] / [`ScriptInvocation`]. +//! +//! The two executors are separate traits because an environment can support one +//! without the other. Script steps are host-only, and are rejected by +//! [`crate::project::verify_sandbox`] before they reach an executor. + +use std::collections::BTreeMap; + +use async_trait::async_trait; +use candid::Principal; +use snafu::prelude::*; + +use crate::manifest::adapter::{ + plugin::{self, NamedPaths}, + prebuilt::SourceField, + script, +}; +use crate::prelude::*; + +/// Resolved context for executing one canister's sync steps. +#[derive(Clone, Debug)] +pub struct SyncStepContext { + /// Directory the canister was declared in (base for relative plugin paths). + pub canister_path: PathBuf, + /// The canister being synced. + pub canister_id: Principal, + /// Store key of the canister being synced (e.g. `backend`, or + /// `services/open-crm:backend` for a canister in a subproject) — the `name` + /// of its [`Canister`](crate::Canister). Its namespace prefix + /// identifies which other canisters are in the same subproject. + pub canister_name: String, + /// Name of the environment being synced (e.g. "local", "production"). + pub environment: String, + /// Name of the network (e.g. "local", "ic"). + pub network: String, + /// IDs of all named canisters in the project for this environment. + pub canister_ids: BTreeMap, + /// Proxy canister to route calls through, if `--proxy` was passed. + pub proxy: Option, +} + +/// A manifest-declared path, tagged with the map key it was declared under. +/// A plain-list entry carries no key. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KeyedPath { + /// The `dirs:`/`files:` map key this path sits under, or `None` for a + /// plain-list entry. Non-unique: the paths of a key that maps to a list all + /// share it. + pub key: Option, + /// The path itself, relative to the canister directory. + pub path: String, +} + +/// Convert a manifest [`NamedPaths`] (or its absence) into the key-tagged path +/// list the executor receives. A missing setting yields an empty list. +fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { + paths + .into_iter() + .flat_map(NamedPaths::entries) + .map(|entry| KeyedPath { + key: entry.key.map(str::to_string), + path: entry.path.to_string(), + }) + .collect() +} + +/// A fully-resolved WASI-plugin sync step. Everything the host needs to fetch +/// and run the plugin has been computed by this crate; the host supplies only +/// the wasm source resolution (through its +/// [`RemoteResourceResolve`](crate::canister::recipe::RemoteResourceResolve)) +/// and the wasmtime runtime, plus its own identity/agent state. +#[derive(Clone, Debug)] +pub struct PluginInvocation { + /// Where the plugin wasm comes from (local path or remote URL). + pub source: SourceField, + /// Optional sha256 the host verifies the wasm against (required for remote). + pub sha256: Option, + /// Canister directory; base for the relative `dirs`/`files` and the source. + pub base_dir: PathBuf, + /// Directories preopened read-only into the WASI sandbox. + pub dirs: Vec, + /// Files the host reads and passes inline to the plugin. + pub files: Vec, + /// Key-value fields passed inline to the plugin. + pub fields: BTreeMap, + /// The canister being synced, which the plugin may always call. + pub canister_id: Principal, + /// Environment name exposed to the plugin via its `SyncExecInput`. + pub environment: String, + /// The canister ID table exposed to the plugin: every named canister in + /// the project, plus a bare-local-name alias for each canister in the same + /// subproject as the one being synced. + pub canister_ids: BTreeMap, + /// The canisters the step's `canisters:` list named, resolved to ids. These + /// are callable in addition to [`canister_id`](Self::canister_id). + pub callable: BTreeMap, + /// Proxy canister to route the plugin's canister calls through, if any. + pub proxy: Option, +} + +/// A plugin step named a canister in its `canisters:` list that the environment +/// does not have. +#[derive(Debug, Snafu)] +#[snafu(display( + "sync plugin lists canister '{name}' as callable, but no canister by that name \ + is known in environment '{environment}'" +))] +pub struct UnknownCallableCanisterError { + name: String, + environment: String, +} + +impl PluginInvocation { + /// Resolve a plugin step's adapter against the sync context. Fails if the + /// step declares a callable canister the environment does not have. + pub fn new( + adapter: &plugin::Adapter, + ctx: &SyncStepContext, + ) -> Result { + let canister_ids = exposed_canister_ids(ctx); + let callable = resolve_callable(adapter, &canister_ids, &ctx.environment)?; + Ok(Self { + source: adapter.source.clone(), + sha256: adapter.sha256.clone(), + base_dir: ctx.canister_path.clone(), + dirs: keyed_paths(adapter.dirs.as_ref()), + files: keyed_paths(adapter.files.as_ref()), + fields: adapter.fields.clone().unwrap_or_default(), + canister_id: ctx.canister_id, + environment: ctx.environment.clone(), + canister_ids, + callable, + proxy: ctx.proxy, + }) + } +} + +/// The canister ID table exposed to a sync plugin: every named canister in the +/// project, plus — for canisters in the same subproject as the one being synced +/// — a duplicate entry under the bare local name. A store key is +/// `:` for a canister in a subproject and a bare local name +/// for a canister defined directly in the app root, so the syncing canister's +/// namespace is the prefix of its own key. +/// +/// A local name never contains a colon but a subproject directory may, so keys +/// split on their *last* colon. The bare-name aliases take precedence over an +/// app-root canister of the same local name: a plugin resolving a bare name is +/// naming what the syncing canister's own manifest calls it. +fn exposed_canister_ids(ctx: &SyncStepContext) -> BTreeMap { + let syncing_namespace = ctx + .canister_name + .rsplit_once(':') + .map(|(namespace, _)| namespace); + + let mut table = ctx.canister_ids.clone(); + for (key, id) in &ctx.canister_ids { + if let Some((namespace, local)) = key.rsplit_once(':') + && Some(namespace) == syncing_namespace + { + table.insert(local.to_owned(), *id); + } + } + table +} + +/// Resolve the step's `canisters:` list against `canister_ids`. A name that does +/// not resolve is a manifest error. +fn resolve_callable( + adapter: &plugin::Adapter, + canister_ids: &BTreeMap, + environment: &str, +) -> Result, UnknownCallableCanisterError> { + let mut by_name = BTreeMap::new(); + for name in adapter.canisters.iter().flatten() { + let principal = canister_ids + .get(name) + .copied() + .context(UnknownCallableCanisterSnafu { + name: name.clone(), + environment: environment.to_owned(), + })?; + by_name.insert(name.clone(), principal); + } + Ok(by_name) +} + +/// A fully-resolved script sync step. This crate has already assembled the +/// working directory and the complete environment the subprocess runs with +/// (see [`system_env_vars`]); the host only spawns the command(s). +#[derive(Clone, Debug)] +pub struct ScriptInvocation { + /// Shell command(s) to run in order. + pub commands: Vec, + /// Working directory (the canister directory). + pub cwd: PathBuf, + /// Environment variables the subprocess inherits, in insertion order. + pub env: Vec<(String, String)>, +} + +impl ScriptInvocation { + /// Resolve a script step's adapter against the sync context, assembling the + /// `ICP_CLI_*` system environment variables the command runs with. + pub fn new(adapter: &script::Adapter, ctx: &SyncStepContext) -> Self { + Self { + commands: adapter.command.as_vec(), + cwd: ctx.canister_path.clone(), + env: system_env_vars(ctx), + } + } +} + +/// The `ICP_CLI_*` system environment variables every script sync step runs +/// with: the environment and network names, the target canister id, and one +/// `ICP_CLI_CID_` per known canister in the environment (name uppercased, +/// non-alphanumerics replaced with `_`). +pub fn system_env_vars(ctx: &SyncStepContext) -> Vec<(String, String)> { + let mut envs = vec![ + ("ICP_CLI_ENVIRONMENT".to_owned(), ctx.environment.clone()), + ("ICP_CLI_NETWORK".to_owned(), ctx.network.clone()), + ("ICP_CLI_CID".to_owned(), ctx.canister_id.to_text()), + ]; + for (name, id) in &ctx.canister_ids { + let key = format!( + "ICP_CLI_CID_{}", + name.to_uppercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect::() + ); + envs.push((key, id.to_text())); + } + envs +} + +/// A sink for streamed sync-step output lines (a presentation concern the host +/// implements, e.g. over a progress bar). +pub trait StepProgress: Send + Sync { + fn line(&self, line: String); +} + +/// A plugin step failed. The concrete cause (a host wasm/runtime error) is boxed +/// because this crate does not depend on the executor's implementation; callers +/// can still walk `source()`. +#[derive(Debug, Snafu)] +#[snafu(display("plugin sync step failed"))] +pub struct PluginExecutorError { + pub source: Box, +} + +/// Host execution of WASI-plugin sync steps. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait PluginExecutor: Send + Sync { + /// Fetch and run a WASI plugin against a canister, returning any stderr + /// lines the plugin emitted that should be retained past the streamed view. + async fn run_plugin( + &self, + invocation: PluginInvocation, + progress: Option<&dyn StepProgress>, + ) -> Result, PluginExecutorError>; +} + +/// A script step failed. Boxed for the same reason as [`PluginExecutorError`]. +#[derive(Debug, Snafu)] +#[snafu(display("script sync step failed"))] +pub struct ScriptRunError { + pub source: Box, +} + +/// Host execution of subprocess script sync steps. +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +pub trait ScriptRunner: Send + Sync { + /// Run a resolved script step, returning any stderr lines to retain past the + /// streamed view. + async fn run_script( + &self, + invocation: ScriptInvocation, + progress: Option<&dyn StepProgress>, + ) -> Result, ScriptRunError>; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::adapter::prebuilt::{LocalSource, SourceField}; + + fn principal(byte: u8) -> Principal { + Principal::from_slice(&[byte; 4]) + } + + fn ctx_named(name: &str, ids: &[(&str, Principal)]) -> SyncStepContext { + SyncStepContext { + canister_path: "/work".into(), + canister_id: principal(0), + canister_name: name.to_owned(), + environment: "demo".to_owned(), + network: "ic".to_owned(), + canister_ids: ids.iter().map(|(n, p)| ((*n).to_owned(), *p)).collect(), + proxy: None, + } + } + + fn adapter_with(canisters: Option>) -> plugin::Adapter { + plugin::Adapter { + source: SourceField::Local(LocalSource { + path: "plugin.wasm".into(), + }), + sha256: None, + dirs: None, + files: None, + fields: None, + canisters, + } + } + + /// Canisters sharing the syncing canister's subproject are additionally + /// exposed under their bare local name; canisters in other subprojects are + /// not. + #[test] + fn exposed_ids_add_bare_names_for_same_subproject() { + let backend = principal(1); + let frontend = principal(2); + let foreign = principal(3); + let ctx = ctx_named( + "services/open-accounts:backend", + &[ + ("services/open-accounts:backend", backend), + ("services/open-accounts:frontend", frontend), + ("services/open-crm:backend", foreign), + ], + ); + + let table = exposed_canister_ids(&ctx); + + // Same-subproject canisters gain a bare-local duplicate... + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + // ...while the fully-qualified keys are still present for everyone. + assert_eq!( + table.get("services/open-accounts:frontend"), + Some(&frontend) + ); + assert_eq!(table.get("services/open-crm:backend"), Some(&foreign)); + } + + /// An app-root canister sharing a local name with a sibling of the syncing + /// canister does not keep the bare name: the syncing subproject's own + /// canister is what that name means to the plugin. + #[test] + fn exposed_ids_sibling_alias_overrides_the_app_root_name() { + let root_backend = principal(1); + let sibling_backend = principal(2); + let ctx = ctx_named( + "services/open-accounts:frontend", + &[ + ("backend", root_backend), + ("services/open-accounts:backend", sibling_backend), + ("services/open-accounts:frontend", principal(3)), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("backend"), Some(&sibling_backend)); + // The app-root canister's only key was that bare name, so it drops out + // of the table entirely rather than answering to a sibling's name. + assert!(!table.values().any(|id| *id == root_backend)); + } + + /// A subproject directory may itself contain a colon, so keys are split on + /// their last one — the same rule bundling uses. + #[test] + fn exposed_ids_split_subproject_prefix_at_the_last_colon() { + let backend = principal(1); + let frontend = principal(2); + let ctx = ctx_named( + "services/odd:name:backend", + &[ + ("services/odd:name:backend", backend), + ("services/odd:name:frontend", frontend), + ], + ); + + let table = exposed_canister_ids(&ctx); + + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + } + + /// A single-project layout keys canisters by bare local name already, so no + /// duplicates are added. + #[test] + fn exposed_ids_unchanged_without_a_subproject() { + let backend = principal(1); + let ctx = ctx_named("backend", &[("backend", backend)]); + let table = exposed_canister_ids(&ctx); + assert_eq!(table.len(), 1); + assert_eq!(table.get("backend"), Some(&backend)); + } + + #[test] + fn resolve_callable_resolves_names() { + let dep = principal(1); + let sibling = principal(2); + let table = BTreeMap::from([ + ("backend".to_owned(), sibling), + ("services/open-crm:backend".to_owned(), dep), + ]); + let adapter = adapter_with(Some(vec![ + "backend".to_owned(), + "services/open-crm:backend".to_owned(), + ])); + + let callable = resolve_callable(&adapter, &table, "demo").unwrap(); + + assert_eq!(callable.get("backend"), Some(&sibling)); + assert_eq!(callable.get("services/open-crm:backend"), Some(&dep)); + } + + #[test] + fn resolve_callable_rejects_unknown_name() { + let adapter = adapter_with(Some(vec!["nope".to_owned()])); + resolve_callable(&adapter, &BTreeMap::new(), "demo") + .expect_err("an undeclared name must fail"); + } + + /// A script step resolves to the manifest's commands, the canister directory + /// as cwd, and the `ICP_CLI_*` environment assembled from the context. + #[test] + fn script_invocation_resolves_commands_cwd_and_env() { + use crate::manifest::adapter::script::{Adapter, CommandField}; + + let cid = principal(7); + let frontend = principal(8); + let ctx = SyncStepContext { + canister_path: "/work/backend".into(), + canister_id: cid, + canister_name: "backend".to_owned(), + environment: "production".to_owned(), + network: "ic".to_owned(), + canister_ids: BTreeMap::from([("my-frontend".to_owned(), frontend)]), + proxy: None, + }; + let adapter = Adapter { + command: CommandField::Command("./deploy.sh".to_owned()), + }; + + let invocation = ScriptInvocation::new(&adapter, &ctx); + + assert_eq!(invocation.commands, vec!["./deploy.sh"]); + assert_eq!(invocation.cwd, PathBuf::from("/work/backend")); + assert_eq!( + invocation.env, + vec![ + ("ICP_CLI_ENVIRONMENT".to_owned(), "production".to_owned()), + ("ICP_CLI_NETWORK".to_owned(), "ic".to_owned()), + ("ICP_CLI_CID".to_owned(), cid.to_text()), + ("ICP_CLI_CID_MY_FRONTEND".to_owned(), frontend.to_text()), + ] + ); + } +} diff --git a/crates/icp-deploy-canister/src/testutil.rs b/crates/icp-deploy-canister/src/testutil.rs new file mode 100644 index 000000000..cdafc37bd --- /dev/null +++ b/crates/icp-deploy-canister/src/testutil.rs @@ -0,0 +1,63 @@ +//! Test-only helpers. + +use async_trait::async_trait; + +use crate::files::{FileAccess, FileAccessError}; +use crate::prelude::*; + +/// A [`FileAccess`] backed by the real host filesystem, for unit tests that +/// write manifests to a temp dir and consolidate them. +pub struct HostFiles; + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +impl FileAccess for HostFiles { + async fn read_file(&self, path: &Path) -> Result, FileAccessError> { + std::fs::read(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn read_to_string(&self, path: &Path) -> Result { + std::fs::read_to_string(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn exists(&self, path: &Path) -> bool { + path.exists() + } + + async fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + async fn is_dir(&self, path: &Path) -> bool { + path.is_dir() + } + + async fn read_dir(&self, path: &Path) -> Result, FileAccessError> { + let rd = std::fs::read_dir(path).map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + let mut out = Vec::new(); + for entry in rd { + let entry = entry.map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + if let Ok(p) = PathBuf::from_path_buf(entry.path()) { + out.push(p); + } + } + Ok(out) + } + + async fn canonicalize(&self, path: &Path) -> Option { + let c = std::fs::canonicalize(path).ok()?; + PathBuf::from_path_buf(c).ok() + } +} diff --git a/crates/icp/Cargo.toml b/crates/icp/Cargo.toml index 40f310b15..e0a3253e2 100644 --- a/crates/icp/Cargo.toml +++ b/crates/icp/Cargo.toml @@ -25,7 +25,6 @@ elliptic-curve = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } glob = { workspace = true } -handlebars = { workspace = true } hex = { workspace = true } hmac = { workspace = true } hybrid-array = { workspace = true } @@ -36,6 +35,7 @@ ic-ledger-types = { workspace = true } ic-management-canister-types = { workspace = true } ic-utils = { workspace = true } icp-canister-interfaces = { workspace = true } +icp-deploy-canister = { workspace = true } icp-sync-plugin = { workspace = true } icrc-ledger-types = { workspace = true } indexmap = { workspace = true } @@ -80,6 +80,11 @@ uuid = { workspace = true } wslpath2 = { workspace = true } zeroize = { workspace = true } +[features] +# Enables `clap::ValueEnum` derives on CLI-facing enums, including the manifest +# enums now defined in `icp-deploy-canister`. +clap = ["dep:clap", "icp-deploy-canister/clap"] + [target.'cfg(windows)'.dependencies] winreg = { workspace = true } diff --git a/crates/icp/src/canister/build/prebuilt.rs b/crates/icp/src/canister/build/prebuilt.rs index 774a102f9..c54164143 100644 --- a/crates/icp/src/canister/build/prebuilt.rs +++ b/crates/icp/src/canister/build/prebuilt.rs @@ -1,7 +1,12 @@ use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{canister::wasm, fs, manifest::adapter::prebuilt::Adapter, package::PackageCache}; +use crate::{ + canister::{ChannelProgress, wasm}, + fs, + manifest::adapter::prebuilt::Adapter, + package::PackageCache, +}; use super::Params; @@ -20,11 +25,12 @@ pub(super) async fn build( stdio: Option>, pkg_cache: &PackageCache, ) -> Result<(), PrebuiltError> { + let progress = ChannelProgress::wrap(stdio.as_ref()); let src = wasm::resolve( &adapter.source, ¶ms.path, adapter.sha256.as_deref(), - stdio.as_ref(), + ChannelProgress::as_dyn(progress.as_ref()), pkg_cache, ) .await?; diff --git a/crates/icp/src/canister/mod.rs b/crates/icp/src/canister/mod.rs index e5277333d..a1abf66a9 100644 --- a/crates/icp/src/canister/mod.rs +++ b/crates/icp/src/canister/mod.rs @@ -1,13 +1,17 @@ -use std::collections::HashMap; - -use candid::{Nat, Principal}; -use ic_management_canister_types::{CanisterSettings, LogVisibility}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -use crate::{ - parsers::{CyclesAmount, DurationAmount, MemoryAmount}, - prelude::*, +//! Host-side canister facade. +//! +//! The canister *model* (`Settings`, `ControllerRef`, `resolve_controllers`, +//! log-visibility types, and the `RemoteResourceResolve` interface) lives in +//! `icp_deploy_canister::canister` and is re-exported here. The build/sync/wasm +//! *executors* (which spawn processes, run wasmtime, and fetch over HTTP) stay +//! here. + +use icp_deploy_canister::sync_exec::StepProgress; +use tokio::sync::mpsc::Sender; + +pub use icp_deploy_canister::canister::{ + ControllerRef, LogVisibilityDef, LogVisibilitySimple, ManifestEnvVar, ManifestSettings, + Settings, resolve_controllers, }; pub mod build; @@ -17,673 +21,27 @@ pub mod sync; mod script; pub mod wasm; -/// Controls who can read canister logs. -/// Supports both string format ("controllers", "public") and object format ({ allowed_viewers: [...] }). -#[derive(Clone, Debug, PartialEq, Serialize)] -#[serde(untagged)] -pub enum LogVisibilityDef { - /// Simple string variants for controllers or public - Simple(LogVisibilitySimple), - /// Object format with allowed_viewers list - AllowedViewers { allowed_viewers: Vec }, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum LogVisibilitySimple { - Controllers, - Public, -} - -impl<'de> Deserialize<'de> for LogVisibilityDef { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::{Error, MapAccess, Visitor}; - use std::fmt; - - struct LogVisibilityVisitor; - - impl<'de> Visitor<'de> for LogVisibilityVisitor { - type Value = LogVisibilityDef; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("'controllers', 'public', or object with 'allowed_viewers'") - } - - fn visit_str(self, value: &str) -> Result { - LogVisibilitySimple::deserialize( - serde::de::value::StrDeserializer::::new(value), - ) - .map(LogVisibilityDef::Simple) - .map_err(|_| { - E::custom(format!( - "unknown log_visibility value: '{}', expected 'controllers' or 'public'", - value - )) - }) - } - - fn visit_map(self, mut map: M) -> Result - where - M: MapAccess<'de>, - { - let mut allowed_viewers: Option> = None; - - while let Some(key) = map.next_key::()? { - match key.as_str() { - "allowed_viewers" => { - if allowed_viewers.is_some() { - return Err(Error::duplicate_field("allowed_viewers")); - } - allowed_viewers = Some(map.next_value()?); - } - _ => { - return Err(Error::unknown_field(&key, &["allowed_viewers"])); - } - } - } - - allowed_viewers - .map(|v| LogVisibilityDef::AllowedViewers { allowed_viewers: v }) - .ok_or_else(|| Error::missing_field("allowed_viewers")) - } - } - - deserializer.deserialize_any(LogVisibilityVisitor) - } -} - -impl JsonSchema for LogVisibilityDef { - fn schema_name() -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed("LogVisibility") - } - - fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "description": "Controls who can read canister logs.", - "oneOf": [ - { - "type": "string", - "enum": ["controllers", "public"], - "description": "Simple log visibility: 'controllers' (only controllers can view) or 'public' (anyone can view)" - }, - { - "type": "object", - "properties": { - "allowed_viewers": { - "type": "array", - "items": { - "type": "string", - "description": "A principal ID that can view logs" - }, - "description": "List of principal IDs that can view canister logs" - } - }, - "required": ["allowed_viewers"], - "additionalProperties": false, - "description": "Specific principals that can view logs" - } - ] - }) - } -} - -impl From for LogVisibility { - fn from(value: LogVisibilityDef) -> Self { - match value { - LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) => { - LogVisibility::Controllers - } - LogVisibilityDef::Simple(LogVisibilitySimple::Public) => LogVisibility::Public, - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - LogVisibility::AllowedViewers(allowed_viewers) - } - } - } -} - -/// A reference to a controller: either an explicit principal or a canister name in this project. -/// -/// During deserialization, principal text format is tried first; strings that don't parse as a -/// principal are treated as canister names. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ControllerRef { - /// An explicitly specified principal (e.g. "2vxsx-fae") - Principal(candid::Principal), - /// A canister name from the same project (e.g. "my_canister") - CanisterName(String), -} - -impl ControllerRef { - /// Resolve to a `Principal` using the provided ID mapping. - /// Returns `None` if this is a `CanisterName` not present in `ids`. - pub fn resolve(&self, ids: &crate::store_id::IdMapping) -> Option { - match self { - ControllerRef::Principal(p) => Some(*p), - ControllerRef::CanisterName(name) => ids.get(name).copied(), - } - } - - /// If this is a `CanisterName`, returns the name; otherwise `None`. - pub fn canister_name(&self) -> Option<&str> { - match self { - ControllerRef::CanisterName(n) => Some(n), - ControllerRef::Principal(_) => None, - } - } -} - -/// Partition a slice of controller references into resolved principals and unresolved canister -/// names, using `ids` for name lookup. -pub fn resolve_controllers( - crefs: &[ControllerRef], - ids: &crate::store_id::IdMapping, -) -> (Vec, Vec) { - let mut resolved = Vec::new(); - let mut unresolved = Vec::new(); - for cref in crefs { - match cref.resolve(ids) { - Some(p) => resolved.push(p), - None => { - if let Some(name) = cref.canister_name() { - unresolved.push(name.to_owned()); - } - } - } - } - (resolved, unresolved) -} - -impl schemars::JsonSchema for ControllerRef { - fn schema_name() -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed("ControllerRef") - } - - fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "type": "string", - "description": "A controller: either a principal text (e.g. '2vxsx-fae') or a canister name in this project (e.g. 'my_canister')" - }) - } -} - -/// An environment variable value as written in a manifest. -/// -/// A plain scalar is the value itself: -/// ```yaml -/// environment_variables: -/// API_ENDPOINT: https://api.example.com -/// ``` -/// -/// The object form reads the value from a file, relative to the canister's own -/// directory — including when an environment overrides the variable, matching how -/// an `init_args` override resolves its path: -/// ```yaml -/// environment_variables: -/// API_KEY: -/// path: ./secrets/api-key -/// ``` -#[derive(Clone, Debug, PartialEq, Deserialize, JsonSchema, Serialize)] -#[serde(untagged, expecting = "a string, or `{ path: }`")] -pub enum ManifestEnvVar { - /// The value, written inline. - Value(String), - /// A file holding the value. Surrounding whitespace is trimmed off the - /// file's contents, so a trailing newline does not become part of the value. - Path { - #[schemars(with = "String")] - path: PathBuf, - }, -} - -impl Default for ManifestEnvVar { - fn default() -> Self { - Self::Value(String::new()) - } -} - -/// Canister settings loaded from a manifest, before file-backed environment -/// variable values have been read. See [`Settings`] for the resolved form. -pub type ManifestSettings = Settings; - -/// Canister settings, such as compute and memory allocation. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Settings { - /// Controls who can read canister logs. - #[serde(skip_serializing_if = "Option::is_none")] - pub log_visibility: Option, - - /// Compute allocation (0 to 100). Represents guaranteed compute capacity. - #[serde(skip_serializing_if = "Option::is_none")] - pub compute_allocation: Option, - - /// Memory allocation in bytes. If unset, memory is allocated dynamically. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_allocation: Option, - - /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. - /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). - #[serde(skip_serializing_if = "Option::is_none")] - pub freezing_threshold: Option, - - /// Upper limit on cycles reserved for future resource payments. - /// Memory allocations that would push the reserved balance above this limit will fail. - /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reserved_cycles_limit: Option, - - /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub wasm_memory_limit: Option, - - /// Wasm memory threshold in bytes. Triggers a callback when exceeded. - /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). - #[serde(skip_serializing_if = "Option::is_none")] - pub wasm_memory_threshold: Option, - - /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. - /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. - #[serde(skip_serializing_if = "Option::is_none")] - pub log_memory_limit: Option, - - /// Environment variables for the canister as key-value pairs. - /// These variables are accessible within the canister and can be used to configure - /// behavior without hardcoding values in the WASM module. - /// A value may also be read from a file with `{ path: }`. - #[serde(skip_serializing_if = "Option::is_none")] - pub environment_variables: Option>, - - /// Controllers for this canister. Each entry is either a principal text - /// (e.g. "2vxsx-fae") or the name of another canister in this project. - /// Named canisters that do not yet exist will be set as controllers once created. - #[serde(default)] - pub controllers: Option>, -} - -impl From for ManifestSettings { - fn from(settings: Settings) -> Self { - let Settings { - log_visibility, - compute_allocation, - memory_allocation, - freezing_threshold, - reserved_cycles_limit, - wasm_memory_limit, - wasm_memory_threshold, - log_memory_limit, - environment_variables, - controllers, - } = settings; +/// Adapts a streamed-output channel to the library's [`StepProgress`] line sink, +/// so host code that already owns a channel can hand one to the library's IO +/// traits. +pub struct ChannelProgress(pub Sender); - Self { - log_visibility, - compute_allocation, - memory_allocation, - freezing_threshold, - reserved_cycles_limit, - wasm_memory_limit, - wasm_memory_threshold, - log_memory_limit, - environment_variables: environment_variables.map(|vars| { - vars.into_iter() - .map(|(name, value)| (name, ManifestEnvVar::Value(value))) - .collect() - }), - controllers, - } +impl StepProgress for ChannelProgress { + fn line(&self, line: String) { + // Status lines are advisory: drop them rather than block the caller if + // the display has fallen behind. + let _ = self.0.try_send(line); } } -impl From for CanisterSettings { - fn from(settings: Settings) -> Self { - CanisterSettings { - freezing_threshold: settings.freezing_threshold.map(|d| Nat::from(d.get())), - controllers: None, - reserved_cycles_limit: settings.reserved_cycles_limit.map(|c| Nat::from(c.get())), - log_visibility: settings.log_visibility.map(Into::into), - memory_allocation: settings.memory_allocation.map(|m| Nat::from(m.get())), - compute_allocation: settings.compute_allocation.map(Nat::from), - ..Default::default() - } +impl ChannelProgress { + /// Wrap an optional channel, as the library's IO traits take it. + pub fn wrap(stdio: Option<&Sender>) -> Option { + stdio.cloned().map(Self) } -} - -#[cfg(test)] -mod tests { - use indoc::indoc; - - use super::*; - - #[test] - fn log_visibility_deserialize_controllers() { - let yaml = "controllers"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - LogVisibilityDef::Simple(LogVisibilitySimple::Controllers) - ); - } - - #[test] - fn log_visibility_deserialize_public() { - let yaml = "public"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - LogVisibilityDef::Simple(LogVisibilitySimple::Public) - ); - } - - #[test] - fn log_visibility_deserialize_allowed_viewers() { - let yaml = r#" -allowed_viewers: - - "aaaaa-aa" - - "2vxsx-fae" -"#; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - match result { - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - assert_eq!(allowed_viewers.len(), 2); - assert_eq!( - allowed_viewers[0], - Principal::from_text("aaaaa-aa").unwrap() - ); - assert_eq!( - allowed_viewers[1], - Principal::from_text("2vxsx-fae").unwrap() - ); - } - _ => panic!("Expected AllowedViewers variant"), - } - } - - #[test] - fn log_visibility_deserialize_allowed_viewers_empty() { - let yaml = "allowed_viewers: []"; - let result: LogVisibilityDef = serde_yaml::from_str(yaml).unwrap(); - match result { - LogVisibilityDef::AllowedViewers { allowed_viewers } => { - assert!(allowed_viewers.is_empty()); - } - _ => panic!("Expected AllowedViewers variant"), - } - } - - #[test] - fn log_visibility_deserialize_invalid_string() { - let yaml = "invalid"; - let result: Result = serde_yaml::from_str(yaml); - assert!(result.is_err()); - let err = result.unwrap_err().to_string(); - assert!(err.contains("unknown log_visibility value")); - } - - #[test] - fn log_visibility_deserialize_invalid_field() { - let yaml = "unknown_field: []"; - let result: Result = serde_yaml::from_str(yaml); - assert!(result.is_err()); - let err = result.unwrap_err().to_string(); - assert!(err.contains("unknown field")); - } - - #[test] - fn log_visibility_serialize_controllers() { - let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert_eq!(yaml.trim(), "controllers"); - } - - #[test] - fn log_visibility_serialize_public() { - let log_vis = LogVisibilityDef::Simple(LogVisibilitySimple::Public); - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert_eq!(yaml.trim(), "public"); - } - - #[test] - fn log_visibility_serialize_allowed_viewers() { - let log_vis = LogVisibilityDef::AllowedViewers { - allowed_viewers: vec![ - Principal::from_text("aaaaa-aa").unwrap(), - Principal::from_text("2vxsx-fae").unwrap(), - ], - }; - let yaml = serde_yaml::to_string(&log_vis).unwrap(); - assert!(yaml.contains("allowed_viewers")); - assert!(yaml.contains("aaaaa-aa")); - assert!(yaml.contains("2vxsx-fae")); - } - - #[test] - fn settings_reserved_cycles_limit_parses_suffix() { - let yaml = "reserved_cycles_limit: 4.3t"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.reserved_cycles_limit.as_ref().map(|c| c.get()), - Some(4_300_000_000_000) - ); - } - - #[test] - fn settings_reserved_cycles_limit_parses_number() { - let yaml = "reserved_cycles_limit: 5000000000000"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.reserved_cycles_limit.as_ref().map(|c| c.get()), - Some(5_000_000_000_000) - ); - } - - #[test] - fn settings_memory_allocation_parses_suffix() { - let yaml = "memory_allocation: 4gib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.memory_allocation.as_ref().map(|m| m.get()), - Some(4 * 1024 * 1024 * 1024) - ); - } - - #[test] - fn settings_memory_allocation_parses_number() { - let yaml = "memory_allocation: 4294967296"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.memory_allocation.as_ref().map(|m| m.get()), - Some(4294967296) - ); - } - - #[test] - fn settings_wasm_memory_limit_parses_suffix() { - let yaml = "wasm_memory_limit: 1.5gib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.wasm_memory_limit.as_ref().map(|m| m.get()), - Some(1610612736) - ); - } - - #[test] - fn settings_log_memory_limit_parses_suffix() { - let yaml = "log_memory_limit: 256kib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.log_memory_limit.as_ref().map(|m| m.get()), - Some(256 * 1024) - ); - } - - #[test] - fn settings_log_memory_limit_parses_mib() { - let yaml = "log_memory_limit: 2mib"; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.log_memory_limit.as_ref().map(|m| m.get()), - Some(2 * 1024 * 1024) - ); - } - - #[test] - fn settings_environment_variables_take_values_or_files() { - let yaml = indoc! {r#" - environment_variables: - API_ENDPOINT: https://api.example.com - API_KEY: - path: ./secrets/api-key - "#}; - let settings: ManifestSettings = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - settings.environment_variables, - Some(HashMap::from([ - ( - "API_ENDPOINT".to_owned(), - ManifestEnvVar::Value("https://api.example.com".to_owned()), - ), - ( - "API_KEY".to_owned(), - ManifestEnvVar::Path { - path: "./secrets/api-key".into(), - }, - ), - ])), - ); - } - - #[test] - fn settings_environment_variable_rejects_unknown_object_form() { - let yaml = indoc! {r#" - environment_variables: - API_KEY: - file: ./secrets/api-key - "#}; - let err = serde_yaml::from_str::(yaml) - .expect_err("only the `path` object form is accepted"); - assert!( - err.to_string().contains("a string, or `{ path: }`"), - "unhelpful error: {err}" - ); - } - - /// A value of the wrong scalar type reports what is accepted, rather than - /// serde's default "did not match any variant" for an untagged enum. - #[test] - fn settings_environment_variable_rejects_non_string_scalar() { - let err = - serde_yaml::from_str::("environment_variables:\n PORT: 8080\n") - .expect_err("a bare integer is not a value"); - assert!( - err.to_string().contains("a string, or `{ path: }`"), - "unhelpful error: {err}" - ); - } - - #[test] - fn resolved_settings_serialize_environment_variables_inline() { - let settings = Settings { - environment_variables: Some(HashMap::from([( - "API_KEY".to_owned(), - "s3cret".to_owned(), - )])), - ..Default::default() - }; - let yaml = serde_yaml::to_string(&ManifestSettings::from(settings)).unwrap(); - assert!( - yaml.contains("environment_variables:\n API_KEY: s3cret\n"), - "unexpected yaml: {yaml}" - ); - } - - #[test] - fn controller_ref_deserializes_principal() { - let yaml = "\"2vxsx-fae\""; - let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - ControllerRef::Principal(Principal::from_text("2vxsx-fae").unwrap()) - ); - } - - #[test] - fn controller_ref_deserializes_canister_name() { - let yaml = "\"my_canister\""; - let result: ControllerRef = serde_yaml::from_str(yaml).unwrap(); - assert_eq!( - result, - ControllerRef::CanisterName("my_canister".to_owned()) - ); - } - - #[test] - fn controller_ref_resolve_principal() { - let p = Principal::from_text("aaaaa-aa").unwrap(); - let cref = ControllerRef::Principal(p); - let ids = crate::store_id::IdMapping::new(); - assert_eq!(cref.resolve(&ids), Some(p)); - } - - #[test] - fn controller_ref_resolve_canister_name_present() { - let p = Principal::from_text("aaaaa-aa").unwrap(); - let cref = ControllerRef::CanisterName("backend".to_owned()); - let mut ids = crate::store_id::IdMapping::new(); - ids.insert("backend".to_owned(), p); - assert_eq!(cref.resolve(&ids), Some(p)); - } - - #[test] - fn controller_ref_resolve_canister_name_absent() { - let cref = ControllerRef::CanisterName("backend".to_owned()); - let ids = crate::store_id::IdMapping::new(); - assert_eq!(cref.resolve(&ids), None); - } - - #[test] - fn settings_controllers_parses_mixed() { - let yaml = r#" -controllers: - - "aaaaa-aa" - - "my_other_canister" -"#; - let settings: Settings = serde_yaml::from_str(yaml).unwrap(); - let controllers = settings.controllers.unwrap(); - assert_eq!(controllers.len(), 2); - assert_eq!( - controllers[0], - ControllerRef::Principal(Principal::from_text("aaaaa-aa").unwrap()) - ); - assert_eq!( - controllers[1], - ControllerRef::CanisterName("my_other_canister".to_owned()) - ); - } - - #[test] - fn log_visibility_conversion_to_ic_type() { - let controllers = LogVisibilityDef::Simple(LogVisibilitySimple::Controllers); - let ic_controllers: LogVisibility = controllers.into(); - assert!(matches!(ic_controllers, LogVisibility::Controllers)); - - let public = LogVisibilityDef::Simple(LogVisibilitySimple::Public); - let ic_public: LogVisibility = public.into(); - assert!(matches!(ic_public, LogVisibility::Public)); - let viewers = LogVisibilityDef::AllowedViewers { - allowed_viewers: vec![Principal::from_text("aaaaa-aa").unwrap()], - }; - let ic_viewers: LogVisibility = viewers.into(); - match ic_viewers { - LogVisibility::AllowedViewers(v) => { - assert_eq!(v.len(), 1); - } - _ => panic!("Expected AllowedViewers"), - } + /// Borrow as the trait object the library's IO traits take. + pub fn as_dyn(this: Option<&Self>) -> Option<&dyn StepProgress> { + this.map(|p| p as &dyn StepProgress) } } diff --git a/crates/icp/src/canister/recipe/mod.rs b/crates/icp/src/canister/recipe/mod.rs index edb92caf3..eeb806a2d 100644 --- a/crates/icp/src/canister/recipe/mod.rs +++ b/crates/icp/src/canister/recipe/mod.rs @@ -1,59 +1,17 @@ -//! Recipe resolution, split into two stages. +//! Host-side recipe facade. //! -//! [`fetch`] retrieves a recipe's Handlebars template — reading a local file, or -//! downloading a remote URL or registry recipe — and returns the raw template -//! text. [`render`] turns that text into concrete build/sync steps. The first -//! stage does I/O and nothing else; the second is a pure function. +//! The [`RemoteResourceResolve`] interface, recipe rendering, and the +//! context/error types live in [`icp_deploy_canister::canister::recipe`]; the +//! concrete resolver — which fetches templates and plugin wasms over HTTP and +//! caches them — stays here in [`resolver`]. //! -//! The [`Resolve`] seam therefore covers only the fetching half, so a caller that -//! already has a template (or must not touch the network) can render without -//! going through a resolver at all. -//! -//! Caching a download is a third step, because whether a template is worth -//! keeping is not known until it renders. A download that carried a `sha256` is -//! cached during the fetch — the checksum already proves the bytes are the ones -//! that were asked for. An *unpinned* download is held back as a -//! [`PendingCache`] and only committed by the caller once rendering succeeds, so -//! that one bad remote response cannot become sticky in the cache. The full -//! sequence is therefore fetch → render → [`Resolve::commit`]. - -use async_trait::async_trait; -use snafu::prelude::*; - -use crate::manifest::recipe::Recipe; - -pub mod fetch; -pub mod render; - -pub use fetch::{Fetched, PendingCache}; -pub use render::{RecipeContext, RenderRecipeError, render_recipe}; - -/// Retrieves the recipe templates a project references. -/// -/// Only *fetching* is behind this trait: rendering a fetched template into build -/// and sync steps is [`render_recipe`], which needs no I/O and so needs no seam. -#[async_trait] -pub trait Resolve: Sync + Send { - /// Fetch the Handlebars template for `recipe`, returning its raw source and - /// any cache write held back until the template is known to render. - async fn resolve(&self, recipe: &Recipe) -> Result; - - /// Write a held-back download to the cache, now that it has rendered. - /// - /// Defaults to doing nothing: only [`fetch::RecipeFetcher`] caches, and only - /// it can construct the [`PendingCache`] that reaches this method, so a - /// resolver that never defers a write never has one to commit. - async fn commit(&self, pending: PendingCache) -> Result<(), ResolveError> { - let _ = pending; - Ok(()) - } -} +//! Resolution is therefore staged: fetch the template, render it, then commit +//! whatever the resolver held back. See [`resolver::ResourceResolver`] for why +//! the cache write waits. -#[derive(Debug, Snafu)] -pub enum ResolveError { - #[snafu(display("failed to fetch recipe template"))] - Fetch { source: fetch::RecipeFetchError }, +pub use icp_deploy_canister::canister::recipe::{ + FetchedRecipe, RecipeContext, RemoteResourceResolve, RenderRecipeError, ResolveError, + render_recipe, +}; - #[snafu(display("failed to cache recipe template"))] - Commit { source: fetch::RecipeFetchError }, -} +pub mod resolver; diff --git a/crates/icp/src/canister/recipe/fetch.rs b/crates/icp/src/canister/recipe/resolver.rs similarity index 69% rename from crates/icp/src/canister/recipe/fetch.rs rename to crates/icp/src/canister/recipe/resolver.rs index cb6ca5c2d..cb1601abb 100644 --- a/crates/icp/src/canister/recipe/fetch.rs +++ b/crates/icp/src/canister/recipe/resolver.rs @@ -1,8 +1,7 @@ -//! Stage one of recipe resolution: get the template text. - use std::{str::FromStr, string::FromUtf8Error}; use async_trait::async_trait; +use icp_deploy_canister::sync_exec::StepProgress; use reqwest::{Method, Request, Url}; use sha2::{Digest, Sha256}; use snafu::prelude::*; @@ -19,53 +18,27 @@ use crate::{ prelude::*, }; -use super::{CommitSnafu, FetchSnafu, Resolve, ResolveError}; +use super::{FetchedRecipe, RemoteResourceResolve, ResolveError}; +use crate::manifest::adapter::prebuilt::SourceField; -/// Fetches recipe templates over HTTP, caching downloads in the package cache. -/// Template *rendering* is a separate stage -/// ([`render_recipe`](super::render_recipe)); this only produces the raw template -/// text. -pub struct RecipeFetcher { +/// Fetches recipe templates and plugin wasms over HTTP, caching downloads in the +/// package cache. Template *rendering* is the library's job +/// ([`icp_deploy_canister::canister::recipe::render_recipe`]); this only produces +/// the raw template text. +/// +/// Whether a download is worth keeping is not known until it renders, so an +/// *unpinned* download is marked [`FetchedRecipe::deferred`] and only written to +/// the cache when the caller reports back through +/// [`commit_recipe`](RemoteResourceResolve::commit_recipe). A checksummed +/// download is cached during the fetch — the checksum already proves the bytes +/// are the ones that were asked for. +pub struct ResourceResolver { /// Http client for fetching remote recipe templates pub http_client: reqwest::Client, /// Package cache for caching downloaded recipe templates pub pkg_cache: PackageCache, } -/// The result of the fetch stage. -pub struct Fetched { - /// Raw Handlebars template source. - pub template: String, - - /// A cache write deliberately held back until the template is known to - /// render; `None` when there is nothing to cache (a local file or a cache - /// hit) or when the download was already cached because it was checksummed. - /// - /// Pass to [`Resolve::commit`] after [`render_recipe`](super::render_recipe) - /// succeeds. - pub pending_cache: Option, -} - -/// A cache write for an unpinned download, held until the template renders. -/// -/// A checksummed download is cached the moment its checksum verifies: the -/// checksum is what establishes the bytes are the ones that were asked for, and -/// refetching would only produce the same bytes again. An unpinned download has -/// no such guarantee — caching it before it is known good would let a single bad -/// response become sticky, and every later resolution would read those bytes -/// back instead of refetching. -pub struct PendingCache { - target: CacheTarget, - hash: [u8; 32], - template: String, -} - -/// Where a fetched template belongs in the package cache. -enum CacheTarget { - Uri(String), - Registry { package: String, version: String }, -} - enum TemplateSource { LocalPath(PathBuf), RemoteUrl(String), @@ -110,27 +83,18 @@ pub enum RecipeFetchError { LockCache { source: crate::fs::lock::LockError }, } -impl RecipeFetcher { +impl ResourceResolver { /// Fetch a recipe's Handlebars template text: read a local file, or fetch a - /// remote URL or registry recipe. Verifies `sha256` when set. + /// remote URL or registry recipe (serving it from the package cache when + /// possible). Verifies `sha256` when set. /// - /// A checksummed download is cached here. An unpinned one is returned as a - /// [`PendingCache`] for the caller to commit once it renders — see - /// [`PendingCache`] for why. - async fn fetch_recipe(&self, recipe: &Recipe) -> Result { - // Determine the template source - let tmpl_source = match &recipe.recipe_type { - RecipeType::File(path) => TemplateSource::LocalPath(Path::new(&path).into()), - RecipeType::Url(url) => TemplateSource::RemoteUrl(url.to_owned()), - RecipeType::Registry { - name, - recipe, - version, - } => TemplateSource::Registry(name.to_owned(), recipe.to_owned(), version.to_owned()), - }; - - // Retrieve the template, using cache for remote/registry sources - let (tmpl, should_cache) = match &tmpl_source { + /// A checksummed download is cached here. An unpinned one is returned with + /// [`FetchedRecipe::deferred`] set, for the caller to commit once it renders. + async fn fetch_recipe(&self, recipe: &Recipe) -> Result { + // Retrieve the template, using cache for remote/registry sources. The + // flag says whether the bytes were freshly downloaded, and so are the + // only ones that could still need caching. + let (tmpl, downloaded) = match &template_source(&recipe.recipe_type) { TemplateSource::LocalPath(path) => { let bytes = read(path).context(ReadFileSnafu)?; (parse_bytes_to_string(bytes)?, false) @@ -188,69 +152,46 @@ impl RecipeFetcher { } }; - let hash = if let Some(sha256) = &recipe.sha256 { - verify_checksum(tmpl.as_bytes(), sha256)? - } else { - Sha256::digest(tmpl.as_bytes()).into() - }; - - // Nothing was downloaded (local file, or a cache hit): nothing to cache. - if !should_cache { - return Ok(Fetched { + if let Some(sha256) = &recipe.sha256 { + verify_checksum(tmpl.as_bytes(), sha256)?; + // The checksum matched, so refetching could only produce these same + // bytes: there is nothing to gain by waiting for a render that may + // never succeed. + if downloaded { + self.cache_recipe(recipe, &tmpl).await?; + } + return Ok(FetchedRecipe { template: tmpl, - pending_cache: None, + deferred: false, }); } - let target = match tmpl_source { - TemplateSource::LocalPath(_) => unreachable!("local files are never cached"), - TemplateSource::RemoteUrl(u) => CacheTarget::Uri(u), - TemplateSource::Registry(registry, recipe_name, version) => CacheTarget::Registry { - package: format!("@{registry}/{recipe_name}"), - version, - }, - }; - - let pending = PendingCache { - target, - hash, + Ok(FetchedRecipe { template: tmpl, - }; - - // A checksummed download is trustworthy the moment the checksum matches, - // so cache it now. An unpinned one waits for a successful render. - if recipe.sha256.is_some() { - self.write_cache(&pending).await?; - return Ok(Fetched { - template: pending.template, - pending_cache: None, - }); - } - - Ok(Fetched { - template: pending.template.clone(), - pending_cache: Some(pending), + deferred: downloaded, }) } - /// Write a fetched template into the package cache. - async fn write_cache(&self, pending: &PendingCache) -> Result<(), RecipeFetchError> { - let hash = hex::encode(pending.hash); - let bytes = pending.template.as_bytes(); - match &pending.target { - CacheTarget::Uri(u) => { + /// Cache a template downloaded by [`Self::fetch_recipe`]. For an unpinned + /// download this runs only after the caller has rendered it, so a malformed + /// response never becomes the entry that later project loads reuse. + async fn cache_recipe(&self, recipe: &Recipe, tmpl: &str) -> Result<(), RecipeFetchError> { + let hash = hex::encode(Sha256::digest(tmpl.as_bytes())); + match template_source(&recipe.recipe_type) { + TemplateSource::LocalPath(_) => unreachable!("local files are never cached"), + TemplateSource::RemoteUrl(u) => { self.pkg_cache .with_write(async |w| { - cache_uri_recipe(w, u, &hash, bytes).context(CacheRecipeSnafu)?; - Ok(()) + cache_uri_recipe(w, &u, &hash, tmpl.as_bytes()).context(CacheRecipeSnafu) }) .await .context(LockCacheSnafu)??; } - CacheTarget::Registry { package, version } => { + TemplateSource::Registry(registry, recipe_name, version) => { + let package = format!("@{registry}/{recipe_name}"); self.pkg_cache .with_write(async |w| { - cache_registry_recipe(w, package, version, &hash, bytes) + cache_registry_recipe(w, &package, &version, &hash, tmpl.as_bytes()) .context(CacheRecipeSnafu) }) .await @@ -284,24 +225,61 @@ impl RecipeFetcher { } #[async_trait] -impl Resolve for RecipeFetcher { - async fn resolve(&self, recipe: &Recipe) -> Result { - self.fetch_recipe(recipe).await.context(FetchSnafu) +impl RemoteResourceResolve for ResourceResolver { + async fn resolve_recipe(&self, recipe: &Recipe) -> Result { + self.fetch_recipe(recipe) + .await + .map_err(|source| ResolveError::Resolve { + source: Box::new(source), + }) + } + + async fn commit_recipe( + &self, + recipe: &Recipe, + fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + if !fetched.deferred { + return Ok(()); + } + self.cache_recipe(recipe, &fetched.template) + .await + .map_err(|source| ResolveError::Resolve { + source: Box::new(source), + }) + } + + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + progress: Option<&dyn StepProgress>, + ) -> Result { + crate::canister::wasm::resolve(source, base_dir, sha256, progress, &self.pkg_cache) + .await + .map_err(|source| ResolveError::ResolveWasm { + source: Box::new(source), + }) } +} - async fn commit(&self, pending: PendingCache) -> Result<(), ResolveError> { - self.write_cache(&pending).await.context(CommitSnafu) +/// Classify where a recipe's template comes from. +fn template_source(recipe_type: &RecipeType) -> TemplateSource { + match recipe_type { + RecipeType::File(path) => TemplateSource::LocalPath(Path::new(&path).into()), + RecipeType::Url(url) => TemplateSource::RemoteUrl(url.to_owned()), + RecipeType::Registry { + name, + recipe, + version, + } => TemplateSource::Registry(name.to_owned(), recipe.to_owned(), version.to_owned()), } } /// Helper function to verify sha256 checksum of recipe template bytes -fn verify_checksum(bytes: &[u8], expected: &str) -> Result<[u8; 32], RecipeFetchError> { - let actual_hash = { - let mut h = Sha256::new(); - h.update(bytes); - h.finalize() - }; - let actual = hex::encode(actual_hash); +fn verify_checksum(bytes: &[u8], expected: &str) -> Result<(), RecipeFetchError> { + let actual = hex::encode(Sha256::digest(bytes)); if actual != expected { return ChecksumMismatchSnafu { expected: expected.to_string(), @@ -309,7 +287,7 @@ fn verify_checksum(bytes: &[u8], expected: &str) -> Result<[u8; 32], RecipeFetch } .fail(); } - Ok(actual_hash.into()) + Ok(()) } /// Helper function to parse bytes into a UTF-8 string @@ -320,10 +298,11 @@ fn parse_bytes_to_string(bytes: Vec) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::canister::recipe::{RecipeContext, render_recipe}; use crate::manifest::recipe::{Recipe, RecipeType}; - fn fetcher(cache_dir: &Path) -> RecipeFetcher { - RecipeFetcher { + fn resolver(cache_dir: &Path) -> ResourceResolver { + ResourceResolver { http_client: reqwest::Client::new(), pkg_cache: PackageCache::new(cache_dir.to_owned()).unwrap(), } @@ -349,15 +328,12 @@ mod tests { sha256: None, }; - let fetched = fetcher(&tmp.path().join("pkg")) + let fetched = resolver(&tmp.path().join("pkg")) .fetch_recipe(&recipe) .await .unwrap(); assert_eq!(fetched.template, body); - assert!( - fetched.pending_cache.is_none(), - "local files are never cached" - ); + assert!(!fetched.deferred, "a local file has nothing to cache"); } /// A sha256 that does not match the template contents is rejected. @@ -374,7 +350,9 @@ mod tests { }; assert!(matches!( - fetcher(&tmp.path().join("pkg")).fetch_recipe(&recipe).await, + resolver(&tmp.path().join("pkg")) + .fetch_recipe(&recipe) + .await, Err(RecipeFetchError::ChecksumMismatch { .. }) )); } @@ -431,7 +409,7 @@ mod tests { let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); let cache_dir = tmp.path().join("pkg"); - let f = fetcher(&cache_dir); + let r = resolver(&cache_dir); let recipe = Recipe { recipe_type: RecipeType::Url(url), configuration: Default::default(), @@ -439,18 +417,18 @@ mod tests { }; // Fetch succeeds and hands back a held-back cache write. - let fetched = f.fetch_recipe(&recipe).await.expect("first fetch"); + let fetched = r.fetch_recipe(&recipe).await.expect("first fetch"); assert!( - fetched.pending_cache.is_some(), + fetched.deferred, "an unpinned download must defer its cache write" ); // Rendering fails, so the caller never commits. - let ctx = super::super::RecipeContext { + let ctx = RecipeContext { canister_name: "c".to_owned(), }; assert!( - super::super::render_recipe(&fetched.template, &recipe, &ctx).is_err(), + render_recipe(&fetched.template, &recipe, &ctx).is_err(), "fixture template must fail to render" ); @@ -463,7 +441,7 @@ mod tests { // bad bytes from cache. assert!( matches!( - f.fetch_recipe(&recipe).await, + r.fetch_recipe(&recipe).await, Err(RecipeFetchError::HttpStatus { status: 500, .. }) ), "second resolution must refetch, not read the uncommitted template back" @@ -484,26 +462,23 @@ mod tests { let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); let cache_dir = tmp.path().join("pkg"); - let f = fetcher(&cache_dir); + let r = resolver(&cache_dir); let recipe = Recipe { recipe_type: RecipeType::Url(url), configuration: Default::default(), sha256: None, }; - let fetched = f.fetch_recipe(&recipe).await.expect("first fetch"); - let pending = fetched.pending_cache.expect("unpinned defers its write"); - f.write_cache(&pending).await.expect("commit"); + let fetched = r.fetch_recipe(&recipe).await.expect("first fetch"); + assert!(fetched.deferred, "unpinned defers its write"); + r.commit_recipe(&recipe, &fetched).await.expect("commit"); assert!(cache_has_template(&cache_dir)); // Served from cache now, even though the server would answer 500. - let again = f.fetch_recipe(&recipe).await.expect("second fetch"); + let again = r.fetch_recipe(&recipe).await.expect("second fetch"); assert_eq!(again.template, fetched.template); - assert!( - again.pending_cache.is_none(), - "a cache hit has nothing to commit" - ); + assert!(!again.deferred, "a cache hit has nothing to commit"); } /// A checksummed download is cached during the fetch: the checksum already @@ -516,16 +491,16 @@ mod tests { let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); let cache_dir = tmp.path().join("pkg"); - let f = fetcher(&cache_dir); + let r = resolver(&cache_dir); let recipe = Recipe { recipe_type: RecipeType::Url(url), configuration: Default::default(), sha256: Some(hex::encode(Sha256::digest(UNRENDERABLE.as_bytes()))), }; - let fetched = f.fetch_recipe(&recipe).await.expect("first fetch"); + let fetched = r.fetch_recipe(&recipe).await.expect("first fetch"); assert!( - fetched.pending_cache.is_none(), + !fetched.deferred, "a checksummed download is cached during the fetch" ); assert!(cache_has_template(&cache_dir)); diff --git a/crates/icp/src/canister/script.rs b/crates/icp/src/canister/script.rs index 6974a745d..665259599 100644 --- a/crates/icp/src/canister/script.rs +++ b/crates/icp/src/canister/script.rs @@ -65,8 +65,8 @@ pub(super) async fn execute( /// Takes already-resolved commands rather than an [`Adapter`], so the subprocess /// executor needs to know nothing about manifest types. The sync path resolves /// its commands and `ICP_CLI_*` environment into a -/// [`ScriptInvocation`](super::sync::script::ScriptInvocation) before calling -/// this; the build path calls [`execute`] with its adapter. +/// [`ScriptInvocation`](icp_deploy_canister::sync_exec::ScriptInvocation) before +/// calling this; the build path calls [`execute`] with its adapter. pub(super) async fn execute_commands( cmds: &[String], cwd: &Path, diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index a90ff93be..199d10d71 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -1,105 +1,75 @@ -use std::collections::BTreeMap; -use std::sync::Arc; - use async_trait::async_trait; -use candid::Principal; use ic_agent::Agent; +use icp_deploy_canister::canister::recipe::RemoteResourceResolve; +use icp_deploy_canister::sync_exec::{PluginInvocation, ScriptInvocation}; use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::manifest::canister::SyncStep; -use crate::package::PackageCache; -use crate::prelude::*; - mod plugin; -pub mod script; - -use script::{HostScripts, ScriptInvocation, ScriptRunError, ScriptRunner}; - -pub struct Params { - pub path: PathBuf, - pub cid: Principal, - /// Fully-qualified store key of the canister being synced (e.g. `backend`, - /// or `services/open-crm:backend` for a canister in a subproject). Its namespace - /// prefix identifies which other canisters are in the same subproject. - pub name: String, - /// Name of the environment being synced (e.g. "local", "production"). - /// Passed to sync plugin steps via `SyncExecInput`. - pub environment: String, - /// Name of the network (e.g. "local", "ic"). - pub network: String, - /// IDs of all named canisters in the project for this environment. - pub canister_ids: BTreeMap, - /// Proxy canister to route calls through, if `--proxy` was passed. - pub proxy: Option, -} #[derive(Debug, Snafu)] pub enum SynchronizeError { #[snafu(transparent)] - Script { source: ScriptRunError }, + Script { source: super::script::ScriptError }, #[snafu(transparent)] Plugin { source: plugin::PluginError }, } +/// Host execution of the two sync-step mechanisms that can't run inside a +/// canister: WASI plugins (wasmtime) and subprocess scripts. +/// +/// Step dispatch and *all* input derivation (plugin dirs/files, the `ICP_CLI_*` +/// script environment) live in `icp-deploy-canister`; implementations here +/// receive a fully-resolved [`PluginInvocation`] / [`ScriptInvocation`] and +/// perform only the irreducible host action. This trait is the injection seam +/// the [`crate::context::Context`] carries so tests can stub it out. #[async_trait] pub trait Synchronize: Sync + Send { - async fn sync( + async fn run_plugin( &self, - step: &SyncStep, - params: &Params, + invocation: &PluginInvocation, agent: &Agent, stdio: Option>, - pkg_cache: &PackageCache, + resolver: &dyn RemoteResourceResolve, ) -> Result, SynchronizeError>; -} -/// Dispatches each sync step to the machinery that runs it. Plugin steps run in -/// the wasmtime WASI sandbox, which this drives directly; script steps go through -/// an injected [`ScriptRunner`], since spawning a subprocess is not available -/// everywhere. -pub struct Syncer { - scripts: Arc, + async fn run_script( + &self, + invocation: &ScriptInvocation, + stdio: Option>, + ) -> Result, SynchronizeError>; } -impl Syncer { - /// A syncer that runs script steps as host subprocesses. - pub fn host() -> Self { - Self::new(Arc::new(HostScripts)) - } - - pub fn new(scripts: Arc) -> Self { - Self { scripts } - } -} +pub struct Syncer; #[async_trait] impl Synchronize for Syncer { - async fn sync( + async fn run_plugin( &self, - step: &SyncStep, - params: &Params, + invocation: &PluginInvocation, agent: &Agent, stdio: Option>, - pkg_cache: &PackageCache, + resolver: &dyn RemoteResourceResolve, + ) -> Result, SynchronizeError> { + Ok(plugin::run(invocation, agent, stdio, resolver).await?) + } + + async fn run_script( + &self, + invocation: &ScriptInvocation, + stdio: Option>, ) -> Result, SynchronizeError> { - match step { - SyncStep::Script(adapter) => Ok(self - .scripts - .run_script(ScriptInvocation::new(adapter, params), stdio) - .await?), - SyncStep::Plugin(adapter) => Ok(plugin::sync( - adapter, - params, - agent, - ¶ms.environment, - params.proxy, - stdio, - pkg_cache, - ) - .await?), - } + let env_refs: Vec<(&str, &str)> = invocation + .env + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + super::script::execute_commands(&invocation.commands, &invocation.cwd, &env_refs, stdio) + .await?; + // Persistent stderr is a sync-plugin feature only; script steps don't + // currently retain any output past the rolling step view. + Ok(vec![]) } } @@ -111,101 +81,21 @@ pub struct UnimplementedMockSyncer; #[cfg(test)] #[async_trait] impl Synchronize for UnimplementedMockSyncer { - async fn sync( + async fn run_plugin( &self, - _step: &SyncStep, - _params: &Params, + _invocation: &PluginInvocation, _agent: &Agent, _stdio: Option>, - _pkg_cache: &PackageCache, + _resolver: &dyn RemoteResourceResolve, ) -> Result, SynchronizeError> { - unimplemented!("UnimplementedMockSyncer::sync") - } -} - -#[cfg(test)] -mod tests { - use std::sync::Mutex; - - use crate::manifest::adapter::script::{Adapter, CommandField}; - - use super::*; - - /// A [`ScriptRunner`] that records what it was asked to run instead of - /// running it, so step dispatch can be tested without spawning a shell. - #[derive(Default)] - struct RecordingScripts { - seen: Mutex>, - } - - #[async_trait] - impl ScriptRunner for RecordingScripts { - async fn run_script( - &self, - invocation: ScriptInvocation, - _stdio: Option>, - ) -> Result, ScriptRunError> { - self.seen.lock().unwrap().push(invocation); - Ok(vec![]) - } + unimplemented!("UnimplementedMockSyncer::run_plugin") } - fn dummy_agent() -> Agent { - Agent::builder() - .with_url("http://127.0.0.1:4943") - .build() - .expect("build test agent") - } - - /// A script step reaches the injected runner fully resolved: the commands - /// from the manifest, the canister directory as cwd, and the `ICP_CLI_*` - /// environment assembled from the sync params. Nothing is spawned. - #[tokio::test] - async fn script_steps_are_dispatched_to_the_injected_runner() { - let scripts = Arc::new(RecordingScripts::default()); - let syncer = Syncer::new(scripts.clone()); - - let cid = Principal::from_slice(&[7; 4]); - let params = Params { - path: "/work/backend".into(), - cid, - name: "backend".to_owned(), - environment: "production".to_owned(), - network: "ic".to_owned(), - canister_ids: BTreeMap::from([( - "my-frontend".to_owned(), - Principal::from_slice(&[8; 4]), - )]), - proxy: None, - }; - let step = SyncStep::Script(Adapter { - command: CommandField::Command("./deploy.sh".to_owned()), - }); - - let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); - let pkg_cache = PackageCache::new(tmp.path().to_owned()).unwrap(); - - let retained = syncer - .sync(&step, ¶ms, &dummy_agent(), None, &pkg_cache) - .await - .expect("script step should dispatch"); - assert!(retained.is_empty()); - - let seen = scripts.seen.lock().unwrap(); - assert_eq!(seen.len(), 1); - assert_eq!(seen[0].commands, vec!["./deploy.sh"]); - assert_eq!(seen[0].cwd, PathBuf::from("/work/backend")); - assert_eq!( - seen[0].env, - vec![ - ("ICP_CLI_ENVIRONMENT".to_owned(), "production".to_owned()), - ("ICP_CLI_NETWORK".to_owned(), "ic".to_owned()), - ("ICP_CLI_CID".to_owned(), cid.to_text()), - ( - "ICP_CLI_CID_MY_FRONTEND".to_owned(), - Principal::from_slice(&[8; 4]).to_text() - ), - ] - ); + async fn run_script( + &self, + _invocation: &ScriptInvocation, + _stdio: Option>, + ) -> Result, SynchronizeError> { + unimplemented!("UnimplementedMockSyncer::run_script") } } diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index fbde4dde4..25f6c9d84 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,8 +1,7 @@ -use std::collections::BTreeMap; - use camino::Utf8PathBuf; -use candid::Principal; use ic_agent::Agent; +use icp_deploy_canister::canister::recipe::{RemoteResourceResolve, ResolveError}; +use icp_deploy_canister::sync_exec; use icp_sync_plugin::{ CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, @@ -10,31 +9,12 @@ use icp_sync_plugin::{ use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{ - canister::wasm, - manifest::adapter::plugin::{Adapter, NamedPaths}, - package::PackageCache, -}; - -use super::Params; - -/// Convert a manifest [`NamedPaths`] (or its absence) into the runtime's -/// key-tagged path list. A missing setting yields an empty list. -fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { - paths - .into_iter() - .flat_map(NamedPaths::entries) - .map(|entry| KeyedPath { - key: entry.key.map(str::to_string), - path: entry.path.to_string(), - }) - .collect() -} +use crate::canister::ChannelProgress; #[derive(Debug, Snafu)] pub enum PluginError { - #[snafu(transparent)] - Wasm { source: wasm::WasmError }, + #[snafu(display("failed to resolve plugin wasm"))] + ResolveWasm { source: ResolveError }, #[snafu(display("failed to get identity principal: {err}"))] GetIdentityPrincipal { err: String }, @@ -46,12 +26,6 @@ pub enum PluginError { #[snafu(display("failed to run plugin"))] Run { source: RunPluginError }, - - #[snafu(display( - "sync plugin lists canister '{name}' as callable, but no canister by that name \ - is known in environment '{environment}'" - ))] - UnknownCallableCanister { name: String, environment: String }, } /// Resolve the plugin compute-time limit, honoring the @@ -82,14 +56,29 @@ fn parse_compute_limit(value: &str) -> Result { } } -pub(super) async fn sync( - adapter: &Adapter, - params: &Params, +/// Restate the library's key-tagged paths as the runtime's. The two types are +/// identical by construction and separate only because the runtime crate cannot +/// be depended on from `icp-deploy-canister`. +fn keyed_paths(paths: &[sync_exec::KeyedPath]) -> Vec { + paths + .iter() + .map(|entry| KeyedPath { + key: entry.key.clone(), + path: entry.path.clone(), + }) + .collect() +} + +/// Fetch and run a WASI plugin against a canister for a fully-resolved +/// [`sync_exec::PluginInvocation`]. Dispatch and input derivation — the +/// key-tagged paths, the fields, the exposed canister-id table and the resolved +/// `canisters:` list — happen in `icp-deploy-canister`; this only performs the +/// host-only wasm resolution and wasmtime execution. +pub(super) async fn run( + invocation: &sync_exec::PluginInvocation, agent: &Agent, - environment: &str, - proxy: Option, stdio: Option>, - pkg_cache: &PackageCache, + resolver: &dyn RemoteResourceResolve, ) -> Result, PluginError> { // 0. Resolve the compute-time limit up front so a malformed // ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS fails fast — before downloading the @@ -100,105 +89,47 @@ pub(super) async fn sync( // - Local: sha256 is verified if present, then the original path is returned. // - Remote: downloaded to cache (sha256 required, enforced at parse time) and the // stable cache path is returned — no temp file needed. - let wasm_path = wasm::resolve( - &adapter.source, - ¶ms.path, - adapter.sha256.as_deref(), - stdio.as_ref(), - pkg_cache, - ) - .await?; - - // 2. Collect inputs as manifest strings. `run_plugin` preopens the `dirs` - // and reads the `files` itself — both anchored at `base_dir`, and both - // subject to the runtime's path-safety checks (no escaping or symlinked - // paths). - let base_dir = Utf8PathBuf::from(params.path.as_str()); - let dirs = keyed_paths(adapter.dirs.as_ref()); - let files = keyed_paths(adapter.files.as_ref()); - let fields: BTreeMap = adapter.fields.clone().unwrap_or_default(); - - // 3. Build the canister ID table exposed to the plugin, then resolve the - // step's `canisters` list against it. - let canister_ids = exposed_canister_ids(params); - let callable = resolve_callable(adapter, &canister_ids, environment)?; - - // 4. Run the plugin (blocking call — signal Tokio that this thread will block). + let progress = ChannelProgress::wrap(stdio.as_ref()); + let wasm_path = resolver + .resolve_wasm( + &invocation.source, + &invocation.base_dir, + invocation.sha256.as_deref(), + ChannelProgress::as_dyn(progress.as_ref()), + ) + .await + .context(ResolveWasmSnafu)?; + + // 2. `run_plugin` preopens the `dirs` and reads the `files` itself — both + // anchored at `base_dir`, and both subject to the runtime's path-safety + // checks (no escaping or symlinked paths). + let base_dir = Utf8PathBuf::from(invocation.base_dir.as_str()); + + // 3. Run the plugin (blocking call — signal Tokio that this thread will block). let identity_principal = agent .get_principal() .map_err(|err| PluginError::GetIdentityPrincipal { err })?; - let agent_clone = agent.clone(); - let environment_owned = environment.to_owned(); - let stdio_clone = stdio.clone(); - - tokio::task::block_in_place(|| { - run_plugin(PluginInvocation { - wasm_path, - base_dir, - dirs, - files, - fields, - host_canister_id: params.cid, - agent: agent_clone, - proxy, - identity_principal, - environment: environment_owned, - compute_limit_secs, - canister_ids, - callable, - stdio: stdio_clone, - }) - }) - .context(RunSnafu) -} - -/// The canister ID table exposed to a sync plugin: every named canister in the -/// project, plus — for canisters in the same subproject as the one being synced -/// — a duplicate entry under the bare local name. A store key is -/// `:` for a canister in a subproject and a bare local name -/// for a canister defined directly in the app root (see the WIT -/// `canister-id-entry` docs), so the syncing canister's namespace is the prefix -/// of its own key. -/// -/// A local name never contains a colon but a subproject directory may, so keys -/// split on their *last* colon. The bare-name aliases take precedence over an -/// app-root canister of the same local name: a plugin resolving a bare name is -/// naming what the syncing canister's own manifest calls it. -fn exposed_canister_ids(params: &Params) -> BTreeMap { - let syncing_namespace = params.name.rsplit_once(':').map(|(namespace, _)| namespace); - - let mut table = params.canister_ids.clone(); - for (key, id) in ¶ms.canister_ids { - if let Some((namespace, local)) = key.rsplit_once(':') - && Some(namespace) == syncing_namespace - { - table.insert(local.to_owned(), *id); - } - } - table -} - -/// Resolve the step's `canisters` list into a [`CallableCanisters`] enforcement -/// set. Each listed name is looked up in `canister_ids`; a name that does not -/// resolve is a manifest error. -fn resolve_callable( - adapter: &Adapter, - canister_ids: &BTreeMap, - environment: &str, -) -> Result { - let mut by_name = BTreeMap::new(); - for name in adapter.canisters.iter().flatten() { - let principal = canister_ids - .get(name) - .copied() - .context(UnknownCallableCanisterSnafu { - name: name.clone(), - environment: environment.to_owned(), - })?; - by_name.insert(name.clone(), principal); - } - Ok(CallableCanisters { by_name }) + let runtime_invocation = PluginInvocation { + wasm_path, + base_dir, + dirs: keyed_paths(&invocation.dirs), + files: keyed_paths(&invocation.files), + fields: invocation.fields.clone(), + host_canister_id: invocation.canister_id, + agent: agent.clone(), + proxy: invocation.proxy, + identity_principal, + environment: invocation.environment.clone(), + compute_limit_secs, + canister_ids: invocation.canister_ids.clone(), + callable: CallableCanisters { + by_name: invocation.callable.clone(), + }, + stdio, + }; + + tokio::task::block_in_place(|| run_plugin(runtime_invocation)).context(RunSnafu) } #[cfg(test)] @@ -223,153 +154,4 @@ mod tests { ); } } - - use crate::manifest::adapter::prebuilt::{LocalSource, SourceField}; - - fn principal(byte: u8) -> Principal { - Principal::from_slice(&[byte; 4]) - } - - fn params_named(name: &str, ids: &[(&str, Principal)]) -> Params { - Params { - path: "/work".into(), - cid: principal(0), - name: name.to_owned(), - environment: "demo".to_owned(), - network: "ic".to_owned(), - canister_ids: ids.iter().map(|(n, p)| ((*n).to_owned(), *p)).collect(), - proxy: None, - } - } - - fn adapter_with(canisters: Option>) -> Adapter { - Adapter { - source: SourceField::Local(LocalSource { - path: "plugin.wasm".into(), - }), - sha256: None, - dirs: None, - files: None, - fields: None, - canisters, - } - } - - /// Canisters sharing the syncing canister's subproject are additionally - /// exposed under their bare local name; canisters in other subprojects are - /// not. - #[test] - fn exposed_ids_add_bare_names_for_same_subproject() { - let backend = principal(1); - let frontend = principal(2); - let foreign = principal(3); - let params = params_named( - "services/open-accounts:backend", - &[ - ("services/open-accounts:backend", backend), - ("services/open-accounts:frontend", frontend), - ("services/open-crm:backend", foreign), - ], - ); - - let table = exposed_canister_ids(¶ms); - - // Same-subproject canisters gain a bare-local duplicate... - assert_eq!(table.get("backend"), Some(&backend)); - assert_eq!(table.get("frontend"), Some(&frontend)); - // ...while the fully-qualified keys are still present for everyone. - assert_eq!( - table.get("services/open-accounts:frontend"), - Some(&frontend) - ); - assert_eq!(table.get("services/open-crm:backend"), Some(&foreign)); - // The other subproject's canister is not reachable by a bare name; the - // bare "backend" belongs to the syncing canister's own subproject. - assert_eq!(table.get("backend"), Some(&backend)); - } - - /// An app-root canister sharing a local name with a sibling of the syncing - /// canister does not keep the bare name: the syncing subproject's own - /// canister is what that name means to the plugin. - #[test] - fn exposed_ids_sibling_alias_overrides_the_app_root_name() { - let root_backend = principal(1); - let sibling_backend = principal(2); - let params = params_named( - "services/open-accounts:frontend", - &[ - ("backend", root_backend), - ("services/open-accounts:backend", sibling_backend), - ("services/open-accounts:frontend", principal(3)), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("backend"), Some(&sibling_backend)); - // The app-root canister's only key was that bare name, so it drops out - // of the table entirely rather than answering to a sibling's name. - assert!(!table.values().any(|id| *id == root_backend)); - } - - /// A subproject directory may itself contain a colon, so keys are split on - /// their last one — the same rule bundling uses. - #[test] - fn exposed_ids_split_subproject_prefix_at_the_last_colon() { - let backend = principal(1); - let frontend = principal(2); - let params = params_named( - "services/odd:name:backend", - &[ - ("services/odd:name:backend", backend), - ("services/odd:name:frontend", frontend), - ], - ); - - let table = exposed_canister_ids(¶ms); - - assert_eq!(table.get("backend"), Some(&backend)); - assert_eq!(table.get("frontend"), Some(&frontend)); - } - - /// A single-project layout keys canisters by bare local name already, so no - /// duplicates are added. - #[test] - fn exposed_ids_unchanged_without_a_subproject() { - let backend = principal(1); - let params = params_named("backend", &[("backend", backend)]); - let table = exposed_canister_ids(¶ms); - assert_eq!(table.len(), 1); - assert_eq!(table.get("backend"), Some(&backend)); - } - - #[test] - fn resolve_callable_resolves_names() { - let dep = principal(1); - let sibling = principal(2); - let table = BTreeMap::from([ - ("backend".to_owned(), sibling), - ("services/open-crm:backend".to_owned(), dep), - ]); - let adapter = adapter_with(Some(vec![ - "backend".to_owned(), - "services/open-crm:backend".to_owned(), - ])); - - let callable = resolve_callable(&adapter, &table, "demo").unwrap(); - - assert_eq!(callable.by_name.get("backend"), Some(&sibling)); - assert_eq!( - callable.by_name.get("services/open-crm:backend"), - Some(&dep) - ); - } - - #[test] - fn resolve_callable_rejects_unknown_name() { - let adapter = adapter_with(Some(vec!["nope".to_owned()])); - let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") - .expect_err("an undeclared name must fail"); - assert!(matches!(err, PluginError::UnknownCallableCanister { .. })); - } } diff --git a/crates/icp/src/canister/sync/script.rs b/crates/icp/src/canister/sync/script.rs deleted file mode 100644 index e26d73171..000000000 --- a/crates/icp/src/canister/sync/script.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! Script sync steps, split into resolution and execution. -//! -//! [`ScriptInvocation::new`] resolves a manifest script step against the sync -//! [`Params`] — the command list, the working directory and the `ICP_CLI_*` -//! environment — without running anything. [`ScriptRunner`] then executes a -//! resolved invocation. -//! -//! Execution sits behind a trait because spawning a subprocess is the one part of -//! sync that cannot be done everywhere: plugin steps run inside the wasmtime WASI -//! sandbox, but a script step needs a shell. Keeping the split here means the -//! resolution half is portable and unit-testable, and an environment without -//! subprocesses can substitute a runner that refuses instead of losing the whole -//! sync path. - -use async_trait::async_trait; -use snafu::prelude::*; -use tokio::sync::mpsc::Sender; - -use crate::manifest::adapter::script::Adapter; -use crate::prelude::*; - -use super::Params; - -use super::super::script::execute_commands; - -/// A fully-resolved script sync step: the command(s), the working directory, and -/// the environment variables to set for them (see [`system_env_vars`]). -#[derive(Clone, Debug, PartialEq)] -pub struct ScriptInvocation { - /// Shell command(s) to run in order. - pub commands: Vec, - /// Working directory (the canister directory). - pub cwd: PathBuf, - /// Environment variables a runner **adds to** its execution environment, in - /// insertion order — not the complete environment. [`HostScripts`] overlays - /// them onto the inherited process environment, so the script still sees - /// ambient variables such as `PATH` and `HOME`; entries here win on a name - /// collision. A runner is not expected to clear what it inherits. - pub env: Vec<(String, String)>, -} - -impl ScriptInvocation { - /// Resolve a script step's adapter against the sync context, assembling the - /// `ICP_CLI_*` system environment variables the command runs with. - pub fn new(adapter: &Adapter, params: &Params) -> Self { - Self { - commands: adapter.command.as_vec(), - cwd: params.path.clone(), - env: system_env_vars(params), - } - } -} - -/// The `ICP_CLI_*` system environment variables every script sync step runs -/// with: the environment and network names, the target canister id, and one -/// `ICP_CLI_CID_` per known canister in the environment (name uppercased, -/// non-alphanumerics replaced with `_`). -pub fn system_env_vars(params: &Params) -> Vec<(String, String)> { - let mut envs = vec![ - ("ICP_CLI_ENVIRONMENT".to_owned(), params.environment.clone()), - ("ICP_CLI_NETWORK".to_owned(), params.network.clone()), - ("ICP_CLI_CID".to_owned(), params.cid.to_text()), - ]; - for (name, id) in ¶ms.canister_ids { - let key = format!( - "ICP_CLI_CID_{}", - name.to_uppercase() - .chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .collect::() - ); - envs.push((key, id.to_text())); - } - envs -} - -#[derive(Debug, Snafu)] -#[snafu(display("script sync step failed"))] -pub struct ScriptRunError { - /// Boxed because the error depends on how the runner executes: the host - /// runner fails with a [`ScriptError`](super::super::script::ScriptError), a - /// runner that refuses scripts fails with something else entirely. - pub source: Box, -} - -/// Executes resolved script sync steps. -#[async_trait] -pub trait ScriptRunner: Sync + Send { - /// Run a resolved script step, streaming output to `stdio`, and return any - /// stderr lines to retain past the streamed view. - async fn run_script( - &self, - invocation: ScriptInvocation, - stdio: Option>, - ) -> Result, ScriptRunError>; -} - -/// The [`ScriptRunner`] that spawns each command as a host subprocess. -pub struct HostScripts; - -#[async_trait] -impl ScriptRunner for HostScripts { - async fn run_script( - &self, - invocation: ScriptInvocation, - stdio: Option>, - ) -> Result, ScriptRunError> { - let env_refs: Vec<(&str, &str)> = invocation - .env - .iter() - .map(|(k, v)| (k.as_str(), v.as_str())) - .collect(); - execute_commands(&invocation.commands, &invocation.cwd, &env_refs, stdio) - .await - .map_err(|source| ScriptRunError { - source: Box::new(source), - })?; - // Persistent stderr is a sync-plugin feature only; script steps don't - // currently retain any output past the rolling step view. - Ok(vec![]) - } -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use tokio::sync::Mutex; - - use candid::Principal; - - use super::*; - use crate::manifest::adapter::script::CommandField; - - /// Serializes the tests here that mutate the process environment, since - /// cargo runs tests in parallel threads. Async-aware because the variable - /// has to stay set across the subprocess `await` that reads it. - static ENV_MUTEX: Mutex<()> = Mutex::const_new(()); - - fn principal(byte: u8) -> Principal { - Principal::from_slice(&[byte; 4]) - } - - fn params(canister_ids: &[(&str, Principal)]) -> Params { - Params { - path: "/work/backend".into(), - cid: principal(1), - name: "backend".to_owned(), - environment: "production".to_owned(), - network: "ic".to_owned(), - canister_ids: canister_ids - .iter() - .map(|(n, p)| ((*n).to_owned(), *p)) - .collect::>(), - proxy: None, - } - } - - /// The environment, network and target canister id are always present. - #[test] - fn base_env_vars_are_always_set() { - let p = params(&[]); - assert_eq!( - system_env_vars(&p), - vec![ - ("ICP_CLI_ENVIRONMENT".to_owned(), "production".to_owned()), - ("ICP_CLI_NETWORK".to_owned(), "ic".to_owned()), - ("ICP_CLI_CID".to_owned(), principal(1).to_text()), - ] - ); - } - - /// Each known canister gets an `ICP_CLI_CID_` variable, with the name - /// uppercased and every non-alphanumeric byte replaced by `_` so the result - /// is a legal shell identifier. - #[test] - fn canister_names_are_normalized_into_env_var_keys() { - let p = params(&[("my-frontend", principal(2)), ("dep:api", principal(3))]); - let keys: Vec = system_env_vars(&p) - .into_iter() - .map(|(k, _)| k) - .filter(|k| k.starts_with("ICP_CLI_CID_")) - .collect(); - // `canister_ids` is a BTreeMap, so ordering follows the canister names. - assert_eq!(keys, vec!["ICP_CLI_CID_DEP_API", "ICP_CLI_CID_MY_FRONTEND"]); - } - - /// Resolution takes the commands and cwd from the step and its canister, and - /// runs nothing. - #[test] - fn invocation_resolves_commands_and_cwd() { - let adapter = Adapter { - command: CommandField::Commands(vec!["first".to_owned(), "second".to_owned()]), - }; - let invocation = ScriptInvocation::new(&adapter, ¶ms(&[])); - - assert_eq!(invocation.commands, vec!["first", "second"]); - assert_eq!(invocation.cwd, PathBuf::from("/work/backend")); - assert_eq!(invocation.env, system_env_vars(¶ms(&[]))); - } - - /// The host runner passes the resolved environment through to the subprocess. - #[tokio::test] - async fn host_runner_applies_the_resolved_environment() { - let out = camino_tempfile::NamedUtf8TempFile::new().unwrap(); - let invocation = ScriptInvocation { - commands: vec![format!("printenv ICP_CLI_NETWORK > '{}'", out.path())], - cwd: "/".into(), - env: system_env_vars(¶ms(&[])), - }; - - HostScripts.run_script(invocation, None).await.unwrap(); - - assert_eq!(std::fs::read_to_string(out.path()).unwrap(), "ic\n"); - } - - /// `env` is an overlay, not the whole environment: the script still sees - /// variables the parent process had. Pins the contract documented on - /// [`ScriptInvocation::env`]. - /// - /// Deliberately avoids two things that are not portable across the shells - /// this runs under. `printenv` accepts only one operand on BSD (macOS) and - /// so silently drops later names, hence one `echo` — a shell builtin - /// everywhere — per variable. And the inherited variable is one this test - /// sets rather than `PATH`, because Git-for-Windows bash rewrites `PATH` - /// into POSIX form, so its value there never equals the `PATH` the Rust side - /// reads. - #[tokio::test] - async fn host_runner_overlays_rather_than_replaces_the_environment() { - const AMBIENT: &str = "ICP_CLI_TEST_AMBIENT_VAR"; - const AMBIENT_VALUE: &str = "inherited-from-parent"; - - let _guard = ENV_MUTEX.lock().await; - // SAFETY: ENV_MUTEX serializes the tests in this module that mutate the - // process environment, and the name is used by this test alone. - unsafe { std::env::set_var(AMBIENT, AMBIENT_VALUE) }; - - let overlaid = camino_tempfile::NamedUtf8TempFile::new().unwrap(); - let inherited = camino_tempfile::NamedUtf8TempFile::new().unwrap(); - let invocation = ScriptInvocation { - commands: vec![ - format!("echo \"$ICP_CLI_NETWORK\" > '{}'", overlaid.path()), - format!("echo \"${AMBIENT}\" > '{}'", inherited.path()), - ], - cwd: "/".into(), - env: system_env_vars(¶ms(&[])), - }; - - let run = HostScripts.run_script(invocation, None).await; - - // SAFETY: as above; the guard is still held. - unsafe { std::env::remove_var(AMBIENT) }; - run.expect("script must run"); - - assert_eq!( - std::fs::read_to_string(overlaid.path()).unwrap().trim(), - "ic", - "the overlaid variable must be set" - ); - assert_eq!( - std::fs::read_to_string(inherited.path()).unwrap().trim(), - AMBIENT_VALUE, - "a variable the parent process had must still reach the script" - ); - } - - /// A command that exits non-zero surfaces as a `ScriptRunError` whose source - /// still names the command and its status, so the `caused by:` line the CLI - /// prints stays specific. - #[tokio::test] - async fn host_runner_reports_a_failing_command() { - let invocation = ScriptInvocation { - commands: vec!["exit 3".to_owned()], - cwd: "/".into(), - env: vec![], - }; - - let err = HostScripts - .run_script(invocation, None) - .await - .expect_err("a non-zero exit must fail the step"); - - assert_eq!(err.to_string(), "script sync step failed"); - assert_eq!( - std::error::Error::source(&err).expect("cause").to_string(), - "command 'exit 3' failed with status code 3" - ); - } -} diff --git a/crates/icp/src/canister/wasm.rs b/crates/icp/src/canister/wasm.rs index 2cf2b219d..3b2ead5e5 100644 --- a/crates/icp/src/canister/wasm.rs +++ b/crates/icp/src/canister/wasm.rs @@ -1,8 +1,8 @@ use camino::{Utf8Path, Utf8PathBuf}; +use icp_deploy_canister::sync_exec::StepProgress; use reqwest::{Client, Method, Request}; use sha2::{Digest, Sha256}; use snafu::prelude::*; -use tokio::sync::mpsc::Sender; use url::Url; use crate::{ @@ -50,21 +50,21 @@ pub async fn resolve( source: &SourceField, base_dir: &Utf8Path, sha256: Option<&str>, - stdio: Option<&Sender>, + progress: Option<&dyn StepProgress>, pkg_cache: &PackageCache, ) -> Result { match source { SourceField::Local(s) => { let path = base_dir.join(&s.path); if let Some(expected) = sha256 { - if let Some(tx) = stdio { - let _ = tx.send(format!("Reading wasm: {}", s.path)).await; + if let Some(p) = progress { + p.line(format!("Reading wasm: {}", s.path)); } let bytes = read(&path).context(ReadLocalSnafu { path: s.path.clone(), })?; - if let Some(tx) = stdio { - let _ = tx.send("Verifying checksum".to_string()).await; + if let Some(p) = progress { + p.line("Verifying checksum".to_string()); } let actual = hex::encode(Sha256::digest(&bytes)); ensure!( @@ -94,16 +94,16 @@ pub async fn resolve( .await .context(LockCacheSnafu)?; if let Some(path) = cached { - if let Some(tx) = stdio { - let _ = tx.send("Using cached file".to_string()).await; + if let Some(p) = progress { + p.line("Using cached file".to_string()); } return Ok(path); } } let url = Url::parse(&s.url).context(ParseUrlSnafu)?; - if let Some(tx) = stdio { - let _ = tx.send(format!("Fetching wasm: {url}")).await; + if let Some(p) = progress { + p.line(format!("Fetching wasm: {url}")); } let resp = Client::new() .execute(Request::new(Method::GET, url)) @@ -118,8 +118,8 @@ pub async fn resolve( // Use provided sha256 as cache key (after verifying), or compute from bytes. let cache_sha = match sha256 { Some(expected) => { - if let Some(tx) = stdio { - let _ = tx.send("Verifying checksum".to_string()).await; + if let Some(p) = progress { + p.line("Verifying checksum".to_string()); } let actual = hex::encode(Sha256::digest(&bytes)); ensure!( diff --git a/crates/icp/src/context/init.rs b/crates/icp/src/context/init.rs index 4af5cda79..1db542d89 100644 --- a/crates/icp/src/context/init.rs +++ b/crates/icp/src/context/init.rs @@ -3,7 +3,7 @@ use std::{env::current_dir, sync::Arc}; use snafu::prelude::*; use crate::canister::build::Builder; -use crate::canister::recipe::fetch::RecipeFetcher; +use crate::canister::recipe::resolver::ResourceResolver; use crate::canister::sync::Syncer; use crate::context::Context; use crate::directories::{Access as _, Directories}; @@ -90,7 +90,7 @@ pub fn initialize( let pkg_cache = dirs.package_cache().context(PackageCacheSnafu)?; // Recipes - let recipe = Arc::new(RecipeFetcher { + let recipe = Arc::new(ResourceResolver { http_client, pkg_cache, }); @@ -99,7 +99,7 @@ pub fn initialize( let builder = Arc::new(Builder); // Canister syncer - let syncer = Arc::new(Syncer::host()); + let syncer = Arc::new(Syncer); // Project loader let pload = ProjectLoadImpl { diff --git a/crates/icp/src/context/mod.rs b/crates/icp/src/context/mod.rs index 3ae5ea72a..8befa0ff5 100644 --- a/crates/icp/src/context/mod.rs +++ b/crates/icp/src/context/mod.rs @@ -111,6 +111,20 @@ pub struct Context { } impl Context { + /// Build a resolver for the project's remote resources — recipe templates + /// and plugin wasms — backed by the package cache and an HTTP client. + pub fn resource_resolver( + &self, + ) -> Result, crate::fs::lock::LockError> + { + Ok(Arc::new( + crate::canister::recipe::resolver::ResourceResolver { + http_client: reqwest::Client::new(), + pkg_cache: self.dirs.package_cache()?, + }, + )) + } + /// Gets an identity based on the provided identity selection. // TODO: refactor the whole codebase to use this method instead of directly accessing `ctx.identity.load()` pub async fn get_identity( diff --git a/crates/icp/src/host_files.rs b/crates/icp/src/host_files.rs new file mode 100644 index 000000000..9129f3e45 --- /dev/null +++ b/crates/icp/src/host_files.rs @@ -0,0 +1,64 @@ +//! Host filesystem implementation of [`icp_deploy_canister::FileAccess`]. +//! +//! Backs project loading/consolidation on the real filesystem. Stateless: +//! operates on the (absolute) paths the model passes in. + +use async_trait::async_trait; +use icp_deploy_canister::files::{FileAccess, FileAccessError}; + +use crate::prelude::*; + +#[derive(Debug, Default, Clone, Copy)] +pub struct HostFileAccess; + +#[async_trait] +impl FileAccess for HostFileAccess { + async fn read_file(&self, path: &Path) -> Result, FileAccessError> { + crate::fs::read(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn read_to_string(&self, path: &Path) -> Result { + crate::fs::read_to_string(path).map_err(|e| FileAccessError::Read { + path: path.to_owned(), + message: e.to_string(), + }) + } + + async fn exists(&self, path: &Path) -> bool { + path.exists() + } + + async fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + async fn is_dir(&self, path: &Path) -> bool { + path.is_dir() + } + + async fn read_dir(&self, path: &Path) -> Result, FileAccessError> { + let rd = std::fs::read_dir(path).map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + let mut out = Vec::new(); + for entry in rd { + let entry = entry.map_err(|e| FileAccessError::ReadDir { + path: path.to_owned(), + message: e.to_string(), + })?; + if let Ok(p) = PathBuf::from_path_buf(entry.path()) { + out.push(p); + } + } + Ok(out) + } + + async fn canonicalize(&self, path: &Path) -> Option { + let canon = dunce::canonicalize(path.as_std_path()).ok()?; + PathBuf::try_from(canon).ok() + } +} diff --git a/crates/icp/src/lib.rs b/crates/icp/src/lib.rs index c5433ce82..4e1da41bb 100644 --- a/crates/icp/src/lib.rs +++ b/crates/icp/src/lib.rs @@ -1,34 +1,35 @@ -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, -}; +use std::sync::Arc; use async_trait::async_trait; -use indexmap::IndexMap; -use serde::Serialize; use snafu::prelude::*; use tokio::sync::Mutex; use tracing::debug; -use candid_parser::parse_idl_args; +pub use icp_deploy_canister::{ + Canister, Environment, InitArgs, InitArgsToBytesError, Network, Project, +}; use crate::{ - canister::{Settings, recipe::Resolve}, + canister::recipe::RemoteResourceResolve, manifest::{ - ArgsFormat, LoadManifestFromPathError, PROJECT_MANIFEST, ProjectRootLocate, - ProjectRootLocateError, - canister::{BuildSteps, SyncSteps}, + LoadManifestFromPathError, PROJECT_MANIFEST, ProjectRootLocate, ProjectRootLocateError, load_manifest_from_path, }, - network::Configuration, prelude::*, }; +// Imports used only by the in-crate test mock builders below. +#[cfg(test)] +use std::collections::{BTreeMap, HashMap}; +#[cfg(test)] +use {crate::canister::Settings, indexmap::IndexMap}; + pub mod agent; pub mod canister; pub mod context; pub mod directories; pub mod fs; +pub mod host_files; pub mod identity; pub mod manifest; pub mod network; @@ -47,157 +48,6 @@ const ICP_BASE: &str = ".icp"; const CACHE_DIR: &str = "cache"; const DATA_DIR: &str = "data"; -/// Resolved initialization arguments, with any file references already loaded. -#[derive(Clone, Debug, PartialEq, Serialize)] -pub enum InitArgs { - /// Text content (inline or loaded from file). Format is always known. - Text { content: String, format: ArgsFormat }, - /// Raw binary bytes (from a file with `format: bin`). Used directly. - Binary(Vec), -} - -#[derive(Debug, Snafu)] -pub enum InitArgsToBytesError { - #[snafu(display("failed to decode hex init args"))] - HexDecode { source: hex::FromHexError }, - - #[snafu(display("failed to parse Candid init args"))] - CandidParse { source: candid_parser::Error }, - - #[snafu(display("failed to encode Candid init args to bytes"))] - CandidEncode { source: candid::Error }, -} - -impl InitArgs { - /// Resolve to raw bytes according to the format. - pub fn to_bytes(&self) -> Result, InitArgsToBytesError> { - match self { - InitArgs::Binary(bytes) => Ok(bytes.clone()), - InitArgs::Text { content, format } => match format { - ArgsFormat::Hex => hex::decode(content.trim()).context(HexDecodeSnafu), - ArgsFormat::Candid => { - let args = parse_idl_args(content.trim()).context(CandidParseSnafu)?; - args.to_bytes().context(CandidEncodeSnafu) - } - ArgsFormat::Bin => { - unreachable!("binary format cannot appear in InitArgs::Text") - } - }, - } - } -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Canister { - pub name: String, - - /// Canister settings, such as memory constaints, etc. - pub settings: Settings, - - /// The build configuration specifying how to compile the canister's source - /// code into a WebAssembly module, including the adapter to use. - pub build: BuildSteps, - - /// The configuration specifying how to sync the canister - pub sync: SyncSteps, - - /// Initialization arguments passed to the canister during installation. - /// Resolved from the manifest — file contents are already loaded. - pub init_args: Option, - - /// If the canister was defined via a recipe reference, this holds the - /// original recipe specifier string (e.g. `@dfinity/motoko@v4.0.0`). - /// `None` when the canister uses explicit build/sync instructions. - pub registry_recipe: Option, - - /// Canister-discovery wiring. Maps the name this canister reads in a - /// `PUBLIC_CANISTER_ID:` environment variable to the store key of the - /// referenced canister. Computed during consolidation so each canister sees - /// the view its owning project expects: its own project's canisters under - /// their local names, plus any declared dependencies under their aliases - /// (`:`). For a project with no dependencies this maps every - /// canister's local name to itself, reproducing the flat "every canister sees - /// every sibling" behavior. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub bindings: BTreeMap, - - /// Subdomain prefixes for the canister's friendly URLs, most-specific label - /// first, e.g. `["backend"]` for an own canister or `["backend.openemail"]` - /// for a dependency canister (dot-nested by alias chain). A de-duplicated - /// shared dependency canister carries one entry per alias chain that reaches - /// it. Consumed only at deploy time to build `custom-domains.txt` entries and - /// the printed URLs; a runtime display aid that is always recomputed during - /// consolidation, so it is never serialized. - #[serde(skip)] - pub friendly_names: Vec, - - /// For each environment variable whose value came from a file, the file it - /// was read from. `settings.environment_variables` already holds the - /// contents; the paths are kept so `icp project bundle` can hold a file - /// backing a variable to the same containment rule it applies to every other - /// file a manifest points at. Bookkeeping for that check rather than part of - /// the resolved configuration, so it is never serialized. - #[serde(skip)] - pub environment_variable_files: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Network { - pub name: String, - pub configuration: Configuration, -} - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Environment { - pub name: String, - pub network: Network, - pub canisters: IndexMap, -} - -impl Environment { - pub fn get_canister_names(&self) -> Vec { - self.canisters.keys().cloned().collect() - } - - pub fn contains_canister(&self, canister_name: &str) -> bool { - self.canisters.contains_key(canister_name) - } - - pub fn get_canister_info(&self, canister: &str) -> Result<(PathBuf, Canister), String> { - self.canisters - .get(canister) - .ok_or_else(|| { - format!( - "canister '{}' not declared in environment '{}'", - canister, self.name - ) - }) - .cloned() - } -} - -/// Consolidated project definition -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct Project { - pub dir: PathBuf, - pub canisters: IndexMap, - pub networks: HashMap, - pub environments: HashMap, - - /// Environments the workspace defines that some vendored member does *not* - /// declare, keyed by environment name → the missing members' store-key - /// prefixes. Enforced when the environment is selected (strict rule). - /// Empty for standalone projects and workspaces whose members are complete. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub member_missing_envs: HashMap>, -} - -impl Project { - pub fn get_canister(&self, canister_name: &str) -> Option<&(PathBuf, Canister)> { - self.canisters.get(canister_name) - } -} - #[derive(Debug, Snafu)] pub enum ProjectLoadError { #[snafu(display("failed to locate project directory"))] @@ -228,7 +78,7 @@ pub trait ProjectLoad: Sync + Send { pub struct ProjectLoadImpl { pub project_root_locate: Arc, - pub recipe: Arc, + pub recipe: Arc, } /// Ensures the "operating on a workspace root above your sub-project" notice is @@ -278,10 +128,15 @@ impl ProjectLoad for ProjectLoadImpl { debug!("Loaded project manifest: {m:#?}"); - // Consolidate manifest into project - let p = project::consolidate_manifest(&pdir, self.recipe.as_ref(), &m) - .await - .context(ProjectSnafu)?; + // Consolidate manifest into project, reading files from the host filesystem. + let p = project::consolidate_manifest( + &crate::host_files::HostFileAccess, + &pdir, + self.recipe.as_ref(), + &m, + ) + .await + .context(ProjectSnafu)?; debug!("Rendered project definition: {p:#?}"); @@ -696,9 +551,12 @@ impl ProjectLoad for NoProjectLoader { #[cfg(test)] mod tests { use super::*; - use crate::canister::recipe::{Fetched, Resolve, ResolveError}; - use crate::manifest::{ProjectRootLocate, ProjectRootLocateError, recipe::Recipe}; + use crate::canister::recipe::{FetchedRecipe, RemoteResourceResolve, ResolveError}; + use crate::manifest::{ + ProjectRootLocate, ProjectRootLocateError, adapter::prebuilt::SourceField, recipe::Recipe, + }; use camino_tempfile::Utf8TempDir; + use icp_deploy_canister::sync_exec::StepProgress; use indoc::indoc; struct MockProjectRootLocate { @@ -724,11 +582,10 @@ mod tests { struct MockRecipeResolver; #[async_trait] - impl Resolve for MockRecipeResolver { - /// A minimal template rendering to a single dummy pre-built step. Nothing - /// is fetched, so there is no cache write to hold back. - async fn resolve(&self, _recipe: &Recipe) -> Result { - Ok(Fetched { + impl RemoteResourceResolve for MockRecipeResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + // A minimal recipe template rendering to a single prebuilt build step. + Ok(FetchedRecipe { template: indoc! {r#" build: steps: @@ -736,9 +593,27 @@ mod tests { path: dummy.wasm "#} .to_owned(), - pending_cache: None, + deferred: false, }) } + + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + Ok(()) + } + + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _progress: Option<&dyn StepProgress>, + ) -> Result { + unimplemented!("MockRecipeResolver::resolve_wasm") + } } #[tokio::test] diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index 0a811e358..c1019bf76 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -1,113 +1,20 @@ +//! Host-side manifest facade. +//! +//! The manifest *model* (types, `Item`, constants) lives in +//! `icp_deploy_canister::manifest` and is re-exported here. This module keeps +//! the pieces that walk the real filesystem: locating the project/workspace root +//! and the `std::fs`-based manifest loader. + use std::collections::HashSet; -use std::marker::PhantomData; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use snafu::prelude::*; +pub use icp_deploy_canister::manifest::*; + use crate::fs; use crate::prelude::*; -pub(crate) mod adapter; -pub(crate) mod canister; -pub(crate) mod dependency; -pub(crate) mod environment; -pub(crate) mod network; -pub(crate) mod project; -pub(crate) mod recipe; -pub(crate) mod serde_helpers; - -pub use { - adapter::plugin, - adapter::prebuilt, - canister::{ - ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, - SyncStep, SyncSteps, - }, - dependency::DependencyManifest, - environment::EnvironmentManifest, - network::{ManagedMode, Mode, NetworkManifest}, - project::ProjectManifest, -}; - -pub const PROJECT_MANIFEST: &str = "icp.yaml"; -pub const CANISTER_MANIFEST: &str = "canister.yaml"; - -// A manifest item that can either be a path to another manifest file or the manifest itself. -// -// The valid path specifications are: -// - CanisterManifest: path or glob pattern to the directory containing "canister.yaml" -// - NetworkManifest: path to network manifest -// - EnvironmentManifest: path to environment manifest -#[derive(Clone, Debug, PartialEq, JsonSchema)] -#[serde(untagged)] -pub enum Item { - /// Path to a manifest - Path(String), - - /// The manifest - Manifest(T), -} - -/// Items in path form serialize back to a bare path string, *not* to the contents of the -/// referenced file. Callers that need a self-contained YAML output (e.g. `icp project bundle`) -/// must convert any `Item::Path` to `Item::Manifest` themselves by loading the referenced -/// manifest first. -impl Serialize for Item { - fn serialize(&self, serializer: S) -> Result { - match self { - Item::Path(p) => p.serialize(serializer), - Item::Manifest(m) => m.serialize(serializer), - } - } -} - -impl<'de, T> Deserialize<'de> for Item -where - T: Deserialize<'de>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::{MapAccess, Visitor, value::MapAccessDeserializer}; - use std::fmt; - - struct ItemVisitor(PhantomData); - - impl<'de, T: Deserialize<'de>> Visitor<'de> for ItemVisitor { - type Value = Item; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a string path or a manifest object") - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Ok(Item::Path(v.to_owned())) - } - - fn visit_string(self, v: String) -> Result - where - E: serde::de::Error, - { - Ok(Item::Path(v)) - } - - fn visit_map(self, map: M) -> Result - where - M: MapAccess<'de>, - { - T::deserialize(MapAccessDeserializer::new(map)).map(Item::Manifest) - } - } - - deserializer.deserialize_any(ItemVisitor(PhantomData)) - } -} - #[derive(Debug, Snafu)] pub enum ProjectRootLocateError { #[snafu(display("project manifest not found in {path}"))] @@ -290,7 +197,7 @@ pub enum LoadManifestFromPathError { }, } -/// Loads a manifest of type `T` from the specified file path. +/// Loads a manifest of type `T` from the specified file path (host filesystem). pub async fn load_manifest_from_path(path: &Path) -> Result where T: for<'de> Deserialize<'de>, diff --git a/crates/icp/src/network/mod.rs b/crates/icp/src/network/mod.rs index 3d025fb40..e075c49e3 100644 --- a/crates/icp/src/network/mod.rs +++ b/crates/icp/src/network/mod.rs @@ -1,29 +1,28 @@ +//! Host-side network facade. +//! +//! The network *configuration* model lives in `icp_deploy_canister::network` +//! and is re-exported here. Runtime access (launching/stopping managed +//! networks, descriptors, agent bootstrap) stays in this crate. + use std::sync::Arc; use async_trait::async_trait; -use schemars::JsonSchema; -use serde::{Deserialize, Deserializer, Serialize}; use snafu::prelude::*; -pub use crate::manifest::network::RootKeySpec; +pub use icp_deploy_canister::network::*; + pub use access::RootKeySource; pub use directory::{LoadPidError, NetworkDirectory, SavePidError}; pub use managed::run::{RunNetworkError, run_network}; -use strum::EnumString; -use url::Url; use crate::{ CACHE_DIR, ICP_BASE, Network, - manifest::{ - ProjectRootLocate, ProjectRootLocateError, - network::{Connected as ManifestConnected, Endpoints, Gateway as ManifestGateway, Mode}, - }, + manifest::{ProjectRootLocate, ProjectRootLocateError}, network::access::{ GetNetworkAccessError, NetworkAccess, get_connected_network_access, get_managed_network_access, }, prelude::*, - project::DEFAULT_LOCAL_NETWORK_PORT, }; pub mod access; @@ -32,309 +31,6 @@ pub mod custom_domains; pub mod directory; pub mod managed; -#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] -pub enum Port { - Fixed(u16), - Random, -} - -impl Default for Port { - fn default() -> Self { - Port::Fixed(8000) - } -} - -impl<'de> Deserialize<'de> for Port { - fn deserialize>(d: D) -> Result { - Ok(match u16::deserialize(d)? { - 0 => Port::Random, - p => Port::Fixed(p), - }) - } -} - -fn default_bind() -> String { - "127.0.0.1".to_string() -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Gateway { - #[serde(default = "default_bind")] - pub bind: String, - - #[serde(default)] - pub port: Port, - - #[serde(default)] - pub domains: Vec, -} - -impl Default for Gateway { - fn default() -> Self { - Self { - bind: default_bind(), - port: Default::default(), - domains: Default::default(), - } - } -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct Managed { - #[serde(flatten)] - pub mode: ManagedMode, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -#[serde(untagged)] -pub enum ManagedMode { - Image(Box), - Launcher(Box), -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct ManagedLauncherConfig { - pub gateway: Gateway, - pub artificial_delay_ms: Option, - pub ii: bool, - pub nns: bool, - pub subnets: Option>, - pub bitcoind_addr: Option>, - pub dogecoind_addr: Option>, - pub version: Option, -} - -#[derive( - Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize, EnumString, strum::Display, -)] -#[serde(rename_all = "kebab-case")] -#[strum(serialize_all = "kebab-case")] -pub enum SubnetKind { - Application, - System, - VerifiedApplication, - Bitcoin, - Fiduciary, - Nns, - Sns, -} - -impl Default for ManagedMode { - fn default() -> Self { - Self::default_for_port(DEFAULT_LOCAL_NETWORK_PORT) - } -} - -impl ManagedMode { - pub fn default_for_port(port: u16) -> Self { - ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway: Gateway { - bind: default_bind(), - port: if port == 0 { - Port::Random - } else { - Port::Fixed(port) - }, - domains: vec![], - }, - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: None, - dogecoind_addr: None, - version: None, - })) - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -pub struct ManagedImageConfig { - pub image: String, - pub port_mapping: Vec, - pub rm_on_exit: bool, - pub args: Vec, - pub entrypoint: Option>, - pub environment: Vec, - pub volumes: Vec, - pub platform: Option, - pub user: Option, - pub shm_size: Option, - pub status_dir: String, - pub mounts: Vec, - pub extra_hosts: Vec, -} - -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub struct Connected { - /// The URL this network's API can be reached at. - pub api_url: Url, - - /// The URL this network's HTTP gateway can be reached at. - pub http_gateway_url: Option, - - /// How to obtain the root key used to verify responses from this network. - pub root_key: RootKeySpec, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] -#[serde(tag = "mode", rename_all = "lowercase")] -pub enum Configuration { - // Note: we must use struct variants to be able to flatten - // and make schemars generate the proper schema - /// A managed network is one which can be controlled and manipulated. - Managed { - #[serde(flatten)] - managed: Managed, - }, - - /// A connected network is one which can be interacted with - /// but cannot be controlled or manipulated. - Connected { - #[serde(flatten)] - connected: Connected, - }, -} - -impl Default for Configuration { - fn default() -> Self { - Configuration::Managed { - managed: Managed::default(), - } - } -} - -impl From for Gateway { - fn from(value: ManifestGateway) -> Self { - let ManifestGateway { - bind, - domains, - port, - } = value; - let bind = bind.unwrap_or("127.0.0.1".to_string()); - let port = match port { - Some(0) => Port::Random, - Some(p) => Port::Fixed(p), - None => Port::default(), - }; - let mut domains = domains.unwrap_or_default(); - if bind == "127.0.0.1" || bind == "0.0.0.0" || bind == "::1" || bind == "::" { - domains.insert(0, "localhost".to_string()); - } - Gateway { - bind, - port, - domains, - } - } -} - -impl From for Connected { - fn from(value: ManifestConnected) -> Self { - let root_key = value.root_key; - match value.endpoints { - Endpoints::Implicit { url } => Connected { - api_url: url.clone(), - http_gateway_url: Some(url), - root_key, - }, - Endpoints::Explicit { - api_url, - http_gateway_url, - } => Connected { - api_url, - http_gateway_url, - root_key, - }, - } - } -} - -impl From for Configuration { - fn from(value: Mode) -> Self { - match value { - Mode::Managed(managed) => match *managed.mode { - crate::manifest::network::ManagedMode::Launcher { - gateway, - artificial_delay_ms, - ii, - nns, - subnets, - bitcoind_addr, - dogecoind_addr, - version, - } => { - let gateway: Gateway = match gateway { - Some(g) => g.into(), - None => Gateway::default(), - }; - let version = match version { - Some(v) => { - if v.starts_with('v') { - Some(v) - } else { - Some(format!("v{v}")) - } - } - None => None, - }; - Configuration::Managed { - managed: Managed { - mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway, - artificial_delay_ms, - ii: ii.unwrap_or(false), - nns: nns.unwrap_or(false), - subnets, - bitcoind_addr, - dogecoind_addr, - version, - })), - }, - } - } - crate::manifest::network::ManagedMode::Image { - image, - port_mapping, - rm_on_exit, - args, - entrypoint, - environment, - volumes, - platform, - user, - shm_size, - status_dir, - mounts: mount, - extra_hosts, - } => Configuration::Managed { - managed: Managed { - mode: ManagedMode::Image(Box::new(ManagedImageConfig { - image, - port_mapping, - rm_on_exit: rm_on_exit.unwrap_or(false), - args: args.unwrap_or_default(), - entrypoint, - environment: environment.unwrap_or_default(), - volumes: volumes.unwrap_or_default(), - platform, - user, - shm_size, - status_dir: status_dir.unwrap_or_else(|| "/app/status".to_string()), - mounts: mount.unwrap_or_default(), - extra_hosts: extra_hosts.unwrap_or_default(), - })), - }, - }, - }, - Mode::Connected(connected) => Configuration::Connected { - connected: connected.into(), - }, - } - } -} - #[derive(Debug, Snafu)] pub enum AccessError { #[snafu(display("failed to find project root"))] @@ -444,51 +140,3 @@ impl Access for MockNetworkAccessor { }) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::manifest::network::{ - Gateway as ManifestGateway, Managed as ManifestManaged, ManagedMode as ManifestManagedMode, - Mode, - }; - - #[test] - fn from_mode_launcher_with_bitcoind_addr() { - let mode = Mode::Managed(ManifestManaged { - mode: Box::new(ManifestManagedMode::Launcher { - gateway: Some(ManifestGateway { - bind: None, - port: Some(8000), - domains: None, - }), - artificial_delay_ms: None, - ii: None, - nns: None, - subnets: None, - bitcoind_addr: Some(vec!["127.0.0.1:18444".to_string()]), - dogecoind_addr: None, - version: None, - }), - }); - - let config: Configuration = mode.into(); - match config { - Configuration::Managed { - managed: - Managed { - mode: ManagedMode::Launcher(launcher_config), - }, - } => { - assert_eq!( - launcher_config.bitcoind_addr, - Some(vec!["127.0.0.1:18444".to_string()]) - ); - assert_eq!(launcher_config.dogecoind_addr, None); - assert!(!launcher_config.ii); - assert!(!launcher_config.nns); - } - _ => panic!("expected ManagedMode::Launcher"), - } - } -} diff --git a/crates/icp/src/parsers.rs b/crates/icp/src/parsers.rs index b0a74730f..44cdff84f 100644 --- a/crates/icp/src/parsers.rs +++ b/crates/icp/src/parsers.rs @@ -1,643 +1,5 @@ -//! Parsing of token, cycle, memory, and duration amounts with support for suffixes and underscores. +//! Parsing of token, cycle, memory, and duration amounts. +//! +//! Defined in `icp_deploy_canister::parsers` and re-exported here. -use bigdecimal::{BigDecimal, Signed}; -use num_bigint::BigUint; -use num_integer::Integer; -use num_traits::{ToPrimitive, Zero}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::fmt; -use std::str::FromStr; - -/// Parse a token amount with support for suffixes (k, m, b, t) and underscores. -/// -/// Examples: -/// - `1` -> 1 -/// - `1_000` -> 1000 -/// - `1k` or `1K` -> 1000 -/// - `1t` or `1T` -> 1000000000000 -/// - `0.5` -> 0.5 -/// - `0.5k` -> 500 -pub fn parse_token_amount(input: &str) -> Result { - let input = input.trim(); - - if input.is_empty() { - return Err("Token amount cannot be empty".to_string()); - } - - let (number_part, multiplier) = if let Some(last_char) = input.chars().last() { - match last_char { - 'k' | 'K' => (&input[..input.len() - 1], 1_000u128), - 'm' | 'M' => (&input[..input.len() - 1], 1_000_000u128), - 'b' | 'B' => (&input[..input.len() - 1], 1_000_000_000u128), - 't' | 'T' => (&input[..input.len() - 1], 1_000_000_000_000u128), - _ => (input, 1u128), - } - } else { - (input, 1u128) - }; - - let cleaned = number_part.replace('_', ""); - let base = - BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid token amount: '{}'", input))?; - - if base.is_negative() { - return Err(format!("Token amount cannot be negative: '{}'", input)); - } - - let multiplier_decimal = BigDecimal::from(multiplier); - Ok(base * multiplier_decimal) -} - -/// Convert a token amount to the smallest unit by multiplying by 10^token_decimals. -/// E.g. 1.5 with 8 decimals -> 150000000. Fails if the result would be fractional. -pub fn to_token_unit_amount( - token_amount: BigDecimal, - token_decimals: u8, -) -> Result { - use num_bigint::BigInt; - use num_traits::pow::Pow; - - let (mantissa, exponent) = token_amount.into_bigint_and_exponent(); - let scale_adjustment = token_decimals as i64 - exponent; - let ten = BigInt::from(10); - - let result = if scale_adjustment >= 0 { - let multiplier = ten.pow(scale_adjustment as u32); - mantissa * multiplier - } else { - let divisor = ten.pow((-scale_adjustment) as u32); - let (quotient, remainder) = mantissa.div_rem(&divisor); - if !remainder.is_zero() { - return Err(format!( - "Token amount cannot be represented with {} decimals (would result in fractional units)", - token_decimals - )); - } - quotient - }; - - result - .try_into() - .map_err(|_| "Token amount cannot be negative".to_string()) -} - -fn parse_cycles_str(s: &str) -> Result { - let token_amount = parse_token_amount(s)?; - let unit_amount = to_token_unit_amount(token_amount, 0)?; - unit_amount - .to_u128() - .ok_or_else(|| format!("Cycles amount too large: '{}'", s)) -} - -/// An amount of cycles. -/// -/// Deserializes from a number or a string with suffixes (k, m, b, t) and optional underscore separators. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum CyclesAmount { - Number(u64), // yaml only supports up to u64 - Str(String), -} - -impl CyclesAmount { - pub fn get(&self) -> u128 { - match self { - CyclesAmount::Number(n) => *n as u128, - CyclesAmount::Str(s) => parse_cycles_str(s) - .unwrap_or_else(|e| panic!("invalid cycles amount '{}': {}", s, e)), - } - } -} - -impl<'de> Deserialize<'de> for CyclesAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - // Identical enum to CyclesAmount. Needed to avoid a circular dependency. - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom("cycles amount must be a number or a string with optional suffix (k, m, b, t), e.g. 1000 or \"4t\"") - })?; - let c = match v { - Raw::Number(n) => CyclesAmount::Number(n), - Raw::Str(ref s) => { - parse_cycles_str(s).map_err(serde::de::Error::custom)?; // validate the string is a valid cycles amount - CyclesAmount::Str(s.clone()) - } - }; - Ok(c) - } -} - -impl Serialize for CyclesAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - CyclesAmount::Number(n) => serializer.serialize_u64(*n), - CyclesAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for CyclesAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_cycles_str(s)?; // validate the string is a valid cycles amount - Ok(CyclesAmount::Str(s.to_string())) - } -} - -impl fmt::Display for CyclesAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u128 { - fn from(c: CyclesAmount) -> Self { - c.get() - } -} - -impl From for CyclesAmount { - fn from(n: u128) -> Self { - if let Ok(n64) = u64::try_from(n) { - CyclesAmount::Number(n64) - } else { - CyclesAmount::Str(n.to_string()) - } - } -} - -const KB: u64 = 1000; -const KIB: u64 = 1024; -const MB: u64 = 1_000_000; -const MIB: u64 = 1024 * 1024; -const GB: u64 = 1_000_000_000; -const GIB: u64 = 1024 * 1024 * 1024; - -fn parse_memory_str(s: &str) -> Result { - let s = s.trim(); - if s.is_empty() { - return Err("Memory amount cannot be empty".to_string()); - } - let lower = s.to_lowercase(); - let (number_part, factor) = if lower.ends_with("gib") { - (&s[..s.len() - 3], GIB) - } else if lower.ends_with("gb") { - (&s[..s.len() - 2], GB) - } else if lower.ends_with("mib") { - (&s[..s.len() - 3], MIB) - } else if lower.ends_with("mb") { - (&s[..s.len() - 2], MB) - } else if lower.ends_with("kib") { - (&s[..s.len() - 3], KIB) - } else if lower.ends_with("kb") { - (&s[..s.len() - 2], KB) - } else { - (s, 1u64) - }; - let cleaned = number_part.trim().replace('_', ""); - let amount = - BigDecimal::from_str(&cleaned).map_err(|_| format!("Invalid memory amount: '{}'", s))?; - if amount.is_negative() { - return Err(format!("Memory amount cannot be negative: '{}'", s)); - } - let product = amount * BigDecimal::from(factor); - if !product.is_integer() { - return Err( - "Memory amount must be a whole number of bytes (fractional bytes not allowed)" - .to_string(), - ); - } - product - .to_u64() - .ok_or_else(|| format!("Memory amount too large: '{}'", s)) -} - -/// An amount of memory in bytes. -/// -/// Deserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib), -/// optional decimals, and optional underscore separators. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum MemoryAmount { - Number(u64), - Str(String), -} - -impl MemoryAmount { - pub fn get(&self) -> u64 { - match self { - MemoryAmount::Number(n) => *n, - MemoryAmount::Str(s) => parse_memory_str(s) - .unwrap_or_else(|e| panic!("invalid memory amount '{}': {}", s, e)), - } - } -} - -impl<'de> Deserialize<'de> for MemoryAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom( - "memory amount must be a number or a string with optional suffix (kb, kib, mb, mib, gb, gib), e.g. 1024 or \"2.5gib\"", - ) - })?; - let m = match v { - Raw::Number(n) => MemoryAmount::Number(n), - Raw::Str(ref s) => { - parse_memory_str(s).map_err(serde::de::Error::custom)?; - MemoryAmount::Str(s.clone()) - } - }; - Ok(m) - } -} - -impl Serialize for MemoryAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - MemoryAmount::Number(n) => serializer.serialize_u64(*n), - MemoryAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for MemoryAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_memory_str(s)?; - Ok(MemoryAmount::Str(s.to_string())) - } -} - -impl fmt::Display for MemoryAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u64 { - fn from(m: MemoryAmount) -> Self { - m.get() - } -} - -impl From for MemoryAmount { - fn from(n: u64) -> Self { - MemoryAmount::Number(n) - } -} - -const SECONDS_PER_MINUTE: u64 = 60; -const SECONDS_PER_HOUR: u64 = 3600; -const SECONDS_PER_DAY: u64 = 86400; -const SECONDS_PER_WEEK: u64 = 604800; - -fn parse_duration_str(s: &str) -> Result { - let s = s.trim(); - if s.is_empty() { - return Err("Duration cannot be empty".to_string()); - } - let lower = s.to_lowercase(); - let (number_part, factor) = if lower.ends_with('w') { - (&s[..s.len() - 1], SECONDS_PER_WEEK) - } else if lower.ends_with('d') { - (&s[..s.len() - 1], SECONDS_PER_DAY) - } else if lower.ends_with('h') { - (&s[..s.len() - 1], SECONDS_PER_HOUR) - } else if lower.ends_with('m') { - (&s[..s.len() - 1], SECONDS_PER_MINUTE) - } else if lower.ends_with('s') { - (&s[..s.len() - 1], 1u64) - } else { - (s, 1u64) - }; - let cleaned = number_part.trim().replace('_', ""); - if cleaned.is_empty() { - return Err(format!("Invalid duration: '{s}'")); - } - let value: u64 = cleaned - .parse() - .map_err(|_| format!("Invalid duration: '{s}'"))?; - value - .checked_mul(factor) - .ok_or_else(|| format!("Duration too large: '{s}'")) -} - -/// A duration in seconds. -/// -/// Deserializes from a number (seconds) or a string with duration suffix (s, m, h, d, w) -/// and optional underscore separators. -/// -/// Suffixes (case-insensitive): -/// - `s` — seconds -/// - `m` — minutes (×60) -/// - `h` — hours (×3600) -/// - `d` — days (×86400) -/// - `w` — weeks (×604800) -/// -/// A bare number without suffix is treated as seconds. -#[derive(Clone, Debug, PartialEq, Eq, JsonSchema)] -#[schemars(untagged)] -pub enum DurationAmount { - Number(u64), - Str(String), -} - -impl DurationAmount { - pub fn get(&self) -> u64 { - match self { - DurationAmount::Number(n) => *n, - DurationAmount::Str(s) => { - parse_duration_str(s).unwrap_or_else(|e| panic!("invalid duration '{}': {}", s, e)) - } - } - } -} - -impl<'de> Deserialize<'de> for DurationAmount { - fn deserialize(d: D) -> Result - where - D: serde::Deserializer<'de>, - { - #[derive(Deserialize)] - #[serde(untagged)] - enum Raw { - Number(u64), - Str(String), - } - let v = Raw::deserialize(d).map_err(|_| { - serde::de::Error::custom( - "duration must be a number (seconds) or a string with optional suffix (s, m, h, d, w), e.g. 2592000 or \"30d\"", - ) - })?; - let c = match v { - Raw::Number(n) => DurationAmount::Number(n), - Raw::Str(ref s) => { - parse_duration_str(s).map_err(serde::de::Error::custom)?; - DurationAmount::Str(s.clone()) - } - }; - Ok(c) - } -} - -impl Serialize for DurationAmount { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - DurationAmount::Number(n) => serializer.serialize_u64(*n), - DurationAmount::Str(s) => serializer.serialize_str(s), - } - } -} - -impl FromStr for DurationAmount { - type Err = String; - - fn from_str(s: &str) -> Result { - parse_duration_str(s)?; - Ok(DurationAmount::Str(s.to_string())) - } -} - -impl fmt::Display for DurationAmount { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.get().fmt(f) - } -} - -impl From for u64 { - fn from(d: DurationAmount) -> Self { - d.get() - } -} - -impl From for DurationAmount { - fn from(n: u64) -> Self { - DurationAmount::Number(n) - } -} - -impl PartialEq for DurationAmount { - fn eq(&self, other: &u64) -> bool { - self.get() == *other - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cycles_amount_from_str_plain() { - assert_eq!("1".parse::().unwrap().get(), 1); - assert_eq!("1000".parse::().unwrap().get(), 1000); - } - - #[test] - fn cycles_amount_from_str_suffixes() { - assert_eq!("1k".parse::().unwrap().get(), 1000); - assert_eq!( - "1t".parse::().unwrap().get(), - 1_000_000_000_000 - ); - assert_eq!( - "4t".parse::().unwrap().get(), - 4_000_000_000_000 - ); - assert_eq!( - "0.5t".parse::().unwrap().get(), - 500_000_000_000 - ); - } - - #[test] - fn cycles_amount_from_str_underscores() { - assert_eq!("1_000".parse::().unwrap().get(), 1000); - } - - #[test] - fn cycles_amount_from_str_fractional_rejected() { - assert!("1.5".parse::().is_err()); - } - - #[test] - fn cycles_amount_deserialize() { - let yaml = "4t"; - let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(c.get(), 4_000_000_000_000); - - let yaml = "5000000000000"; - let c: CyclesAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(c.get(), 5_000_000_000_000); - } - - #[test] - fn parse_token_amount_plain_and_suffixes() { - use std::str::FromStr; - assert_eq!( - parse_token_amount("1").unwrap(), - BigDecimal::from_str("1").unwrap() - ); - assert_eq!( - parse_token_amount("1k").unwrap(), - BigDecimal::from_str("1000").unwrap() - ); - assert_eq!( - parse_token_amount("0.5t").unwrap(), - BigDecimal::from_str("500000000000").unwrap() - ); - } - - #[test] - fn memory_amount_from_str_plain() { - assert_eq!("1".parse::().unwrap().get(), 1); - assert_eq!("1024".parse::().unwrap().get(), 1024); - } - - #[test] - fn memory_amount_from_str_suffixes() { - assert_eq!("1kb".parse::().unwrap().get(), 1000); - assert_eq!("1kib".parse::().unwrap().get(), 1024); - assert_eq!("1mb".parse::().unwrap().get(), 1_000_000); - assert_eq!("1mib".parse::().unwrap().get(), 1024 * 1024); - assert_eq!("1gb".parse::().unwrap().get(), 1_000_000_000); - assert_eq!( - "1gib".parse::().unwrap().get(), - 1024 * 1024 * 1024 - ); - assert_eq!( - "2 GiB".parse::().unwrap().get(), - 2 * 1024 * 1024 * 1024 - ); - } - - #[test] - fn memory_amount_from_str_decimals() { - assert_eq!("0.5kib".parse::().unwrap().get(), 512); - assert_eq!("1.5gib".parse::().unwrap().get(), 1610612736); - } - - #[test] - fn memory_amount_fractional_bytes_rejected() { - assert!("1.5".parse::().is_err()); // 1.5 bytes - assert!("0.3kib".parse::().is_err()); // 307.2 bytes - } - - #[test] - fn memory_amount_from_str_underscores() { - assert_eq!("1_024".parse::().unwrap().get(), 1024); - } - - #[test] - fn memory_amount_deserialize() { - let yaml = "2gib"; - let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(m.get(), 2 * 1024 * 1024 * 1024); - - let yaml = "4294967296"; - let m: MemoryAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(m.get(), 4294967296); - } - - #[test] - fn duration_amount_from_str_plain() { - assert_eq!("60".parse::().unwrap().get(), 60); - assert_eq!("2592000".parse::().unwrap().get(), 2592000); - } - - #[test] - fn duration_amount_from_str_underscores() { - assert_eq!( - "2_592_000".parse::().unwrap().get(), - 2592000 - ); - } - - #[test] - fn duration_amount_from_str_suffixes() { - assert_eq!("60s".parse::().unwrap().get(), 60); - assert_eq!("90m".parse::().unwrap().get(), 5400); - assert_eq!("24h".parse::().unwrap().get(), 86400); - assert_eq!("30d".parse::().unwrap().get(), 2592000); - assert_eq!("4w".parse::().unwrap().get(), 2419200); - } - - #[test] - fn duration_amount_from_str_case_insensitive() { - assert_eq!("30D".parse::().unwrap().get(), 2592000); - assert_eq!("1W".parse::().unwrap().get(), 604800); - assert_eq!("24H".parse::().unwrap().get(), 86400); - assert_eq!("60S".parse::().unwrap().get(), 60); - assert_eq!("90M".parse::().unwrap().get(), 5400); - } - - #[test] - fn duration_amount_from_str_underscores_with_suffix() { - assert_eq!( - "2_592_000s".parse::().unwrap().get(), - 2592000 - ); - } - - #[test] - fn duration_amount_from_str_errors() { - assert!("abc".parse::().is_err()); - assert!("".parse::().is_err()); - assert!("1x".parse::().is_err()); - assert!("1.5d".parse::().is_err()); - assert!("-1d".parse::().is_err()); - } - - #[test] - fn duration_amount_deserialize() { - let yaml = "30d"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - - let yaml = "2592000"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - - let yaml = "2_592_000"; - let d: DurationAmount = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(d.get(), 2592000); - } - - #[test] - fn duration_amount_partial_eq_u64() { - let d = DurationAmount::Number(2592000); - assert!(d == 2592000); - assert!(d != 0); - - let d = DurationAmount::Str("30d".to_string()); - assert!(d == 2592000); - } -} +pub use icp_deploy_canister::parsers::*; diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index 1f5520def..4dda9dc90 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -1,520 +1,32 @@ -use std::collections::{BTreeMap, HashMap, HashSet, hash_map::Entry}; +//! Host-side project facade. +//! +//! Consolidation of manifests into a [`Project`] lives in +//! `icp_deploy_canister::project` (over an injected `FileAccess`) and is +//! re-exported here. Walking a workspace's dependency edges on disk and +//! member-scoping both resolve real filesystem paths against the current working +//! directory, so they stay here. -use indexmap::{IndexMap, map::Entry as IndexEntry}; +use std::collections::HashSet; use snafu::prelude::*; +pub use icp_deploy_canister::project::{ + ConsolidateManifestError, EnvironmentError, LoadProjectError, VerifySandboxError, + consolidate_manifest, load_project, relative_prefix, verify_sandbox, +}; + use crate::{ - Canister, Environment, InitArgs, Network, Project, - canister::{ControllerRef, ManifestEnvVar, ManifestSettings, Settings, recipe}, - fs, + Environment, manifest::{ - ArgsFormat, CANISTER_MANIFEST, CanisterManifest, DependencyManifest, EnvironmentManifest, - Item, LoadManifestFromPathError, ManifestInitArgs, NetworkManifest, PROJECT_MANIFEST, - ProjectManifest, ProjectRootLocateError, - canister::{Instructions, SyncSteps}, - environment::CanisterSelection, - load_manifest_from_path, - network::RootKeySpec, - recipe::RecipeType, - }, - network::{ - Configuration, Connected, Gateway, Managed, ManagedLauncherConfig, ManagedMode, Port, + LoadManifestFromPathError, PROJECT_MANIFEST, ProjectManifest, load_manifest_from_path, }, prelude::*, }; -pub const DEFAULT_LOCAL_NETWORK_BIND: &str = "127.0.0.1"; -pub const DEFAULT_LOCAL_NETWORK_PORT: u16 = 8000; - -#[derive(Debug, Snafu)] -pub enum EnvironmentError { - #[snafu(display("environment '{environment}' points to invalid network '{network}'"))] - InvalidNetwork { - environment: String, - network: String, - }, - - #[snafu(display("environment '{environment}' points to invalid canister '{canister}'"))] - InvalidCanister { - environment: String, - canister: String, - }, -} - -#[derive(Debug, Snafu)] -pub enum ConsolidateManifestError { - #[snafu(display("failed to locate project directory"))] - Locate { source: ProjectRootLocateError }, - - #[snafu(display("failed to perform glob parsing"))] - GlobParse { source: glob::PatternError }, - - #[snafu(display("failed to get glob iter"))] - GlobIter { source: glob::GlobError }, - - #[snafu(display("failed to convert path to UTF-8"))] - Utf8Path { source: FromPathBufError }, - - #[snafu(display("failed to load canister manifest"))] - LoadCanister { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load network manifest"))] - LoadNetwork { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load environment manifest"))] - LoadEnvironment { source: LoadManifestFromPathError }, - - #[snafu(display("failed to load {kind} manifest at: {path}"))] - Failed { kind: String, path: String }, - - #[snafu(display("failed to fetch canister recipe: {recipe_type:?}"))] - FetchRecipe { - #[snafu(source(from(recipe::ResolveError, Box::new)))] - source: Box, - recipe_type: RecipeType, - }, - - #[snafu(display("failed to render canister recipe: {recipe_type:?}"))] - RenderRecipe { - #[snafu(source(from(recipe::RenderRecipeError, Box::new)))] - source: Box, - recipe_type: RecipeType, - }, - - #[snafu(display("failed to cache canister recipe: {recipe_type:?}"))] - CacheRecipe { - #[snafu(source(from(recipe::ResolveError, Box::new)))] - source: Box, - recipe_type: RecipeType, - }, - - #[snafu(display("project contains two similarly named {kind}s: '{name}'"))] - Duplicate { kind: String, name: String }, - - #[snafu(display("`{name}` is a reserved {kind} name."))] - Reserved { kind: String, name: String }, - - #[snafu(display("could not locate a {kind} manifest at: '{path}'"))] - NotFound { kind: String, path: String }, - - #[snafu(display("failed to read init_args file for canister '{canister}'"))] - ReadInitArgs { - source: fs::IoError, - canister: String, - }, - - #[snafu(display( - "failed to read the file backing environment variable '{variable}' of canister '{canister}'" - ))] - ReadEnvironmentVariable { - source: fs::IoError, - canister: String, - variable: String, - }, - - #[snafu(display( - "init_args for canister '{canister}' uses format 'bin' with inline content; \ - binary format requires a file path" - ))] - BinFormatInlineContent { canister: String }, - - #[snafu(display( - "canister '{canister}' lists controller '{controller}', but no canister with that \ - name is declared in the project" - ))] - UnknownControllerCanister { - canister: String, - controller: String, - }, - - #[snafu(display( - "canister name '{name}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ - (':' is reserved as the dependency namespace separator)" - ))] - InvalidCanisterName { name: String }, - - #[snafu(display( - "dependency alias '{alias}' is invalid: only ASCII letters, digits, '_' and '-' are allowed \ - (':' is reserved as the dependency namespace separator)" - ))] - InvalidDependencyAlias { alias: String }, - - #[snafu(display("project declares two dependencies with the same alias '{alias}'"))] - DuplicateDependencyAlias { alias: String }, - - #[snafu(display( - "dependency alias '{alias}' collides with a canister of the same name in the same project" - ))] - DependencyAliasCollision { alias: String }, - - #[snafu(display("could not find a project manifest for dependency '{alias}' at: '{path}'"))] - DependencyNotFound { alias: String, path: String }, - - #[snafu(display("failed to canonicalize path for dependency '{alias}' at: '{path}'"))] - DependencyCanonicalize { alias: String, path: String }, - - #[snafu(display("failed to load project manifest for dependency '{alias}'"))] - LoadDependencyManifest { - source: LoadManifestFromPathError, - alias: String, - }, - - #[snafu(display( - "dependency '{alias}' selects canister '{canister}', which the dependency does not declare" - ))] - UnknownDependencyCanister { alias: String, canister: String }, - - #[snafu(display("dependency cycle detected: {chain}"))] - CircularDependency { chain: String }, - - #[snafu(transparent)] - Environment { source: EnvironmentError }, -} - -/// Resolve a [`ManifestInitArgs`] into a canonical [`InitArgs`] by reading -/// any file references relative to `base_path`. -fn resolve_manifest_init_args( - manifest_init_args: &ManifestInitArgs, - base_path: &Path, - canister: &str, -) -> Result { - match manifest_init_args { - ManifestInitArgs::String(content) => Ok(InitArgs::Text { - content: content.trim().to_owned(), - format: ArgsFormat::Candid, - }), - ManifestInitArgs::Path { path, format } => { - let file_path = base_path.join(path); - match format { - ArgsFormat::Bin => { - let bytes = fs::read(&file_path).context(ReadInitArgsSnafu { canister })?; - Ok(InitArgs::Binary(bytes)) - } - fmt => { - let content = - fs::read_to_string(&file_path).context(ReadInitArgsSnafu { canister })?; - Ok(InitArgs::Text { - content: content.trim().to_owned(), - format: fmt.clone(), - }) - } - } - } - ManifestInitArgs::Value { value, format } => match format { - ArgsFormat::Bin => BinFormatInlineContentSnafu { canister }.fail(), - fmt => Ok(InitArgs::Text { - content: value.trim().to_owned(), - format: fmt.clone(), - }), - }, - } -} - -/// Resolve a manifest's [`ManifestSettings`] into the model's [`Settings`] by -/// reading any file-backed environment variable values relative to `base_path`. -/// Also returns the file each such value came from, for -/// [`Canister::environment_variable_files`]. -fn resolve_manifest_settings( - manifest_settings: &ManifestSettings, - base_path: &Path, - canister: &str, -) -> Result<(Settings, BTreeMap), ConsolidateManifestError> { - let ManifestSettings { - log_visibility, - compute_allocation, - memory_allocation, - freezing_threshold, - reserved_cycles_limit, - wasm_memory_limit, - wasm_memory_threshold, - log_memory_limit, - environment_variables, - controllers, - } = manifest_settings; - - let mut files = BTreeMap::new(); - let environment_variables = environment_variables - .as_ref() - .map(|vars| { - vars.iter() - .map(|(name, var)| { - let value = match var { - ManifestEnvVar::Value(value) => value.to_owned(), - ManifestEnvVar::Path { path } => { - let file = base_path.join(path); - let contents = fs::read_to_string(&file).context( - ReadEnvironmentVariableSnafu { - canister, - variable: name, - }, - )?; - files.insert(name.to_owned(), file); - contents.trim().to_owned() - } - }; - Ok((name.to_owned(), value)) - }) - .collect::, ConsolidateManifestError>>() - }) - .transpose()?; - - let settings = Settings { - log_visibility: log_visibility.clone(), - compute_allocation: *compute_allocation, - memory_allocation: memory_allocation.clone(), - freezing_threshold: freezing_threshold.clone(), - reserved_cycles_limit: reserved_cycles_limit.clone(), - wasm_memory_limit: wasm_memory_limit.clone(), - wasm_memory_threshold: wasm_memory_threshold.clone(), - log_memory_limit: log_memory_limit.clone(), - environment_variables, - controllers: controllers.clone(), - }; - Ok((settings, files)) -} - -fn is_glob(s: &str) -> bool { - s.contains('*') || s.contains('?') || s.contains('[') || s.contains('{') -} - -/// Whether `name` is a valid canister name or dependency alias: non-empty and -/// containing only ASCII letters, digits, `_`, or `-`. -/// -/// A single strict rule keeps names safe for every purpose they are reused for — -/// store-key segments, `PUBLIC_CANISTER_ID:` env vars, DNS subdomains, and -/// archive paths — so no per-site sanitizing is needed. In particular `:` is the -/// dependency namespace separator, and `.` / `/` would be ambiguous in -/// subdomains and paths. -fn is_valid_name(name: &str) -> bool { - !name.is_empty() - && name - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') -} - -/// Builds the canonical canisters declared directly in one project manifest, -/// resolving glob/path/inline entries, recipes, and init-args relative to -/// `pdir`. Returns `(local name, canister dir, canister)` with empty bindings; -/// callers assign store keys and bindings. Does not check for duplicate names -/// across projects — that is the caller's responsibility (via the global map). -async fn build_manifest_canisters( - pdir: &Path, - manifest_canisters: &[Item], - recipe_resolver: &dyn recipe::Resolve, -) -> Result, ConsolidateManifestError> { - let mut result: Vec<(String, PathBuf, Canister)> = Vec::new(); - - for i in manifest_canisters { - let ms = match i { - Item::Path(pattern) => { - let is_glob_pattern = is_glob(pattern); - let paths = match is_glob_pattern { - // Explicit path - false => vec![pdir.join(pattern)], - - // Glob pattern - true => { - let paths = - glob::glob(pdir.join(pattern).as_str()).context(GlobParseSnafu)?; - - let mut v = vec![]; - for p in paths { - let path = p.context(GlobIterSnafu)?; - let utf8_path = PathBuf::try_from(path).context(Utf8PathSnafu)?; - v.push(utf8_path); - } - v - } - }; - - let paths = if is_glob_pattern { - // For glob patterns, filter out non-directories and non-canister directories - paths - .into_iter() - .filter(|p| p.is_dir()) - .filter(|p| p.join(CANISTER_MANIFEST).exists()) - .collect::>() - } else { - // For explicit paths, validate that they exist and contain canister.yaml - let mut validated_paths = vec![]; - for p in paths { - if !p.join(CANISTER_MANIFEST).is_file() { - return NotFoundSnafu { - kind: "canister".to_string(), - path: pattern.to_string(), - } - .fail(); - } - validated_paths.push(p); - } - validated_paths - }; - - let mut ms = vec![]; - for p in paths { - ms.push(( - p.to_owned(), - load_manifest_from_path::(&p.join(CANISTER_MANIFEST)) - .await - .context(LoadCanisterSnafu)?, - )); - } - ms - } - - Item::Manifest(m) => vec![(pdir.to_owned(), m.to_owned())], - }; - - for (cdir, m) in ms { - if !is_valid_name(&m.name) { - return InvalidCanisterNameSnafu { - name: m.name.clone(), - } - .fail(); - } - - let registry_recipe = match &m.instructions { - Instructions::BuildSync { .. } => None, - Instructions::Recipe { recipe } => match &recipe.recipe_type { - RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), - _ => None, - }, - }; - - let (build, sync) = match &m.instructions { - // Build/Sync - Instructions::BuildSync { build, sync } => ( - build.to_owned(), - match sync { - Some(sync) => sync.to_owned(), - None => SyncSteps::default(), - }, - ), - - // Recipe - Instructions::Recipe { recipe } => { - let fetched = - recipe_resolver - .resolve(recipe) - .await - .context(FetchRecipeSnafu { - recipe_type: recipe.recipe_type.clone(), - })?; - let ctx = recipe::RecipeContext { - canister_name: m.name.clone(), - }; - let steps = recipe::render_recipe(&fetched.template, recipe, &ctx).context( - RenderRecipeSnafu { - recipe_type: recipe.recipe_type.clone(), - }, - )?; - - // The template rendered, so an unpinned download is now known - // good and safe to cache. Committing only here is what keeps a - // bad remote response from becoming sticky. - if let Some(pending) = fetched.pending_cache { - recipe_resolver - .commit(pending) - .await - .context(CacheRecipeSnafu { - recipe_type: recipe.recipe_type.clone(), - })?; - } - - steps - } - }; - - let (settings, environment_variable_files) = - resolve_manifest_settings(&m.settings, &cdir, &m.name)?; - - let init_args = m - .init_args - .as_ref() - .map(|mia| resolve_manifest_init_args(mia, &cdir, &m.name)) - .transpose()?; - - result.push(( - m.name.clone(), - cdir, - Canister { - name: m.name.clone(), - settings, - build, - sync, - init_args, - registry_recipe, - bindings: BTreeMap::new(), - // Default to the bare local name; overwritten with the - // dot-nested alias form when the canister is imported as a - // dependency (see `import_dependency`). - friendly_names: vec![m.name.clone()], - environment_variable_files, - }, - )); - } - } - - Ok(result) -} - -/// A dependency instance imported into the workspace. Returned by -/// [`import_dependency`] and cached per canonical path so diamond dependencies -/// reuse the same instance. -#[derive(Clone)] -struct ImportedInstance { - /// This instance's own canisters, as `(local name, full store key)` — the - /// set exposable to the parent via `canisters:` selection. - own: Vec<(String, String)>, - /// Every canister in this instance's subtree (its own canisters plus all - /// transitively imported ones), as `(store key, local name, alias chain - /// from this instance down to the canister's owning project)`. Used to - /// register a friendly URL per alias chain when the instance is reached - /// again via de-duplication (a diamond), including for its descendants. - subtree: Vec<(String, String, Vec)>, -} - -/// A member environment's per-canister config, to be folded into the root's -/// same-named environment beneath any root overrides. -#[derive(Default, Clone)] -struct MemberCanisterOverride { - settings: Option, - init_args: Option, -} - -/// Per-environment member overrides: env name → store key → override. -type MemberEnvOverrides = HashMap>; - -/// A member's identity (store-key prefix) and the environment names it defines, -/// used to enforce that a member declares every environment the root targets -/// (strict rule). -struct MemberEnvInfo { - prefix: String, - defined: HashSet, -} - -/// Canonicalize a dependency root (resolving symlinks and `..`) for use as a -/// de-dup / cycle-detection identity. -fn canonicalize_dep(alias: &str, dep_root: &Path) -> Result { - let build_err = || { - DependencyCanonicalizeSnafu { - alias: alias.to_owned(), - path: dep_root.to_string(), - } - .build() - }; - let canon = dunce::canonicalize(dep_root.as_std_path()).map_err(|_| build_err())?; - PathBuf::try_from(canon).map_err(|_| build_err()) -} - -/// Store-key prefix for a dependency instance: its canonical directory relative -/// to the canonical app root, forward-slash separated so keys are stable across -/// platforms and independent of how each edge spells the path. -fn relative_prefix(app_root_canonical: &Path, dep_canonical: &Path) -> String { - let rel = pathdiff::diff_utf8_paths(dep_canonical, app_root_canonical) - .unwrap_or_else(|| dep_canonical.to_owned()); - rel.as_str().replace('\\', "/") +/// Canonicalize into a UTF-8 path, or `None` if it does not exist / is not UTF-8. +fn canonicalize_or(dir: &Path) -> Option { + let canon = dunce::canonicalize(dir.as_std_path()).ok()?; + PathBuf::try_from(canon).ok() } /// One project in a workspace: the root project, or a dependency instance @@ -617,7 +129,7 @@ fn resolve_edges( /// every dependency instance reachable from it, in depth-first declaration /// order. /// -/// This retraces the same edges [`import_dependency`] follows — de-duplicating +/// This retraces the same edges `import_dependency` follows — de-duplicating /// instances by canonical directory, so a diamond yields one entry whose /// `prefix` matches the store-key prefix its canisters were assigned — but keeps /// each instance's *raw* manifest instead of folding its canisters into the @@ -685,348 +197,6 @@ pub async fn workspace_instances( Ok(out) } -/// Build a dependency canister's friendly-URL subdomain prefix: the canister's -/// local name as the most-specific label, followed by its alias chain reversed -/// (root-most alias last). E.g. local `backend` reached via `[service-a, -/// openemail]` → `backend.openemail.service-a`. Dot-nested so it stays a valid, -/// collision-free multi-label host; see DESIGN §17.2. -fn friendly_name_for(local: &str, alias_chain: &[String]) -> String { - let mut labels = Vec::with_capacity(alias_chain.len() + 1); - labels.push(local.to_string()); - labels.extend(alias_chain.iter().rev().cloned()); - labels.join(".") -} - -/// Rewrite `CanisterName` controller references from a dependency's local -/// canister names to their store keys, so global controller validation and -/// deploy-time id lookup operate uniformly on store keys. -fn translate_controllers(canister: &mut Canister, local_to_key: &BTreeMap) { - translate_settings_controllers(&mut canister.settings, local_to_key); -} - -/// Rewrite `CanisterName` controller references in a `Settings` from a -/// dependency's local canister names to their store keys. -fn translate_settings_controllers( - settings: &mut Settings, - local_to_key: &BTreeMap, -) { - if let Some(controllers) = &mut settings.controllers { - for cref in controllers.iter_mut() { - if let ControllerRef::CanisterName(name) = cref - && let Some(key) = local_to_key.get(name) - { - *name = key.clone(); - } - } - } -} - -/// Compute the `PUBLIC_CANISTER_ID` env-var wiring for canisters in one project -/// scope: its own canisters by local name, plus each dependency's exposed -/// canisters under `:`. -fn compute_bindings( - own: &[(String, String)], - edges: &[(String, Vec<(String, String)>)], -) -> BTreeMap { - let mut bindings = BTreeMap::new(); - for (local, key) in own { - bindings.insert(local.clone(), key.clone()); - } - for (alias, exposed) in edges { - for (dep_local, key) in exposed { - bindings.insert(format!("{alias}:{dep_local}"), key.clone()); - } - } - bindings -} - -/// Select which of a dependency instance's own canisters are exposed to the -/// parent, per the dependency's `canisters` selection. -fn select_exposed( - own: &[(String, String)], - selection: &CanisterSelection, - alias: &str, -) -> Result, ConsolidateManifestError> { - match selection { - CanisterSelection::Everything => Ok(own.to_vec()), - CanisterSelection::None => Ok(vec![]), - CanisterSelection::Named(names) => { - let mut out = Vec::new(); - for name in names { - match own.iter().find(|(local, _)| local == name) { - Some(pair) => out.push(pair.clone()), - None => { - return UnknownDependencyCanisterSnafu { - alias: alias.to_owned(), - canister: name.clone(), - } - .fail(); - } - } - } - Ok(out) - } - } -} - -/// Validate the dependency aliases declared in one project scope: no `:`, no -/// collision with a local canister name, and no duplicate alias. -fn validate_dependency_aliases( - deps: &[DependencyManifest], - own_canister_names: &HashSet, -) -> Result<(), ConsolidateManifestError> { - let mut seen: HashSet<&str> = HashSet::new(); - for d in deps { - if !is_valid_name(&d.name) { - return InvalidDependencyAliasSnafu { - alias: d.name.clone(), - } - .fail(); - } - if own_canister_names.contains(&d.name) { - return DependencyAliasCollisionSnafu { - alias: d.name.clone(), - } - .fail(); - } - if !seen.insert(&d.name) { - return DuplicateDependencyAliasSnafu { - alias: d.name.clone(), - } - .fail(); - } - } - Ok(()) -} - -/// Recursively import a dependency's canisters into `canisters`, keyed by their -/// app-root-relative store keys. De-duplicates instances by canonical path -/// (diamond dependencies deploy once) and detects cycles. Returns the imported -/// instance's prefix and its own canisters. -#[allow(clippy::too_many_arguments)] -async fn import_dependency( - app_root_canonical: &Path, - parent_dir: &Path, - dep: &DependencyManifest, - recipe_resolver: &dyn recipe::Resolve, - canisters: &mut IndexMap, - registry: &mut HashMap, - stack: &mut Vec, - member_env_overrides: &mut MemberEnvOverrides, - members: &mut Vec, - // Alias chain from the workspace root to and including this dependency, - // used to build friendly-URL subdomains (§17.2). - alias_chain: &[String], -) -> Result { - let dep_root = parent_dir.join(&dep.path); - let manifest_path = dep_root.join(PROJECT_MANIFEST); - if !manifest_path.is_file() { - return DependencyNotFoundSnafu { - alias: dep.name.clone(), - path: dep_root.to_string(), - } - .fail(); - } - - let canonical = canonicalize_dep(&dep.name, &dep_root)?; - - // Cycle detection. - if stack.contains(&canonical) { - let mut chain: Vec = stack.iter().map(|p| p.to_string()).collect(); - chain.push(canonical.to_string()); - return CircularDependencySnafu { - chain: chain.join(" -> "), - } - .fail(); - } - - // Diamond de-dup: same resolved directory means the same instance, deployed - // once. It is still reachable via this new alias chain, so register an - // additional friendly URL per chain (§17.3) rather than picking one — for - // the whole subtree (its own canisters *and* its transitive dependencies), - // each named by this chain extended with the canister's alias path below the - // instance. - if let Some(inst) = registry.get(&canonical) { - let inst = inst.clone(); - for (key, local, rel_chain) in &inst.subtree { - let mut chain = alias_chain.to_vec(); - chain.extend(rel_chain.iter().cloned()); - let fname = friendly_name_for(local, &chain); - if let Some((_, canister)) = canisters.get_mut(key) - && !canister.friendly_names.contains(&fname) - { - canister.friendly_names.push(fname); - } - } - return Ok(inst); - } - - stack.push(canonical.clone()); - - let prefix = relative_prefix(app_root_canonical, &canonical); - - let dep_manifest: ProjectManifest = - load_manifest_from_path(&manifest_path) - .await - .context(LoadDependencyManifestSnafu { - alias: dep.name.clone(), - })?; - - // Build the dependency's own canisters and key them under the prefix. All of - // them are imported (deploy-all); the `canisters` exposure subset is applied - // by the caller when wiring env vars. - let built = - build_manifest_canisters(&dep_root, &dep_manifest.canisters, recipe_resolver).await?; - - let mut own: Vec<(String, String)> = Vec::new(); - let mut local_to_key: BTreeMap = BTreeMap::new(); - for (local, cdir, mut canister) in built { - let store_key = format!("{prefix}:{local}"); - canister.name = store_key.clone(); - // Friendly URL from the alias chain, not the path-based store key. - canister.friendly_names = vec![friendly_name_for(&local, alias_chain)]; - own.push((local.clone(), store_key.clone())); - local_to_key.insert(local.clone(), store_key.clone()); - match canisters.entry(store_key.clone()) { - IndexEntry::Occupied(_) => { - return DuplicateSnafu { - kind: "canister".to_string(), - name: store_key, - } - .fail(); - } - IndexEntry::Vacant(e) => { - e.insert((cdir, canister)); - } - } - } - - // Now that every sibling's store key is known, translate the dependency's - // controller references (local sibling name -> store key). - for (_, key) in &own { - if let Some((_, canister)) = canisters.get_mut(key) { - translate_controllers(canister, &local_to_key); - } - } - - // Capture the member's own environments so the parent can honor its - // per-canister settings/init_args for the same-named environment - // (standalone-equivalence). The network binding and canister selection are - // ignored; only overrides on the member's *own* canisters are - // folded in — keys naming its dependencies are left to those dependencies. - let mut defined_envs: HashSet = HashSet::new(); - for env_item in &dep_manifest.environments { - let em: EnvironmentManifest = match env_item { - Item::Manifest(m) => m.clone(), - Item::Path(path) => { - let p = dep_root.join(path); - if !p.is_file() { - return NotFoundSnafu { - kind: "environment".to_string(), - path: p.to_string(), - } - .fail(); - } - load_manifest_from_path::(&p) - .await - .context(LoadEnvironmentSnafu)? - } - }; - defined_envs.insert(em.name.clone()); - if let Some(settings) = &em.settings { - for (local, s) in settings { - if let Some(key) = local_to_key.get(local) { - // Translate the override's own controller references from the - // member's local names to store keys, so name-based controllers - // resolve against the workspace id map just like base settings. - let mut s = s.clone(); - translate_settings_controllers(&mut s, &local_to_key); - member_env_overrides - .entry(em.name.clone()) - .or_default() - .entry(key.clone()) - .or_default() - .settings = Some(s); - } - } - } - if let Some(init_args) = &em.init_args { - for (local, ia) in init_args { - if let Some(key) = local_to_key.get(local) { - member_env_overrides - .entry(em.name.clone()) - .or_default() - .entry(key.clone()) - .or_default() - .init_args = Some(ia.clone()); - } - } - } - } - members.push(MemberEnvInfo { - prefix: prefix.clone(), - defined: defined_envs, - }); - - // Recurse into the dependency's own dependencies. - let own_names: HashSet = own.iter().map(|(l, _)| l.clone()).collect(); - validate_dependency_aliases(&dep_manifest.dependencies, &own_names)?; - - // The instance's subtree, for diamond-hit friendly-URL propagation: its own - // canisters sit at the instance root (empty relative alias chain); each - // nested dependency contributes its subtree prefixed with the nested alias. - let mut subtree: Vec<(String, String, Vec)> = own - .iter() - .map(|(local, key)| (key.clone(), local.clone(), Vec::new())) - .collect(); - - let mut edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for nested in &dep_manifest.dependencies { - let mut nested_chain = alias_chain.to_vec(); - nested_chain.push(nested.name.clone()); - let inst = Box::pin(import_dependency( - app_root_canonical, - &dep_root, - nested, - recipe_resolver, - canisters, - registry, - stack, - member_env_overrides, - members, - &nested_chain, - )) - .await?; - for (key, local, rel) in &inst.subtree { - let mut r = Vec::with_capacity(rel.len() + 1); - r.push(nested.name.clone()); - r.extend(rel.iter().cloned()); - subtree.push((key.clone(), local.clone(), r)); - } - let exposed = select_exposed(&inst.own, &nested.canisters, &nested.name)?; - edges.push((nested.name.clone(), exposed)); - } - - // Assign env-var bindings for this instance's own canisters. - let bindings = compute_bindings(&own, &edges); - for (_, key) in &own { - if let Some((_, canister)) = canisters.get_mut(key) { - canister.bindings = bindings.clone(); - } - } - - stack.pop(); - let instance = ImportedInstance { own, subtree }; - registry.insert(canonical, instance.clone()); - Ok(instance) -} - -/// Canonicalize into a UTF-8 path, or `None` if it does not exist / is not UTF-8. -fn canonicalize_or(dir: &Path) -> Option { - let canon = dunce::canonicalize(dir.as_std_path()).ok()?; - PathBuf::try_from(canon).ok() -} - /// The default set of target canisters when the user names none, honoring /// member-scoping. /// @@ -1063,587 +233,98 @@ pub fn member_scoped_canisters( Some(names) } -/// Build one environment's canister map: select from `canisters`, then apply the -/// member overrides for this environment (standalone-equivalence), then -/// the root's own overrides (highest precedence). Precedence is therefore -/// root-explicit > member-env > canister-base. -fn build_environment_canisters( - canisters: &IndexMap, - env_name: &str, - selection: &CanisterSelection, - member_overrides: Option<&HashMap>, - root_settings: Option<&HashMap>, - root_init_args: Option<&HashMap>, -) -> Result, ConsolidateManifestError> { - let mut cs = match selection { - CanisterSelection::None => IndexMap::new(), - CanisterSelection::Everything => canisters.clone(), - CanisterSelection::Named(names) => { - let mut cs: IndexMap = IndexMap::new(); - for name in names { - let v = canisters.get(name).ok_or( - InvalidCanisterSnafu { - environment: env_name.to_owned(), - canister: name.to_owned(), - } - .build(), - )?; - cs.insert(name.to_owned(), v.to_owned()); - } - cs - } - }; +#[cfg(test)] +mod tests { + use super::*; + use crate::canister::recipe::{FetchedRecipe, RemoteResourceResolve, ResolveError}; + use crate::host_files::HostFileAccess; + use crate::manifest::adapter::prebuilt::SourceField; + use crate::manifest::recipe::Recipe; + use crate::manifest::{PROJECT_MANIFEST, ProjectManifest, load_manifest_from_path}; + use crate::prelude::LOCAL; + use camino_tempfile::Utf8TempDir; + use icp_deploy_canister::sync_exec::StepProgress; - // Member overrides first (lower precedence than the root's own overrides). - if let Some(overrides) = member_overrides { - for (key, ov) in overrides { - if let Some((cpath, canister)) = cs.get_mut(key) { - if let Some(s) = &ov.settings { - (canister.settings, canister.environment_variable_files) = - resolve_manifest_settings(s, cpath, key)?; - } - if let Some(ia) = &ov.init_args { - canister.init_args = Some(resolve_manifest_init_args(ia, cpath, key)?); - } - } - } - } + /// Recipes and plugins are never used in this test; every canister is pre-built. + struct PanicResolver; - // Root overrides last (highest precedence). - if let Some(settings) = root_settings { - for (name, s) in settings { - if let Some((cpath, canister)) = cs.get_mut(name) { - (canister.settings, canister.environment_variable_files) = - resolve_manifest_settings(s, cpath, name)?; - } - } - } - if let Some(init_args) = root_init_args { - for (name, ia) in init_args { - if let Some((cpath, canister)) = cs.get_mut(name) { - canister.init_args = Some(resolve_manifest_init_args(ia, cpath, name)?); - } + #[async_trait::async_trait] + impl RemoteResourceResolve for PanicResolver { + async fn resolve_recipe(&self, _recipe: &Recipe) -> Result { + panic!("recipe resolver should not be called in this test"); } - } - - Ok(cs) -} - -/// Turns the ProjectManifest into a Project struct -/// - Adds the default Networks -/// - Adds the default Environment -/// - Imports any dependency projects' canisters -/// - Validates the manifest to make sure that: -/// - There are no duplicates -/// - All the environments have networks -/// - All the referenced canisters exist -/// - All the recipes have been resolved -pub async fn consolidate_manifest( - pdir: &Path, - recipe_resolver: &dyn recipe::Resolve, - m: &ProjectManifest, -) -> Result { - // Canisters. IndexMap (not HashMap) so the order from the project manifest is preserved - // through to consumers like `icp project bundle`, which needs reproducible output. - let mut canisters: IndexMap = IndexMap::new(); - - // Canonical app root, used to derive stable, order-independent store-key - // prefixes for imported dependency canisters. - let app_root_canonical = - canonicalize_dep("", pdir).unwrap_or_else(|_| pdir.to_owned()); - // This project's own canisters, keyed by their bare local names. - let app_built = build_manifest_canisters(pdir, &m.canisters, recipe_resolver).await?; - let mut app_own: Vec<(String, String)> = Vec::new(); - for (local, cdir, canister) in app_built { - app_own.push((local.clone(), local.clone())); - match canisters.entry(local.clone()) { - IndexEntry::Occupied(e) => { - return DuplicateSnafu { - kind: "canister".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - IndexEntry::Vacant(e) => { - e.insert((cdir, canister)); - } + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + panic!("recipe resolver should not be called in this test"); } - } - - // Import dependency projects. Each dependency is deployed in full and keyed - // under its app-root-relative path; diamonds (the same directory reached via - // multiple edges) resolve to a single instance. - let mut registry: HashMap = HashMap::new(); - let mut stack: Vec = Vec::new(); - // Member environment config folded into the root's same-named environments, - // and the per-member set of declared environment names for the strict rule. - let mut member_env_overrides: MemberEnvOverrides = HashMap::new(); - let mut members: Vec = Vec::new(); - let app_own_names: HashSet = app_own.iter().map(|(l, _)| l.clone()).collect(); - validate_dependency_aliases(&m.dependencies, &app_own_names)?; - let mut app_edges: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for dep in &m.dependencies { - let inst = import_dependency( - &app_root_canonical, - pdir, - dep, - recipe_resolver, - &mut canisters, - &mut registry, - &mut stack, - &mut member_env_overrides, - &mut members, - std::slice::from_ref(&dep.name), - ) - .await?; - let exposed = select_exposed(&inst.own, &dep.canisters, &dep.name)?; - app_edges.push((dep.name.clone(), exposed)); - } - - // Assign env-var bindings for this project's own canisters (own canisters by - // local name plus each dependency's exposed canisters under `:`). - let app_bindings = compute_bindings(&app_own, &app_edges); - for (_, key) in &app_own { - if let Some((_, canister)) = canisters.get_mut(key) { - canister.bindings = app_bindings.clone(); + async fn resolve_wasm( + &self, + _source: &SourceField, + _base_dir: &Path, + _sha256: Option<&str>, + _progress: Option<&dyn StepProgress>, + ) -> Result { + panic!("wasm resolver should not be called in this test"); } } - // Friendly URLs need no de-collision pass: the strict name rule (no '.') makes - // own canisters single-label and dependency canisters multi-label (dot-nested - // by alias chain), so their hostnames are disjoint by construction (§17.2). + fn write(dir: &Path, rel: &str, contents: &str) { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, contents).unwrap(); + } - // Validate that every canister-name controller reference points to a declared canister. - // Catching typos here turns "perpetual warning" into a clear load-time error. - for (canister_name, (_, canister)) in &canisters { - let Some(crefs) = &canister.settings.controllers else { - continue; - }; - for cref in crefs { - if let Some(ref_name) = cref.canister_name() - && !canisters.contains_key(ref_name) - { - return UnknownControllerCanisterSnafu { - canister: canister_name.to_owned(), - controller: ref_name.to_owned(), - } - .fail(); + fn manifest(canisters: &[&str], deps: &str) -> String { + let mut s = String::new(); + if canisters.is_empty() { + s.push_str("canisters: []\n"); + } else { + s.push_str("canisters:\n"); + for c in canisters { + s.push_str(&format!( + " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" + )); } } + s.push_str(deps); + s } - // Networks - let mut networks: HashMap = HashMap::new(); + #[tokio::test] + async fn member_scope_targets_only_the_members_canisters() { + let tmp = Utf8TempDir::new().unwrap(); + write( + tmp.path(), + "openemail/icp.yaml", + &manifest(&["backend", "frontend"], ""), + ); + write( + tmp.path(), + "icp.yaml", + &manifest( + &["backend"], + "dependencies:\n - name: openemail\n path: ./openemail\n", + ), + ); - // Add IC network first - this is always protected and non-overridable - networks.insert( - IC.to_string(), - Network { - name: IC.to_string(), - configuration: Configuration::Connected { - connected: Connected { - api_url: IC_MAINNET_NETWORK_API_URL.parse().unwrap(), - http_gateway_url: Some(IC_MAINNET_NETWORK_GATEWAY_URL.parse().unwrap()), - root_key: RootKeySpec::Mainnet, - }, - }, - }, - ); + let m: ProjectManifest = load_manifest_from_path(&tmp.path().join(PROJECT_MANIFEST)) + .await + .unwrap(); + let p = consolidate_manifest(&HostFileAccess, tmp.path(), &PanicResolver, &m) + .await + .unwrap(); + let env = p.environments.get(LOCAL).expect("local environment"); - // Track which network names are protected (only IC network) - let protected_network_names: HashSet = [IC.to_string()].into_iter().collect(); + // At the workspace root (member == root): no scoping. + assert_eq!(member_scoped_canisters(&p.dir, Some(&p.dir), env), None); - // Resolve NetworkManifests and add them (including user-defined "local" if provided) - for i in &m.networks { - let m = match i { - Item::Path(path) => { - let path = pdir.join(path); - if !path.exists() || !path.is_file() { - return NotFoundSnafu { - kind: "network".to_string(), - path: path.to_string(), - } - .fail(); - } - load_manifest_from_path::(&path) - .await - .context(LoadNetworkSnafu)? - } - Item::Manifest(ms) => ms.clone(), - }; - - match networks.entry(m.name.to_owned()) { - // Duplicate - Entry::Occupied(e) => { - // Only error if trying to override a protected network - if protected_network_names.contains(&m.name) { - return ReservedSnafu { - kind: "network".to_string(), - name: m.name.to_string(), - } - .fail(); - } - - // For non-protected duplicates, this is a user error (defining same network twice) - return DuplicateSnafu { - kind: "network".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - - // Ok - Entry::Vacant(e) => { - e.insert(Network { - name: m.name.to_owned(), - configuration: m.configuration.into(), // Convert manifest to config struct - }); - } - } - } - - // After processing user networks, add default "local" if not already defined - // This provides backward compatibility for projects that don't define their own "local" network - if !networks.contains_key(LOCAL) { - networks.insert( - LOCAL.to_string(), - Network { - name: LOCAL.to_string(), - configuration: Configuration::Managed { - managed: Managed { - mode: ManagedMode::Launcher(Box::new(ManagedLauncherConfig { - gateway: Gateway { - bind: DEFAULT_LOCAL_NETWORK_BIND.to_string(), - port: Port::Fixed(DEFAULT_LOCAL_NETWORK_PORT), - domains: vec![], - }, - artificial_delay_ms: None, - ii: false, - nns: false, - subnets: None, - bitcoind_addr: None, - dogecoind_addr: None, - version: None, - })), - }, - }, - }, - ); - } - - // Environments - let mut environments: HashMap = HashMap::new(); - - for i in &m.environments { - let m = match i { - Item::Path(path) => { - let path = pdir.join(path); - if !path.exists() || !path.is_file() { - return NotFoundSnafu { - kind: "environment".to_string(), - path: path.to_string(), - } - .fail(); - } - load_manifest_from_path::(&path) - .await - .context(LoadEnvironmentSnafu)? - } - Item::Manifest(ms) => ms.clone(), - }; - - match environments.entry(m.name.to_owned()) { - // Duplicate - Entry::Occupied(e) => { - return DuplicateSnafu { - kind: "environment".to_string(), - name: e.key().to_owned(), - } - .fail(); - } - - // Ok - Entry::Vacant(e) => { - e.insert(Environment { - name: m.name.to_owned(), - - // Embed network in environment - network: { - let v = networks.get(&m.network).ok_or( - InvalidNetworkSnafu { - environment: m.name.to_owned(), - network: m.network.to_owned(), - } - .build(), - )?; - - v.to_owned() - }, - - // Embed canisters in environment, folding member overrides - // beneath the root's own settings/init_args overrides. - canisters: build_environment_canisters( - &canisters, - &m.name, - &m.canisters, - member_env_overrides.get(&m.name), - m.settings.as_ref(), - m.init_args.as_ref(), - )?, - }); - } - } - } - - // We're done adding all the user environments - // Now we add the implicit `local` and `ic` environment if the user hasn't overriden it - if let Entry::Vacant(vacant_entry) = environments.entry(LOCAL.to_string()) { - let network = networks - .get(LOCAL) - .ok_or( - InvalidNetworkSnafu { - environment: LOCAL.to_owned(), - network: LOCAL.to_owned(), - } - .build(), - )? - .to_owned(); - vacant_entry.insert(Environment { - name: LOCAL.to_string(), - network, - canisters: build_environment_canisters( - &canisters, - LOCAL, - &CanisterSelection::Everything, - member_env_overrides.get(LOCAL), - None, - None, - )?, - }); - } - if let Entry::Vacant(vacant_entry) = environments.entry(IC.to_string()) { - let network = networks - .get(IC) - .ok_or( - InvalidNetworkSnafu { - environment: IC.to_owned(), - network: IC.to_owned(), - } - .build(), - )? - .to_owned(); - vacant_entry.insert(Environment { - name: IC.to_string(), - network, - canisters: build_environment_canisters( - &canisters, - IC, - &CanisterSelection::Everything, - member_env_overrides.get(IC), - None, - None, - )?, - }); - } - - // Strict rule: every member must declare each environment the root targets. - // `local`/`ic` are implicit for every project, so they never count - // as missing; other environments must be declared explicitly by the member. - // Recorded per-environment and enforced lazily when that environment is - // selected (so a missing `staging` never blocks `deploy -e local`). - let mut member_missing_envs: HashMap> = HashMap::new(); - for env_name in environments.keys() { - if env_name == LOCAL || env_name == IC { - continue; - } - for member in &members { - if !member.defined.contains(env_name) { - member_missing_envs - .entry(env_name.clone()) - .or_default() - .push(member.prefix.clone()); - } - } - } - - Ok(Project { - dir: pdir.into(), - canisters, - networks, - environments, - member_missing_envs, - }) -} - -#[cfg(test)] -mod dependency_tests { - use super::*; - use crate::canister::recipe::{Fetched, Resolve, ResolveError}; - use crate::manifest::recipe::Recipe; - use camino_tempfile::Utf8TempDir; - - /// Recipes are never used in these tests; every canister is pre-built. - struct PanicResolver; - - #[async_trait::async_trait] - impl Resolve for PanicResolver { - async fn resolve(&self, _recipe: &Recipe) -> Result { - panic!("recipe resolver should not be called in dependency tests"); - } - } - - fn write(dir: &Path, rel: &str, contents: &str) { - let p = dir.join(rel); - std::fs::create_dir_all(p.parent().unwrap()).unwrap(); - std::fs::write(p, contents).unwrap(); - } - - /// A minimal `icp.yaml` body declaring the given pre-built canisters, - /// followed by a raw `dependencies:` block (may be empty). - fn manifest(canisters: &[&str], deps: &str) -> String { - let mut s = String::new(); - if canisters.is_empty() { - s.push_str("canisters: []\n"); - } else { - s.push_str("canisters:\n"); - for c in canisters { - s.push_str(&format!( - " - name: {c}\n build:\n steps:\n - type: pre-built\n path: {c}.wasm\n" - )); - } - } - s.push_str(deps); - s - } - - async fn consolidate(pdir: &Path) -> Result { - let m: ProjectManifest = load_manifest_from_path(&pdir.join(PROJECT_MANIFEST)) - .await - .expect("failed to parse project manifest"); - consolidate_manifest(pdir, &PanicResolver, &m).await - } - - fn bindings_of<'a>(p: &'a Project, key: &str) -> &'a BTreeMap { - &p.canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - p.canisters.keys().collect::>() - ) - }) - .1 - .bindings - } - - fn friendly_names_of<'a>(p: &'a Project, key: &str) -> &'a [String] { - &p.canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - p.canisters.keys().collect::>() - ) - }) - .1 - .friendly_names - } - - #[tokio::test] - async fn single_project_bindings_are_self_and_siblings() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Flat behavior preserved: every canister maps every sibling (incl. self) - // to itself. - let expected = BTreeMap::from([ - ("backend".to_string(), "backend".to_string()), - ("frontend".to_string(), "frontend".to_string()), - ]); - assert_eq!(bindings_of(&p, "backend"), &expected); - assert_eq!(bindings_of(&p, "frontend"), &expected); - } - - #[tokio::test] - async fn dependency_import_and_exposure_subset() { - let tmp = Utf8TempDir::new().unwrap(); - // Dependency nested inside the app (mirrors a submodule under the app). - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [backend]\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // The whole dependency is deployed (both canisters imported), keyed by path. - assert!(p.canisters.contains_key("backend")); - assert!(p.canisters.contains_key("openemail:backend")); - assert!(p.canisters.contains_key("openemail:frontend")); - - // App's own canister sees itself and only the *exposed* dependency canister. - assert_eq!( - bindings_of(&p, "backend"), - &BTreeMap::from([ - ("backend".to_string(), "backend".to_string()), - ( - "openemail:backend".to_string(), - "openemail:backend".to_string() - ), - ]) - ); - - // The dependency's own canisters keep their standalone view (bare names). - assert_eq!( - bindings_of(&p, "openemail:backend"), - &BTreeMap::from([ - ("backend".to_string(), "openemail:backend".to_string()), - ("frontend".to_string(), "openemail:frontend".to_string()), - ]) - ); - } - - #[tokio::test] - async fn member_scope_targets_only_the_members_canisters() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend", "frontend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let env = p.environments.get(LOCAL).expect("local environment"); - - // At the workspace root (member == root): no scoping. - assert_eq!(member_scoped_canisters(&p.dir, Some(&p.dir), env), None); - - // Unknown member dir: no scoping. - assert_eq!(member_scoped_canisters(&p.dir, None, env), None); + // Unknown member dir: no scoping. + assert_eq!(member_scoped_canisters(&p.dir, None, env), None); // Inside the member: only the member's own canisters, not the app's. let member = tmp.path().join("openemail"); @@ -1658,777 +339,4 @@ mod dependency_tests { ] ); } - - #[tokio::test] - async fn member_env_config_folds_in_with_root_override_winning() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail defines `staging` with per-canister settings for its own - // canisters. - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - name: backend - build: - steps: - - type: pre-built - path: backend.wasm - - name: frontend - build: - steps: - - type: pre-built - path: frontend.wasm -environments: - - name: staging - settings: - backend: - compute_allocation: 5 - frontend: - compute_allocation: 7 -"#, - ); - // The app declares openemail and also defines `staging`, overriding the - // imported backend's settings (the root override must win). - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging - settings: - "openemail:backend": - compute_allocation: 99 -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - - // Root override wins over the member's config. - assert_eq!( - staging - .canisters - .get("openemail:backend") - .unwrap() - .1 - .settings - .compute_allocation, - Some(99), - ); - // No root override → the member's own config applies (standalone-equivalence). - assert_eq!( - staging - .canisters - .get("openemail:frontend") - .unwrap() - .1 - .settings - .compute_allocation, - Some(7), - ); - // Both projects declared staging, so nothing is recorded as missing. - assert!(p.member_missing_envs.is_empty()); - } - - #[tokio::test] - async fn missing_member_environment_is_recorded() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - // openemail does not declare `staging`, so it is recorded as missing. - assert_eq!( - p.member_missing_envs.get("staging"), - Some(&vec!["openemail".to_string()]), - ); - // Implicit environments are never recorded as missing. - assert!(!p.member_missing_envs.contains_key("local")); - assert!(!p.member_missing_envs.contains_key("ic")); - } - - #[tokio::test] - async fn diamond_dedups_to_single_instance() { - let tmp = Utf8TempDir::new().unwrap(); - // umbrella layout: service-a and service-b both depend on ../openemail. - write( - tmp.path(), - "umbrella/openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "umbrella/service-a/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "umbrella/service-b/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // openemail is imported exactly once despite two edges reaching it. - let openemail_keys: Vec<_> = p - .canisters - .keys() - .filter(|k| k.contains("openemail")) - .collect(); - assert_eq!( - openemail_keys, - vec![&"umbrella/openemail:backend".to_string()], - "expected a single shared openemail instance" - ); - - // Both services' code reads `openemail:backend`, resolving to the one instance. - assert_eq!( - bindings_of(&p, "umbrella/service-a:backend").get("openemail:backend"), - Some(&"umbrella/openemail:backend".to_string()) - ); - assert_eq!( - bindings_of(&p, "umbrella/service-b:backend").get("openemail:backend"), - Some(&"umbrella/openemail:backend".to_string()) - ); - - // The single shared instance is reachable at one friendly URL per alias - // chain (§17.3) — the store-key path (`umbrella/`) never appears. - assert_eq!( - friendly_names_of(&p, "umbrella/openemail:backend"), - &["backend.openemail.service-a", "backend.openemail.service-b"] - ); - // Each service's own canister is named by its own alias chain. - assert_eq!( - friendly_names_of(&p, "umbrella/service-a:backend"), - &["backend.service-a"] - ); - assert_eq!( - friendly_names_of(&p, "umbrella/service-b:backend"), - &["backend.service-b"] - ); - } - - #[tokio::test] - async fn diamond_transitive_dependency_gets_url_per_chain() { - let tmp = Utf8TempDir::new().unwrap(); - // The shared openemail itself depends on libfoo, and is reached via both - // service-a and service-b. - write( - tmp.path(), - "umbrella/openemail/libfoo/icp.yaml", - &manifest(&["bar"], ""), - ); - write( - tmp.path(), - "umbrella/openemail/icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: libfoo\n path: ./libfoo\n", - ), - ); - write( - tmp.path(), - "umbrella/service-a/icp.yaml", - &manifest( - &["service"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "umbrella/service-b/icp.yaml", - &manifest( - &["service"], - "dependencies:\n - name: openemail\n path: ../openemail\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: service-a\n path: ./umbrella/service-a\n - name: service-b\n path: ./umbrella/service-b\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // The shared instance's own canister gets one URL per chain... - assert_eq!( - friendly_names_of(&p, "umbrella/openemail:backend"), - &["backend.openemail.service-a", "backend.openemail.service-b"] - ); - // ...and so does its *transitive* dependency (the subtree is revisited on - // the diamond hit, not just the instance's own canisters). - assert_eq!( - friendly_names_of(&p, "umbrella/openemail/libfoo:bar"), - &[ - "bar.libfoo.openemail.service-a", - "bar.libfoo.openemail.service-b" - ] - ); - } - - #[tokio::test] - async fn dot_in_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - // '.' is banned: it would be ambiguous in a dot-nested friendly subdomain - // (an own canister named `frontend.openemail` could collide with dependency - // `openemail`'s `frontend`). The strict name rule rejects it up front. - write( - tmp.path(), - "icp.yaml", - &manifest(&["frontend.openemail"], ""), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn invalid_dependency_alias_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["app"], - "dependencies:\n - name: open.email\n path: ./openemail\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidDependencyAlias { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn member_override_controllers_are_translated_to_store_keys() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail's `staging` override names a controller by its local name. - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - name: backend - build: - steps: - - type: pre-built - path: backend.wasm - - name: frontend - build: - steps: - - type: pre-built - path: frontend.wasm -environments: - - name: staging - settings: - backend: - controllers: ["frontend"] -"#, - ); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: app - build: - steps: - - type: pre-built - path: app.wasm -dependencies: - - name: openemail - path: ./openemail -environments: - - name: staging -"#, - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - let controllers = staging - .canisters - .get("openemail:backend") - .unwrap() - .1 - .settings - .controllers - .clone() - .expect("controllers set by the member override"); - - // The member-local `frontend` must be translated to its store key, so it - // resolves against the workspace id map at deploy time. - assert_eq!( - controllers, - vec![ControllerRef::CanisterName( - "openemail:frontend".to_string() - )] - ); - } - - fn env_vars_of<'a>( - canisters: &'a IndexMap, - key: &str, - ) -> &'a HashMap { - canisters - .get(key) - .unwrap_or_else(|| { - panic!( - "canister '{key}' not found; have {:?}", - canisters.keys().collect::>() - ) - }) - .1 - .settings - .environment_variables - .as_ref() - .expect("environment variables set") - } - - /// A canister manifest's file-backed environment variable resolves against - /// the canister's own directory, and the file's trailing newline is not part - /// of the value. - #[tokio::test] - async fn env_var_file_resolves_against_canister_dir() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "canisters/backend/canister.yaml", - r#" -name: backend -settings: - environment_variables: - API_KEY: - path: secrets/api-key -build: - steps: - - type: pre-built - path: backend.wasm -"#, - ); - write(tmp.path(), "canisters/backend/secrets/api-key", "s3cret\n"); - write( - tmp.path(), - "icp.yaml", - "canisters:\n - ./canisters/backend\n", - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - assert_eq!( - env_vars_of(&p.canisters, "backend"), - &HashMap::from([("API_KEY".to_string(), "s3cret".to_string())]), - ); - } - - /// A canister declared in its own directory, for the override tests below: - /// its directory is neither the project's nor an environment manifest's, so - /// the base a path resolves against is unambiguous. - fn write_backend_canister(dir: &Path, at: &str) { - write( - dir, - &format!("{at}/canister.yaml"), - r#" -name: backend -build: - steps: - - type: pre-built - path: backend.wasm -"#, - ); - } - - /// An environment override resolves a path against the *canister's* directory - /// — the same base an `init_args` override uses — not against the manifest - /// declaring the override, even when that is an environment manifest of its - /// own. - #[tokio::test] - async fn env_var_file_in_environment_override_resolves_against_canister_dir() { - let tmp = Utf8TempDir::new().unwrap(); - write_backend_canister(tmp.path(), "canisters/backend"); - write( - tmp.path(), - "icp.yaml", - "canisters:\n - ./canisters/backend\nenvironments:\n - ./environments/staging.yaml\n", - ); - write( - tmp.path(), - "environments/staging.yaml", - r#" -name: staging -settings: - backend: - environment_variables: - API_KEY: - path: secrets/api-key -"#, - ); - write( - tmp.path(), - "canisters/backend/secrets/api-key", - "staging-key\n", - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - - assert_eq!( - env_vars_of(&staging.canisters, "backend"), - &HashMap::from([("API_KEY".to_string(), "staging-key".to_string())]), - ); - // The override applies to the environment only; the canister's own - // settings are untouched. - assert_eq!( - p.canisters - .get("backend") - .unwrap() - .1 - .settings - .environment_variables, - None, - ); - } - - /// A member's own environment override resolves against the member's - /// canister, not against the member's or the root's project directory. - #[tokio::test] - async fn env_var_file_in_member_environment_resolves_against_member_canister_dir() { - let tmp = Utf8TempDir::new().unwrap(); - write_backend_canister(tmp.path(), "openemail/canisters/backend"); - write( - tmp.path(), - "openemail/icp.yaml", - r#" -canisters: - - ./canisters/backend -environments: - - name: staging - settings: - backend: - environment_variables: - API_KEY: - path: secrets/api-key -"#, - ); - write( - tmp.path(), - "openemail/canisters/backend/secrets/api-key", - "member-key\n", - ); - write( - tmp.path(), - "icp.yaml", - &format!( - "{}environments:\n - name: staging\n", - manifest( - &["app"], - "dependencies:\n - name: openemail\n path: ./openemail\n" - ) - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - let staging = p.environments.get("staging").expect("staging environment"); - - assert_eq!( - env_vars_of(&staging.canisters, "openemail:backend"), - &HashMap::from([("API_KEY".to_string(), "member-key".to_string())]), - ); - } - - #[tokio::test] - async fn missing_env_var_file_is_reported_with_the_variable_and_canister() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - r#" -canisters: - - name: backend - settings: - environment_variables: - API_KEY: - path: secrets/api-key - build: - steps: - - type: pre-built - path: backend.wasm -"#, - ); - - let err = consolidate(tmp.path()) - .await - .expect_err("the environment variable's file does not exist"); - - assert!( - matches!( - &err, - ConsolidateManifestError::ReadEnvironmentVariable { canister, variable, .. } - if canister == "backend" && variable == "API_KEY" - ), - "unexpected error: {err}" - ); - } - - #[tokio::test] - async fn friendly_names_are_bare_for_own_and_dotted_for_dependencies() { - let tmp = Utf8TempDir::new().unwrap(); - // openemail (with a transitive dep libfoo) vendored under the app. - write( - tmp.path(), - "openemail/libfoo/icp.yaml", - &manifest(&["bar"], ""), - ); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest( - &["backend", "frontend"], - "dependencies:\n - name: libfoo\n path: ./libfoo\n", - ), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Own canister: bare name (unchanged from single-project behavior). - assert_eq!(friendly_names_of(&p, "backend"), &["backend"]); - // Direct dependency: dot-nested by alias (no `vendor/` path noise). - assert_eq!( - friendly_names_of(&p, "openemail:backend"), - &["backend.openemail"] - ); - assert_eq!( - friendly_names_of(&p, "openemail:frontend"), - &["frontend.openemail"] - ); - // Transitive dependency: full alias chain, canister-most-specific first. - assert_eq!( - friendly_names_of(&p, "openemail/libfoo:bar"), - &["bar.libfoo.openemail"] - ); - } - - #[tokio::test] - async fn cycle_is_detected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest(&[], "dependencies:\n - name: a\n path: ./a\n"), - ); - write( - tmp.path(), - "a/icp.yaml", - &manifest(&["x"], "dependencies:\n - name: b\n path: ../b\n"), - ); - write( - tmp.path(), - "b/icp.yaml", - &manifest(&["y"], "dependencies:\n - name: a\n path: ../a\n"), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::CircularDependency { .. }), - "expected CircularDependency, got {err:?}" - ); - } - - #[tokio::test] - async fn alias_colliding_with_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["openemail"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::DependencyAliasCollision { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn duplicate_alias_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write(tmp.path(), "one/icp.yaml", &manifest(&["backend"], "")); - write(tmp.path(), "two/icp.yaml", &manifest(&["backend"], "")); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: dup\n path: ./one\n - name: dup\n path: ./two\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::DuplicateDependencyAlias { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn colon_in_canister_name_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write(tmp.path(), "icp.yaml", &manifest(&["foo:bar"], "")); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::InvalidCanisterName { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn unknown_exposed_canister_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: openemail\n path: ./openemail\n canisters: [nope]\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!( - err, - ConsolidateManifestError::UnknownDependencyCanister { .. } - ), - "got {err:?}" - ); - } - - #[tokio::test] - async fn missing_dependency_path_is_rejected() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "icp.yaml", - &manifest( - &[], - "dependencies:\n - name: openemail\n path: ./does-not-exist\n", - ), - ); - - let err = consolidate(tmp.path()).await.unwrap_err(); - assert!( - matches!(err, ConsolidateManifestError::DependencyNotFound { .. }), - "got {err:?}" - ); - } - - #[tokio::test] - async fn imported_canisters_appear_in_implicit_environments() { - let tmp = Utf8TempDir::new().unwrap(); - write( - tmp.path(), - "openemail/icp.yaml", - &manifest(&["backend"], ""), - ); - write( - tmp.path(), - "icp.yaml", - &manifest( - &["backend"], - "dependencies:\n - name: openemail\n path: ./openemail\n", - ), - ); - - let p = consolidate(tmp.path()).await.unwrap(); - - // Deploy-all: the implicit `local` environment includes the dependency. - let local = p.environments.get("local").unwrap(); - assert!(local.canisters.contains_key("backend")); - assert!(local.canisters.contains_key("openemail:backend")); - } } From 9ade34030c2928be0581dab4e4ebf97620a2ce4d Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Tue, 25 Aug 2026 08:54:10 -0700 Subject: [PATCH 24/51] feat(deploy-canister): add in-library implementations of the host seams `InMemoryIdStore`, `NoScripts`, and `NoResolve` let a host stand the library up without supplying its own implementation of every seam. `NoScripts` rejects script sync steps, for an environment with no subprocesses; `NoResolve` serves recipe files and plugin wasms through the injected `FileAccess` and rejects anything that would have to be fetched, still verifying a configured `sha256`. --- .../src/canister/recipe/mod.rs | 133 ++++++++++++++++++ crates/icp-deploy-canister/src/ids.rs | 47 ++++++- crates/icp-deploy-canister/src/sync_exec.rs | 25 +++- crates/icp/src/canister/recipe/mod.rs | 4 +- 4 files changed, 204 insertions(+), 5 deletions(-) diff --git a/crates/icp-deploy-canister/src/canister/recipe/mod.rs b/crates/icp-deploy-canister/src/canister/recipe/mod.rs index edc150541..9ca7b5852 100644 --- a/crates/icp-deploy-canister/src/canister/recipe/mod.rs +++ b/crates/icp-deploy-canister/src/canister/recipe/mod.rs @@ -3,8 +3,10 @@ use std::collections::HashMap; use async_trait::async_trait; use handlebars::{Context, Handlebars, Helper, HelperDef, HelperResult, Output, RenderContext}; use serde::Deserialize; +use sha2::{Digest, Sha256}; use snafu::prelude::*; +use crate::files::FileAccess; use crate::manifest::{ adapter::prebuilt::SourceField, canister::{BuildSteps, SyncSteps}, @@ -112,6 +114,107 @@ pub enum ResolveError { }, } +/// A [`RemoteResourceResolve`] that serves only local resources — recipe files +/// read through the injected [`FileAccess`], and plugin wasms already on hand — +/// rejecting anything that would have to be fetched. For environments with no +/// HTTP access. +pub struct NoResolve(pub F); + +#[derive(Debug, Snafu)] +#[snafu(display("remote recipes are not allowed"))] +pub struct NoResolveError; + +#[derive(Debug, Snafu)] +#[snafu(display("remote modules are not allowed"))] +pub struct NoResolveModuleError; + +#[derive(Debug, Snafu)] +#[snafu(display( + "sha256 checksum mismatch for plugin wasm '{path}': expected {expected}, actual {actual}" +))] +pub struct WasmChecksumMismatchError { + path: PathBuf, + expected: String, + actual: String, +} + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +impl RemoteResourceResolve for NoResolve { + async fn resolve_recipe(&self, recipe: &Recipe) -> Result { + match &recipe.recipe_type { + RecipeType::File(path) => { + let template = self.0.read_to_string(path.as_ref()).await.map_err(|e| { + ResolveError::Resolve { + source: Box::new(e), + } + })?; + Ok(FetchedRecipe { + template, + deferred: false, + }) + } + _ => Err(ResolveError::Resolve { + source: Box::new(NoResolveError), + }), + } + } + + /// Local reads defer nothing. + async fn commit_recipe( + &self, + _recipe: &Recipe, + _fetched: &FetchedRecipe, + ) -> Result<(), ResolveError> { + Ok(()) + } + + async fn resolve_wasm( + &self, + source: &SourceField, + base_dir: &Path, + sha256: Option<&str>, + progress: Option<&dyn StepProgress>, + ) -> Result { + let SourceField::Local(source) = source else { + return Err(ResolveError::ResolveWasm { + source: Box::new(NoResolveModuleError), + }); + }; + let path = base_dir.join(&source.path); + + if let Some(expected) = sha256 { + if let Some(p) = progress { + p.line(format!("Reading wasm: {path}")); + } + let bytes = self + .0 + .read_file(&path) + .await + .map_err(|e| ResolveError::ResolveWasm { + source: Box::new(e), + })?; + if let Some(p) = progress { + p.line("Verifying checksum".to_string()); + } + let actual = hex::encode(Sha256::digest(&bytes)); + if actual != expected { + return Err(ResolveError::ResolveWasm { + source: Box::new( + WasmChecksumMismatchSnafu { + path, + expected, + actual, + } + .build(), + ), + }); + } + } + Ok(path) + } +} + #[derive(Debug, Snafu)] pub enum RenderRecipeError { #[snafu(display("recipe template for '{recipe}' failed to render"))] @@ -194,7 +297,9 @@ impl HelperDef for ReplaceHelper { #[cfg(test)] mod tests { use super::*; + use crate::manifest::adapter::prebuilt::LocalSource; use crate::manifest::canister::BuildStep; + use crate::testutil::HostFiles; fn recipe(config: &[(&str, &str)]) -> Recipe { Recipe { @@ -315,4 +420,32 @@ mod tests { Err(RenderRecipeError::Parse { .. }) )); } + + /// A plugin wasm resolved locally is still checked against a configured + /// `sha256`, and the path is taken relative to `base_dir`. + #[tokio::test] + async fn no_resolve_verifies_local_wasm_checksum() { + let tmp = camino_tempfile::Utf8TempDir::new().unwrap(); + std::fs::write(tmp.path().join("plugin.wasm"), b"plugin bytes").unwrap(); + + let resolver = NoResolve(HostFiles); + let source = SourceField::Local(LocalSource { + path: "plugin.wasm".into(), + }); + let good = hex::encode(Sha256::digest(b"plugin bytes")); + + assert_eq!( + resolver + .resolve_wasm(&source, tmp.path(), Some(&good), None) + .await + .unwrap(), + tmp.path().join("plugin.wasm") + ); + assert!( + resolver + .resolve_wasm(&source, tmp.path(), Some(&"00".repeat(32)), None) + .await + .is_err() + ); + } } diff --git a/crates/icp-deploy-canister/src/ids.rs b/crates/icp-deploy-canister/src/ids.rs index 4665f445e..962a11bfa 100644 --- a/crates/icp-deploy-canister/src/ids.rs +++ b/crates/icp-deploy-canister/src/ids.rs @@ -1,9 +1,9 @@ //! Canister-id store: per-environment `name → principal` mappings. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, sync::Mutex}; use candid::Principal; -use snafu::Snafu; +use snafu::{OptionExt, Snafu}; /// Mapping of canister names to their principals within an environment. pub type IdMapping = BTreeMap; @@ -40,3 +40,46 @@ pub trait IdStore: Send + Sync { canister_id: Principal, ) -> Result<(), IdStoreError>; } + +#[derive(Debug, Default)] +pub struct InMemoryIdStore(pub Mutex>); + +impl IdStore for InMemoryIdStore { + fn lookup( + &self, + _is_cache: bool, + env: &str, + canister_name: &str, + ) -> Result { + let mapping = self.lookup_by_environment(_is_cache, env)?; + mapping + .get(canister_name) + .cloned() + .context(NotFoundSnafu { env, canister_name }) + } + + fn lookup_by_environment(&self, _is_cache: bool, env: &str) -> Result { + self.0 + .lock() + .unwrap() + .get(env) + .cloned() + .context(AccessSnafu { + env, + message: "environment not found", + }) + } + + fn register( + &self, + _is_cache: bool, + env: &str, + canister_name: &str, + canister_id: Principal, + ) -> Result<(), IdStoreError> { + let mut store = self.0.lock().unwrap(); + let mapping = store.entry(env.to_string()).or_default(); + mapping.insert(canister_name.to_string(), canister_id); + Ok(()) + } +} diff --git a/crates/icp-deploy-canister/src/sync_exec.rs b/crates/icp-deploy-canister/src/sync_exec.rs index 7047459b8..2c7c62ca9 100644 --- a/crates/icp-deploy-canister/src/sync_exec.rs +++ b/crates/icp-deploy-canister/src/sync_exec.rs @@ -11,7 +11,8 @@ //! //! The two executors are separate traits because an environment can support one //! without the other. Script steps are host-only, and are rejected by -//! [`crate::project::verify_sandbox`] before they reach an executor. +//! [`crate::project::verify_sandbox`] before they reach an executor; [`NoScripts`] +//! is the ready-made [`ScriptRunner`] for a host that has no subprocesses. use std::collections::BTreeMap; @@ -289,6 +290,28 @@ pub trait ScriptRunner: Send + Sync { ) -> Result, ScriptRunError>; } +/// A [`ScriptRunner`] that always fails, for environments that don't support +/// subprocess script sync steps. +pub struct NoScripts; + +#[derive(Debug, Snafu)] +#[snafu(display("script sync steps are not supported in this environment"))] +pub struct NoScriptsError; + +#[cfg_attr(target_family = "wasm", async_trait(?Send))] +#[cfg_attr(not(target_family = "wasm"), async_trait)] +impl ScriptRunner for NoScripts { + async fn run_script( + &self, + _invocation: ScriptInvocation, + _progress: Option<&dyn StepProgress>, + ) -> Result, ScriptRunError> { + Err(ScriptRunError { + source: Box::new(NoScriptsError), + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/icp/src/canister/recipe/mod.rs b/crates/icp/src/canister/recipe/mod.rs index eeb806a2d..834d19c65 100644 --- a/crates/icp/src/canister/recipe/mod.rs +++ b/crates/icp/src/canister/recipe/mod.rs @@ -10,8 +10,8 @@ //! the cache write waits. pub use icp_deploy_canister::canister::recipe::{ - FetchedRecipe, RecipeContext, RemoteResourceResolve, RenderRecipeError, ResolveError, - render_recipe, + FetchedRecipe, NoResolve, RecipeContext, RemoteResourceResolve, RenderRecipeError, + ResolveError, render_recipe, }; pub mod resolver; From c7f168f9716af520ea4ec4dd1163a82ff42f5938 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Tue, 25 Aug 2026 08:54:38 -0700 Subject: [PATCH 25/51] feat(deploy-canister): add bundle_get_canister_module_path Extract the local wasm module path a bundled canister is built from, erroring when the build is not the single-`pre-built`-local-path shape a bundle requires. Lets a host resolve a bundled canister's module without reaching into the build steps itself. --- crates/icp-deploy-canister/src/lib.rs | 122 ++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/crates/icp-deploy-canister/src/lib.rs b/crates/icp-deploy-canister/src/lib.rs index 2445e37b1..33383194a 100644 --- a/crates/icp-deploy-canister/src/lib.rs +++ b/crates/icp-deploy-canister/src/lib.rs @@ -146,6 +146,55 @@ pub struct Canister { pub environment_variable_files: BTreeMap, } +#[derive(Debug, Snafu)] +pub enum BundleModulePathError { + #[snafu(display( + "canister '{canister}' does not have a single build step (found {count}); a bundled \ + canister must be built by exactly one pre-built step" + ))] + NotSingleBuildStep { canister: String, count: usize }, + + #[snafu(display( + "canister '{canister}' is not built by a pre-built step; a bundled canister's module \ + must come from a `pre-built` build step" + ))] + NotPrebuilt { canister: String }, + + #[snafu(display("canister '{canister}' is built from a remote URL, not a local module path"))] + NotLocal { canister: String }, +} + +/// Extract the local wasm module path a bundled canister is built from. +/// +/// A bundle's canisters are each built by a single `pre-built` step pointing at +/// the module on disk; this returns that path, erroring if the build is not that +/// single-prebuilt-local-path shape. +pub fn bundle_get_canister_module_path( + canister: &Canister, +) -> Result<&Path, BundleModulePathError> { + let steps = &canister.build.steps; + let [step] = steps.as_slice() else { + return NotSingleBuildStepSnafu { + canister: canister.name.clone(), + count: steps.len(), + } + .fail(); + }; + let manifest::BuildStep::Prebuilt(adapter) = step else { + return NotPrebuiltSnafu { + canister: canister.name.clone(), + } + .fail(); + }; + match &adapter.source { + manifest::prebuilt::SourceField::Local(local) => Ok(&local.path), + manifest::prebuilt::SourceField::Remote(_) => NotLocalSnafu { + canister: canister.name.clone(), + } + .fail(), + } +} + #[derive(Clone, Debug, PartialEq, Serialize)] pub struct Network { pub name: String, @@ -202,3 +251,76 @@ impl Project { self.canisters.get(canister_name) } } + +#[cfg(test)] +mod bundle_tests { + use super::*; + use crate::canister::Settings; + use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; + use crate::manifest::adapter::{prebuilt, script}; + use crate::manifest::canister::{BuildStep, SyncSteps}; + + fn canister_with_build(steps: Vec) -> Canister { + Canister { + name: "backend".to_owned(), + settings: Settings::default(), + build: BuildSteps { steps }, + sync: SyncSteps { steps: vec![] }, + init_args: None, + registry_recipe: None, + bindings: BTreeMap::new(), + friendly_names: vec![], + environment_variable_files: BTreeMap::new(), + } + } + + fn prebuilt_local(path: &str) -> BuildStep { + BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Local(LocalSource { path: path.into() }), + sha256: None, + }) + } + + #[test] + fn extracts_the_single_prebuilt_local_path() { + let c = canister_with_build(vec![prebuilt_local("out/backend.wasm")]); + assert_eq!( + bundle_get_canister_module_path(&c).unwrap(), + Path::new("out/backend.wasm") + ); + } + + #[test] + fn rejects_zero_or_multiple_build_steps() { + let two = canister_with_build(vec![prebuilt_local("a.wasm"), prebuilt_local("b.wasm")]); + assert!(matches!( + bundle_get_canister_module_path(&two), + Err(BundleModulePathError::NotSingleBuildStep { count: 2, .. }) + )); + } + + #[test] + fn rejects_a_non_prebuilt_step() { + let c = canister_with_build(vec![BuildStep::Script(script::Adapter { + command: script::CommandField::Command("make".to_owned()), + })]); + assert!(matches!( + bundle_get_canister_module_path(&c), + Err(BundleModulePathError::NotPrebuilt { .. }) + )); + } + + #[test] + fn rejects_a_remote_prebuilt_source() { + let c = canister_with_build(vec![BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Remote(RemoteSource { + url: "https://example.com/backend.wasm".to_owned(), + }), + sha256: Some("abc".to_owned()), + })]); + assert!(matches!( + bundle_get_canister_module_path(&c), + Err(BundleModulePathError::NotLocal { .. }) + )); + } +} From 3d51f26858e7086c2503af960b6b5b9545c54635 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Wed, 19 Aug 2026 09:06:11 -0700 Subject: [PATCH 26/51] Add parallel plugin interface version support --- Cargo.lock | 1 + crates/icp-sync-plugin/Cargo.toml | 1 + crates/icp-sync-plugin/DESIGN.md | 34 +- crates/icp-sync-plugin/build.rs | 20 +- crates/icp-sync-plugin/src/runtime.rs | 298 ++++++++++++--- crates/icp-sync-plugin/sync-plugin-v1.wit | 99 +++++ crates/icp-sync-plugin/sync-plugin.wit | 2 +- .../tests/fixtures/test-plugin-v1/Cargo.lock | 349 ++++++++++++++++++ .../tests/fixtures/test-plugin-v1/Cargo.toml | 13 + .../tests/fixtures/test-plugin-v1/build.rs | 3 + .../tests/fixtures/test-plugin-v1/src/lib.rs | 24 ++ docs/concepts/sync-plugins.md | 2 + 12 files changed, 781 insertions(+), 65 deletions(-) create mode 100644 crates/icp-sync-plugin/sync-plugin-v1.wit create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs create mode 100644 crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 67ca726a3..330024485 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3785,6 +3785,7 @@ dependencies = [ "hex", "ic-agent", "icp-canister-interfaces", + "semver", "snafu", "tokio", "wasmtime", diff --git a/crates/icp-sync-plugin/Cargo.toml b/crates/icp-sync-plugin/Cargo.toml index 216e9c761..74feb682e 100644 --- a/crates/icp-sync-plugin/Cargo.toml +++ b/crates/icp-sync-plugin/Cargo.toml @@ -15,6 +15,7 @@ console.workspace = true hex.workspace = true ic-agent.workspace = true icp-canister-interfaces.workspace = true +semver.workspace = true snafu.workspace = true tokio.workspace = true wasmtime.workspace = true diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 099a14714..411376f93 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -103,11 +103,12 @@ the WASI sandbox — cap-std — at runtime.) ### `HostState` and bindgen +Both interface versions are bound, each `bindgen!` in its own module so their +generated types don't collide: + ```rust -wasmtime::component::bindgen!({ - world: "sync-plugin", - path: "sync-plugin.wit", -}); +mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugin.wit" }); } +mod v1 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugin-v1.wit" }); } struct HostState { target_canister_id: Principal, @@ -118,17 +119,30 @@ struct HostState { epoch_extension: Arc, } -impl SyncPluginImports for HostState { - fn canister_call(&mut self, req: CanisterCallRequest) -> Result, String> { ... } -} +// Implemented for both v1::SyncPluginImports and v2::SyncPluginImports; both +// delegate to one shared `do_canister_call(...)`. ``` `HostState` implements `WasiView` so wasmtime_wasi can access the WASI context. `canister_call` uses `tokio::runtime::Handle::current().block_on(...)` because the caller already wraps the synchronous `run_plugin` in -`tokio::task::block_in_place`. When a proxy is configured and the call is a -non-`direct` update, it is encoded as `ProxyArgs` and routed through the proxy's -`proxy` method; otherwise it goes straight to the target via `ic-agent`. +`tokio::task::block_in_place`. Both interface versions call the canister being +synced. When a proxy is configured and the call is a non-`direct` update, it is +encoded as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise +it goes straight to the target via `ic-agent`. + +### Interface versioning (parallel v0.1.0 / v0.2.0 support) + +A component built with wit-bindgen imports the interface it `use`s as a +versioned instance — `icp:sync-plugin/types@0.1.0` or `@0.2.0`. `run_plugin` +reads that name off `Component::component_type().imports(...)` and matches the +version with semver caret requirements (`^0.1`, `^0.2`) to pick the ABI, then +instantiates the matching `bindgen!` world and builds the matching +`sync-exec-input`. Reading the plugin's declared metadata is preferred over +trial instantiation: it is unambiguous and needs no throwaway `Store`. A +component with no recognized `icp:sync-plugin/types@` import, or an +unsupported version, is rejected with `UnsupportedInterface`. Both `.wit` files +are checked in; `sync-plugin-v1.wit` is the frozen v0.1.0 contract. ### Compute budget (epoch interruption) diff --git a/crates/icp-sync-plugin/build.rs b/crates/icp-sync-plugin/build.rs index cbe6461f3..8e08a442e 100644 --- a/crates/icp-sync-plugin/build.rs +++ b/crates/icp-sync-plugin/build.rs @@ -3,11 +3,17 @@ use std::process::Command; fn main() { println!("cargo:rerun-if-changed=sync-plugin.wit"); + println!("cargo:rerun-if-changed=sync-plugin-v1.wit"); println!("cargo:rerun-if-changed=tests/fixtures/test-plugin/src/lib.rs"); println!("cargo:rerun-if-changed=tests/fixtures/test-plugin/Cargo.toml"); + println!("cargo:rerun-if-changed=tests/fixtures/test-plugin-v1/src/lib.rs"); + println!("cargo:rerun-if-changed=tests/fixtures/test-plugin-v1/Cargo.toml"); if wasm32_wasip2_is_installed() { - build_test_fixture(); + // Current-interface fixture, and a legacy-interface one so tests can + // exercise both load paths. + build_test_fixture("test-plugin", "test_plugin", "TEST_PLUGIN_WASM"); + build_test_fixture("test-plugin-v1", "test_plugin_v1", "TEST_PLUGIN_V1_WASM"); } } @@ -24,11 +30,11 @@ fn wasm32_wasip2_is_installed() -> bool { .exists() } -fn build_test_fixture() { +fn build_test_fixture(crate_dir: &str, wasm_stem: &str, env_var: &str) { let manifest_dir = Utf8PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); let out_dir = Utf8PathBuf::from(std::env::var("OUT_DIR").unwrap()); - let fixture_manifest = manifest_dir.join("tests/fixtures/test-plugin/Cargo.toml"); - let fixture_target_dir = out_dir.join("fixture-target"); + let fixture_manifest = manifest_dir.join(format!("tests/fixtures/{crate_dir}/Cargo.toml")); + let fixture_target_dir = out_dir.join(format!("fixture-target/{crate_dir}")); let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); let status = Command::new(&cargo) @@ -47,8 +53,8 @@ fn build_test_fixture() { .expect("failed to spawn cargo build for test fixture"); assert!( status.success(), - "cargo build --target wasm32-wasip2 failed for test fixture" + "cargo build --target wasm32-wasip2 failed for test fixture {crate_dir}" ); - let wasm = fixture_target_dir.join("wasm32-wasip2/release/test_plugin.wasm"); - println!("cargo:rustc-env=TEST_PLUGIN_WASM={wasm}"); + let wasm = fixture_target_dir.join(format!("wasm32-wasip2/release/{wasm_stem}.wasm")); + println!("cargo:rustc-env={env_var}={wasm}"); } diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index fb284fb7f..27091fc34 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -24,19 +24,38 @@ use bytes::Bytes; use camino::Utf8PathBuf; use candid::{Encode, Principal}; use ic_agent::Agent; +use semver::{Version, VersionReq}; use snafu::prelude::*; use tokio::io::{self, AsyncWrite}; use tokio::sync::mpsc::Sender; +use wasmtime::component::{Component, HasSelf, Linker}; +use wasmtime::{Config, Engine, Store}; use wasmtime_wasi::cli::{IsTerminal, StdoutStream}; use wasmtime_wasi::p2::{OutputStream, Pollable, StreamError}; use wasmtime_wasi::{DirPerms, FilePerms}; -wasmtime::component::bindgen!({ - world: "sync-plugin", - path: "sync-plugin.wit", -}); +// Both the current and the legacy plugin interfaces are bound, each in its own +// module so their generated type names don't collide. `run_plugin` reads the +// interface version from the component's own metadata (see `detect_plugin_abi`) +// and drives it through the matching module, so plugins built against either +// interface load. The two interfaces are currently structurally identical; the +// split exists so later breaking changes to the current interface can land +// without dropping support for already-built plugins. +mod v2 { + wasmtime::component::bindgen!({ + world: "sync-plugin", + path: "sync-plugin.wit", + }); +} + +mod v1 { + wasmtime::component::bindgen!({ + world: "sync-plugin", + path: "sync-plugin-v1.wit", + }); +} -use icp::sync_plugin::types::CallType; +use v2::icp::sync_plugin::types::CallType; // HostState holds everything the plugin's import functions need. struct HostState { @@ -64,31 +83,35 @@ impl wasmtime_wasi::WasiView for HostState { } } -// `types::Host` is an empty marker trait generated for the `types` interface. -impl icp::sync_plugin::types::Host for HostState {} - -impl SyncPluginImports for HostState { - fn canister_call(&mut self, req: CanisterCallRequest) -> Result, String> { +impl HostState { + /// Perform a canister call to the canister being synced. Shared by both + /// interface versions. + fn do_canister_call( + &mut self, + method: String, + arg_bytes: Vec, + call_type: CallType, + direct: bool, + cycles: u64, + ) -> Result, String> { use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; - let arg_bytes = req.arg; let cid = self.target_canister_id; - let method = req.method.clone(); let agent = Arc::clone(&self.agent); - let proxy = if req.direct { None } else { self.proxy }; + let proxy = if direct { None } else { self.proxy }; // We are already inside tokio::task::block_in_place (see sync/plugin.rs), // so blocking the thread here is safe. let start = Instant::now(); let result = tokio::runtime::Handle::current().block_on(async move { - match req.call_type { + match call_type { CallType::Update => { if let Some(proxy_cid) = proxy { let proxy_args = ProxyArgs { canister_id: cid, method: method.clone(), args: arg_bytes, - cycles: candid::Nat::from(req.cycles), + cycles: candid::Nat::from(cycles), }; let encoded = Encode!(&proxy_args) .map_err(|e| format!("proxy encode failed: {e}"))?; @@ -128,6 +151,38 @@ impl SyncPluginImports for HostState { } } +// -- v0.2.0 interface. --------------------------------------------------------- + +// `types::Host` is an empty marker trait generated for the `types` interface. +impl v2::icp::sync_plugin::types::Host for HostState {} + +impl v2::SyncPluginImports for HostState { + fn canister_call( + &mut self, + req: v2::icp::sync_plugin::types::CanisterCallRequest, + ) -> Result, String> { + self.do_canister_call(req.method, req.arg, req.call_type, req.direct, req.cycles) + } +} + +// -- v0.1.0 interface. --------------------------------------------------------- + +impl v1::icp::sync_plugin::types::Host for HostState {} + +impl v1::SyncPluginImports for HostState { + fn canister_call( + &mut self, + req: v1::icp::sync_plugin::types::CanisterCallRequest, + ) -> Result, String> { + // v1's `call-type` is a distinct generated enum; map it to the shared one. + let call_type = match req.call_type { + v1::icp::sync_plugin::types::CallType::Update => CallType::Update, + v1::icp::sync_plugin::types::CallType::Query => CallType::Query, + }; + self.do_canister_call(req.method, req.arg, call_type, req.direct, req.cycles) + } +} + // Used as the error payload inside the epoch_deadline_callback closure, which // must return wasmtime::Error (= anyhow::Error). Snafu derives std::error::Error // so .into() converts it via anyhow's blanket From. @@ -191,6 +246,12 @@ pub enum RunPluginError { path: Utf8PathBuf, }, + #[snafu(display( + "wasm component at {path} does not implement a supported sync-plugin interface ({detail}). \ + Supported: icp:sync-plugin@0.1 and icp:sync-plugin@0.2." + ))] + UnsupportedInterface { path: Utf8PathBuf, detail: String }, + #[snafu(display("failed to call exec() on plugin at {path}"))] CallExec { source: wasmtime::Error, @@ -201,6 +262,72 @@ pub enum RunPluginError { PluginFailed { message: String }, } +/// Which version of the sync-plugin interface a component was built against. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PluginAbi { + /// Current interface (`icp:sync-plugin@0.2.x`). + V2, + /// Legacy interface (`icp:sync-plugin@0.1.x`). + V1, +} + +/// The interface package a `use`-ing plugin component imports, whose version we +/// read to pick the ABI. wit-bindgen emits this import for any world that pulls +/// types from the interface, so it is present on every real plugin. +const TYPES_INTERFACE_PREFIX: &str = "icp:sync-plugin/types@"; + +/// Determine which interface a component implements by reading the version off +/// its imported `icp:sync-plugin/types@` instance — the plugin's own +/// declared metadata — rather than probing with a trial instantiation. The +/// version is matched with semver caret requirements, so each supported minor +/// (the breaking unit for 0.x) accepts any patch release within it. +fn detect_plugin_abi( + engine: &Engine, + component: &Component, + wasm_path: &Utf8PathBuf, +) -> Result { + let raw = component + .component_type() + .imports(engine) + .find_map(|(name, _)| name.strip_prefix(TYPES_INTERFACE_PREFIX).map(str::to_owned)); + + let Some(raw) = raw else { + return UnsupportedInterfaceSnafu { + path: wasm_path.clone(), + detail: format!("no {TYPES_INTERFACE_PREFIX} import found"), + } + .fail(); + }; + + let version = Version::parse(&raw).map_err(|source| { + UnsupportedInterfaceSnafu { + path: wasm_path.clone(), + detail: format!("interface version '{raw}' is not valid semver: {source}"), + } + .build() + })?; + + // `^0.1`/`^0.2` follow semver's 0.x rule: they match within the minor and + // exclude the next one (>=0.1.0, <0.2.0 and >=0.2.0, <0.3.0 respectively). + if VersionReq::parse("^0.2") + .expect("valid req") + .matches(&version) + { + Ok(PluginAbi::V2) + } else if VersionReq::parse("^0.1") + .expect("valid req") + .matches(&version) + { + Ok(PluginAbi::V1) + } else { + UnsupportedInterfaceSnafu { + path: wasm_path.clone(), + detail: format!("unsupported interface version {version}"), + } + .fail() + } +} + #[allow(clippy::too_many_arguments)] pub fn run_plugin( wasm_path: Utf8PathBuf, @@ -215,9 +342,6 @@ pub fn run_plugin( compute_limit_secs: u64, stdio: Option>, ) -> Result, RunPluginError> { - use wasmtime::component::{Component, Linker}; - use wasmtime::{Config, Engine, Store}; - let mut config = Config::new(); config.wasm_component_model(true); config.max_wasm_stack(MAX_WASM_STACK); @@ -283,7 +407,9 @@ pub fn run_plugin( // Read each declared file on the host and pass its content inline. The same // path-safety checks as `dirs` apply: reject escaping or symlinked paths so // a read cannot leave `base_dir`. - let mut file_inputs: Vec = Vec::with_capacity(files.len()); + // Held as plain (name, content) pairs so they can be converted to whichever + // interface version's `file-input` record the plugin turns out to use. + let mut file_contents: Vec<(String, String)> = Vec::with_capacity(files.len()); for name in &files { ensure!(!crate::path::escapes_base(name), UnsafeFileSnafu { name }); if let Some(link) = crate::path::first_symlink_component(&base_dir, name) { @@ -292,10 +418,7 @@ pub fn run_plugin( let path = base_dir.join(name); let content = std::fs::read_to_string(path.as_std_path()).context(ReadFileSnafu { path })?; - file_inputs.push(FileInput { - name: name.clone(), - content, - }); + file_contents.push((name.clone(), content)); } let persistent_stderr: Arc>> = Arc::default(); @@ -315,16 +438,6 @@ pub fn run_plugin( epoch_extension: epoch_extension.clone(), }; - let mut linker: Linker = Linker::new(&engine); - wasmtime_wasi::p2::add_to_linker_sync(&mut linker).context(InstantiateSnafu { - path: wasm_path.clone(), - })?; - SyncPlugin::add_to_linker::<_, wasmtime::component::HasSelf<_>>(&mut linker, |s| s).context( - InstantiateSnafu { - path: wasm_path.clone(), - }, - )?; - let mut store = Store::new(&engine, host_state); store.set_epoch_deadline(compute_limit_secs); store.epoch_deadline_callback(move |_| { @@ -339,22 +452,72 @@ pub fn run_plugin( } }); - let plugin = - SyncPlugin::instantiate(&mut store, &component, &linker).context(InstantiateSnafu { - path: wasm_path.clone(), - })?; - - let input = SyncExecInput { - canister_id: target_canister_id.to_text(), - environment, - dirs, - files: file_inputs, - identity_principal: identity_principal.to_text(), - proxy_canister_id: proxy.map(|p| p.to_text()), + let canister_id_text = target_canister_id.to_text(); + let identity_text = identity_principal.to_text(); + let proxy_text = proxy.map(|p| p.to_text()); + + // Which interface the plugin was built against is read from the component's + // own declared metadata (see `detect_plugin_abi`) rather than probed by + // trial instantiation, then driven through the matching bindgen world. + let call_result = match detect_plugin_abi(&engine, &component, &wasm_path)? { + PluginAbi::V2 => { + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_sync(&mut linker).context(InstantiateSnafu { + path: wasm_path.clone(), + })?; + v2::SyncPlugin::add_to_linker::<_, HasSelf<_>>(&mut linker, |s| s).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let plugin = v2::SyncPlugin::instantiate(&mut store, &component, &linker).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let input = v2::SyncExecInput { + canister_id: canister_id_text, + environment, + dirs, + files: file_contents + .into_iter() + .map(|(name, content)| v2::FileInput { name, content }) + .collect(), + identity_principal: identity_text, + proxy_canister_id: proxy_text, + }; + plugin.call_exec(&mut store, &input) + } + PluginAbi::V1 => { + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::p2::add_to_linker_sync(&mut linker).context(InstantiateSnafu { + path: wasm_path.clone(), + })?; + v1::SyncPlugin::add_to_linker::<_, HasSelf<_>>(&mut linker, |s| s).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let plugin = v1::SyncPlugin::instantiate(&mut store, &component, &linker).context( + InstantiateSnafu { + path: wasm_path.clone(), + }, + )?; + let input = v1::SyncExecInput { + canister_id: canister_id_text, + environment, + dirs, + files: file_contents + .into_iter() + .map(|(name, content)| v1::FileInput { name, content }) + .collect(), + identity_principal: identity_text, + proxy_canister_id: proxy_text, + }; + plugin.call_exec(&mut store, &input) + } }; - let call_result = plugin.call_exec(&mut store, &input); - // Flush any partial line and emit the truncation note (if any) before // we hand control back, so the last line of plugin output isn't lost. stdout_capture.finalize(); @@ -782,6 +945,47 @@ mod tests { assert!(msg.contains("stdout from plugin"), "got: {msg}"); } + #[test] + fn legacy_v1_plugin_is_detected_and_driven() { + // A plugin built against the v0.1.0 interface must still load: the host + // reads its declared interface version and drives it through the v1 path. + let Some(wasm_path) = option_env!("TEST_PLUGIN_V1_WASM") else { + return; + }; + let result = run_plugin( + wasm_path.into(), + ".".into(), + vec![], + vec![], + anon(), + dummy_agent(), + None, + anon(), + "ok".to_string(), + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, + None, + ); + assert!(result.is_ok()); + // Its error surface flows through the same machinery as v0.2.0 plugins. + let result = run_plugin( + wasm_path.into(), + ".".into(), + vec![], + vec![], + anon(), + dummy_agent(), + None, + anon(), + "error".to_string(), + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, + None, + ); + assert!(matches!( + result, + Err(RunPluginError::PluginFailed { ref message }) if message == "deliberate v1 failure" + )); + } + #[tokio::test(flavor = "multi_thread")] async fn plugin_stderr_lines_returned_as_persistent_output() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { diff --git a/crates/icp-sync-plugin/sync-plugin-v1.wit b/crates/icp-sync-plugin/sync-plugin-v1.wit new file mode 100644 index 000000000..65c0283a9 --- /dev/null +++ b/crates/icp-sync-plugin/sync-plugin-v1.wit @@ -0,0 +1,99 @@ +// Version 0.1.0 of the sync-plugin interface, preserved verbatim so the host +// can still load plugins built against it. New plugins should target the +// current interface in `sync-plugin.wit`; see that file for the up-to-date +// contract. The host tries the current interface first and falls back to this +// one, so both APIs are supported in parallel. +package icp:sync-plugin@0.1.0; + +/// Types shared between the host runtime and sync plugins. +interface types { + /// Whether a canister call is an update or a query. + enum call-type { update, query } + + /// A file the host read on behalf of the plugin. + record file-input { + /// Path of the file as declared in the manifest (relative to + /// the canister directory). + name: string, + /// UTF-8 contents of the file. + content: string, + } + + /// Input passed by the runtime to the plugin's exec() export. + record sync-exec-input { + /// Textual principal of the canister being synced. + canister-id: string, + /// Name of the environment being synced (e.g. "production", "local"). + environment: string, + /// Directories declared in the manifest step's `dirs` setting. + /// The host preopens each entry via WASI; the plugin can traverse + /// them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). + dirs: list, + /// Files declared in the manifest step's `files` setting, read by + /// the host and passed inline. The plugin decides how to use them. + files: list, + /// Textual principal of the signing identity used for canister calls. + identity-principal: string, + /// Textual principal of the proxy canister, if one was configured via + /// `--proxy`. None when no proxy is in use. + proxy-canister-id: option, + } + + /// A request to call a method on the target canister. + record canister-call-request { + /// The canister method to call. + method: string, + /// Candid-encoded argument bytes. The plugin is responsible for + /// encoding; the host forwards these bytes unchanged. + arg: list, + /// Whether to perform an `update` or `query` call. + call-type: call-type, + /// When true, the call bypasses any proxy canister configured via + /// `--proxy`, going directly to the target canister. When false + /// (the default), update calls are routed through the proxy if one + /// is configured; query calls always go directly to the target + /// canister regardless of this flag. + direct: bool, + /// Cycles to attach to a proxied update call. Only meaningful when + /// `direct` is `false`, a proxy canister is configured, and + /// `call-type` is `update`; silently ignored for direct calls and + /// for query calls. + cycles: u64, + } +} + +/// The complete interface of a sync plugin. +world sync-plugin { + use types.{sync-exec-input, canister-call-request, file-input}; + + // ------------------------------------------------------------------------- + // Host functions (imports) — provided by icp-cli, called by the plugin + // ------------------------------------------------------------------------- + + /// Make an update or query call to the canister being synced. + /// The host always calls the canister from sync-exec-input.canister-id; + /// the plugin does not choose the target. + /// Returns the raw Candid-encoded response bytes on success or an error + /// message on failure. The plugin is responsible for decoding. + import canister-call: func(req: canister-call-request) -> result, string>; + + // The plugin's stdout is captured and shown as transient progress in + // the rolling step view of icp-cli; it is discarded when the step ends. + // + // The plugin's stderr is captured and shown in the rolling step view AND + // printed persistently after the step completes successfully. (On + // failure, the error message and the rolling-view dump already surface + // stderr, so it is not reprinted.) + // + // Use stdout for in-flight progress chatter the user doesn't need to see + // once the step is done. Use stderr for messages the user must still see + // after the step completes — warnings, summaries, deprecation notices. + + // ------------------------------------------------------------------------- + // Plugin exports — implemented by the plugin, called by the host + // ------------------------------------------------------------------------- + + /// Execute the sync plugin for the canister being synced. Returns an + /// error message on failure. + export exec: func(input: sync-exec-input) -> result<_, string>; +} diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 64fc8d11d..9e7ffb297 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -1,4 +1,4 @@ -package icp:sync-plugin@0.1.0; +package icp:sync-plugin@0.2.0; /// Types shared between the host runtime and sync plugins. interface types { diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock new file mode 100644 index 000000000..edb77b53d --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.lock @@ -0,0 +1,349 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "macro-string" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test-plugin-v1" +version = "0.1.0" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "wasm-encoder" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61fb705ce81adde29d2a8e99d87995e39a6e927358c91398f374474746070ef7" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e4c2aa916c425dcca61a6887d3e135acdee2c6d0ed51fd61c08d41ddaf62b1" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71cde4757396defafd25417cfb36aa3161027d06d865b0c24baaae229aac005d" +dependencies = [ + "bitflags", + "hashbrown 0.16.1", + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7607d30e7e5e8fd5a0695f7cb8b2128829e0bf9dca7a1fe8c4d6ed3ca1058fce" +dependencies = [ + "bitflags", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda3a4ce47c08d27f575d451a60102bab5251776abd0a7a323d1f038eb6339ab" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920a1c8c0f89397431db4900a7bf7c511b78e1b7068289fe812dc76e993f1491" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn 2.0.119", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "857a143d2373abfcd31ad946393efe775ed8c90a2a365ce73c61bf38f36a1000" +dependencies = [ + "anyhow", + "macro-string", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1936c26cb24b93dc36bf78fb5dc35c55cd37f66ecdc2d2663a717d9fb3ee951e" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.246.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd979042b5ff288607ccf3b314145435453f20fc67173195f91062d2289b204d" +dependencies = [ + "anyhow", + "hashbrown 0.16.1", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml new file mode 100644 index 000000000..25d44def3 --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] + +[package] +name = "test-plugin-v1" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = { version = "0.56", features = ["realloc"] } diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs new file mode 100644 index 000000000..b23d985e0 --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=../../../sync-plugin-v1.wit"); +} diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs new file mode 100644 index 000000000..24ebea1bf --- /dev/null +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs @@ -0,0 +1,24 @@ +// A plugin built against the *legacy* (v0.1.0) interface, used to prove the host +// still loads and drives v0.1.0 plugins alongside v0.2.0 ones. It cannot choose +// a call target and never sees the canister ID table — that is the whole point. +wit_bindgen::generate!({ + world: "sync-plugin", + path: "../../../sync-plugin-v1.wit", +}); + +struct TestPluginV1; + +impl Guest for TestPluginV1 { + fn exec(input: SyncExecInput) -> Result<(), String> { + match input.environment.as_str() { + "error" => Err("deliberate v1 failure".to_string()), + "hello" => { + eprintln!("hello from v1"); + Ok(()) + } + _ => Ok(()), + } + } +} + +export!(TestPluginV1); diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index b586ac0d5..f8310deea 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -54,6 +54,8 @@ world sync-plugin { } ``` +The interface is versioned (currently `icp:sync-plugin@0.2.0`). icp-cli reads the version a plugin was built against from the component itself and drives it accordingly, so plugins built against the earlier `@0.1.0` interface — which could only call the canister being synced — continue to load unchanged. + The authoritative interface, including all record fields, lives in [`sync-plugin.wit`](https://github.com/dfinity/icp-cli/blob/main/crates/icp-sync-plugin/sync-plugin.wit) in the icp-cli repository. ### What the plugin receives — `sync-exec-input` From 7dff16a084019a903ba36c24b6a524d1feb68ff0 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 08:36:05 -0700 Subject: [PATCH 27/51] fix comment --- crates/icp-sync-plugin/sync-plugin-v1.wit | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/icp-sync-plugin/sync-plugin-v1.wit b/crates/icp-sync-plugin/sync-plugin-v1.wit index 65c0283a9..5cb8ee2f6 100644 --- a/crates/icp-sync-plugin/sync-plugin-v1.wit +++ b/crates/icp-sync-plugin/sync-plugin-v1.wit @@ -1,8 +1,6 @@ -// Version 0.1.0 of the sync-plugin interface, preserved verbatim so the host -// can still load plugins built against it. New plugins should target the +// Version 0.1.0 of the sync-plugin interface. New plugins should target the // current interface in `sync-plugin.wit`; see that file for the up-to-date -// contract. The host tries the current interface first and falls back to this -// one, so both APIs are supported in parallel. +// contract. The host will load this version if the plugin imports it. package icp:sync-plugin@0.1.0; /// Types shared between the host runtime and sync plugins. From daa1b7508fb32e2612c3e80e54e84ff3daa8f24c Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 09:28:53 -0700 Subject: [PATCH 28/51] fix clippy (???) --- .../icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs index 24ebea1bf..6d38c71ea 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin-v1/src/lib.rs @@ -1,6 +1,9 @@ +#![allow(clippy::too_many_arguments)] + // A plugin built against the *legacy* (v0.1.0) interface, used to prove the host // still loads and drives v0.1.0 plugins alongside v0.2.0 ones. It cannot choose // a call target and never sees the canister ID table — that is the whole point. + wit_bindgen::generate!({ world: "sync-plugin", path: "../../../sync-plugin-v1.wit", From 7464f803a09b1f61432e5917e6fe48898cca6fdd Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 09:36:49 -0700 Subject: [PATCH 29/51] copilot --- crates/icp-sync-plugin/DESIGN.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 411376f93..dd54eb68f 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -12,7 +12,8 @@ invokes its `exec()` export during `icp sync` for a single canister. > - [Sync Plugins](../../docs/concepts/sync-plugins.md) — concept, WIT interface, sandbox, resource limits > - [Writing a Sync Plugin](../../docs/guides/writing-sync-plugins.md) — authoring guide (Rust) > - [Plugin Sync (Configuration Reference)](../../docs/reference/configuration.md) — `type: plugin` manifest fields -> - [`sync-plugin.wit`](sync-plugin.wit) — the interface, and the sole source of truth +> - [`sync-plugin.wit`](sync-plugin.wit) — the current interface (v0.2.0), and its source of truth +> - [`sync-plugin-v1.wit`](sync-plugin-v1.wit) — the frozen v0.1.0 interface, still loadable --- @@ -36,8 +37,8 @@ docs; the *reasons* behind those choices are recorded here. - **Logging via stdio, not a host import** — stdout/stderr are captured by the host and forwarded to the CLI. Plugins use normal print facilities. - **No generated bindings checked in** — `wasmtime::component::bindgen!` (host) - and `wit_bindgen::generate!` (guest) both run at build time from the WIT file, - which stays the single source of truth. + and `wit_bindgen::generate!` (guest) both run at build time from the WIT files, + which stay the source of truth for the interface they define. --- @@ -50,10 +51,12 @@ Host-side Component Model runtime for sync plugins. ``` crates/icp-sync-plugin/ src/ - lib.rs — public API: run_plugin(), RunPluginError - runtime.rs — wasmtime component setup, HostState, bindgen!, exec() call - sync-plugin.wit — WIT interface (source of truth) - Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, candid, camino, snafu, tokio + lib.rs — public API: run_plugin(), RunPluginError + runtime.rs — wasmtime component setup, HostState, bindgen!, exec() call + path.rs — declared-path safety checks (escapes_base, symlinks) + sync-plugin.wit — current WIT interface, v0.2.0 + sync-plugin-v1.wit — frozen WIT interface, v0.1.0 + Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, candid, camino, snafu, tokio, semver ``` Public function: From 327c5bc390520e80ee0bcc6ae6d259cfb41a646e Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 04:51:42 -0700 Subject: [PATCH 30/51] Expose the project canister ID table to sync plugins --- crates/icp-cli/src/operations/sync.rs | 1 + crates/icp-sync-plugin/DESIGN.md | 33 ++- crates/icp-sync-plugin/src/lib.rs | 3 +- crates/icp-sync-plugin/src/runtime.rs | 297 ++++++++++--------------- crates/icp-sync-plugin/sync-plugin.wit | 26 ++- crates/icp/src/canister/sync/mod.rs | 5 + crates/icp/src/canister/sync/plugin.rs | 103 ++++++++- crates/icp/src/canister/sync/script.rs | 1 + docs/concepts/sync-plugins.md | 6 +- 9 files changed, 267 insertions(+), 208 deletions(-) diff --git a/crates/icp-cli/src/operations/sync.rs b/crates/icp-cli/src/operations/sync.rs index 18c77efcc..77ea04174 100644 --- a/crates/icp-cli/src/operations/sync.rs +++ b/crates/icp-cli/src/operations/sync.rs @@ -59,6 +59,7 @@ async fn sync_canister( &Params { path: canister_path.clone(), cid: canister_id, + name: canister_info.name.clone(), environment: environment.to_owned(), network: network.to_owned(), canister_ids: canister_ids.clone(), diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index dd54eb68f..0f636dd0b 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -31,6 +31,10 @@ docs; the *reasons* behind those choices are recorded here. from `sync-exec-input.canister-id`. There is deliberately no field for a different target, so the single-canister restriction is *structural* rather than a policy the plugin could bypass. +- **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes + the project's name→principal map for the environment, so a plugin can resolve + canister names it knows about. It is informational only; calling still + requires a declaration. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. @@ -62,21 +66,14 @@ crates/icp-sync-plugin/ Public function: ```rust -pub fn run_plugin( - wasm_path: Utf8PathBuf, - base_dir: Utf8PathBuf, - dirs: Vec, - files: Vec, - target_canister_id: Principal, - agent: Agent, - proxy: Option, - identity_principal: Principal, - environment: String, - compute_limit_secs: u64, - stdio: Option>, -) -> Result, RunPluginError> +pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPluginError> ``` +`PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `dirs`, `files`, +`target_canister_id` (the canister being synced), `agent`, `proxy`, +`identity_principal`, `environment`, `compute_limit_secs`, and the exposed +`canister_ids` table, plus `stdio`. + `dirs` and `files` are the manifest-relative path strings, straight from the adapter. The runtime owns *all* filesystem access anchored at `base_dir`: it preopens each `dir` from `base_dir.join(dir)` and reads each `file` from @@ -188,7 +185,9 @@ pub struct Adapter { ### `crates/icp/src/canister/sync/plugin.rs` Resolves the wasm (local read or remote HTTP fetch into the package cache), -verifies sha256, then calls `icp_sync_plugin::run_plugin(...)`, forwarding the -manifest's `dirs`/`files` strings unchanged. The runtime — not the CLI — opens -those paths and enforces the path-safety checks, so the CLI no longer touches -the plugin's input files itself. +verifies sha256, builds the exposed canister ID table, then calls +`icp_sync_plugin::run_plugin(...)` with a `PluginInvocation`. The runtime — not +the CLI — opens the declared paths and enforces the path-safety checks, so the +CLI no longer touches the plugin's input files itself. `exposed_canister_ids` +adds a bare-local-name duplicate for every canister in the same subproject as +the one being synced. diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index dfffddc4c..84c5f4e41 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -2,5 +2,6 @@ mod path; mod runtime; pub use runtime::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, RunPluginError, run_plugin, + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, + run_plugin, }; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 27091fc34..2cf39ad3b 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1,4 +1,5 @@ // Host-side Component Model runtime for sync plugins. +use std::collections::BTreeMap; use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex as StdMutex; @@ -55,7 +56,7 @@ mod v1 { }); } -use v2::icp::sync_plugin::types::CallType; +use v2::icp::sync_plugin::types::{CallType, CanisterIdEntry}; // HostState holds everything the plugin's import functions need. struct HostState { @@ -328,20 +329,53 @@ fn detect_plugin_abi( } } -#[allow(clippy::too_many_arguments)] -pub fn run_plugin( - wasm_path: Utf8PathBuf, - base_dir: Utf8PathBuf, - dirs: Vec, - files: Vec, - target_canister_id: Principal, - agent: Agent, - proxy: Option, - identity_principal: Principal, - environment: String, - compute_limit_secs: u64, - stdio: Option>, -) -> Result, RunPluginError> { +/// Everything [`run_plugin`] needs to load and drive one sync plugin. +#[derive(Debug)] +pub struct PluginInvocation { + /// On-disk path to the plugin's wasm component. + pub wasm_path: Utf8PathBuf, + /// Directory the declared `dirs`/`files` are anchored at (the canister dir). + pub base_dir: Utf8PathBuf, + /// Manifest-relative directories to preopen read-only. + pub dirs: Vec, + /// Manifest-relative files to read and pass inline. + pub files: Vec, + /// The canister being synced. + pub target_canister_id: Principal, + /// Agent used for canister calls. + pub agent: Agent, + /// Proxy canister to route update calls through, if configured. + pub proxy: Option, + /// Signing identity principal, surfaced to the plugin. + pub identity_principal: Principal, + /// Name of the environment being synced. + pub environment: String, + /// Pure-wasm compute-time budget in seconds. + pub compute_limit_secs: u64, + /// The project's canister ID table for this environment, as exposed to the + /// plugin. Same-project canisters appear both under their fully-qualified + /// key and their bare local name (see the WIT `canister-id-entry` docs). + pub canister_ids: BTreeMap, + /// Channel for live rolling-view output, if any. + pub stdio: Option>, +} + +pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPluginError> { + let PluginInvocation { + wasm_path, + base_dir, + dirs, + files, + target_canister_id, + agent, + proxy, + identity_principal, + environment, + compute_limit_secs, + canister_ids, + stdio, + } = invocation; + let mut config = Config::new(); config.wasm_component_model(true); config.max_wasm_stack(MAX_WASM_STACK); @@ -485,6 +519,13 @@ pub fn run_plugin( .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, + canister_ids: canister_ids + .into_iter() + .map(|(name, id)| CanisterIdEntry { + name, + id: id.to_text(), + }) + .collect(), }; plugin.call_exec(&mut store, &input) } @@ -702,25 +743,34 @@ mod tests { Principal::anonymous() } + /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister + /// and identity, no proxy, an empty canister ID table, the default compute + /// limit, and the current directory as the base. Individual tests override + /// the few fields they care about. + fn invocation(wasm_path: &str, environment: &str) -> PluginInvocation { + PluginInvocation { + wasm_path: wasm_path.into(), + base_dir: ".".into(), + dirs: vec![], + files: vec![], + target_canister_id: anon(), + agent: dummy_agent(), + proxy: None, + identity_principal: anon(), + environment: environment.to_string(), + compute_limit_secs: DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, + canister_ids: BTreeMap::new(), + stdio: None, + } + } + // ------------------------------------------------------------------------- // Error-path tests — no fixture WASM needed // ------------------------------------------------------------------------- #[test] fn load_component_error_on_missing_file() { - let result = run_plugin( - "nonexistent.wasm".into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); + let result = run_plugin(invocation("nonexistent.wasm", "test")); assert!(matches!(result, Err(RunPluginError::LoadComponent { .. }))); } @@ -746,20 +796,12 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec!["nonexistent_dir".to_string()], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::PreopenDir { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.dirs = vec!["nonexistent_dir".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::PreopenDir { .. }) + )); } #[cfg(unix)] @@ -774,20 +816,13 @@ mod tests { std::fs::create_dir_all(base.join("real")).expect("create real dir"); symlink(base.join("real"), base.join("link")).expect("create symlink"); - let result = run_plugin( - wasm_path.into(), - base.to_path_buf(), - vec!["link".to_string()], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::SymlinkDir { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.base_dir = base.to_path_buf(); + inv.dirs = vec!["link".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::SymlinkDir { .. }) + )); } #[test] @@ -795,20 +830,12 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec!["nonexistent_file.txt".to_string()], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::ReadFile { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.files = vec!["nonexistent_file.txt".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::ReadFile { .. }) + )); } #[cfg(unix)] @@ -823,20 +850,13 @@ mod tests { std::fs::write(base.join("real.txt"), b"data").expect("write real file"); symlink(base.join("real.txt"), base.join("link.txt")).expect("create symlink"); - let result = run_plugin( - wasm_path.into(), - base.to_path_buf(), - vec![], - vec!["link.txt".to_string()], - anon(), - dummy_agent(), - None, - anon(), - "test".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(matches!(result, Err(RunPluginError::SymlinkFile { .. }))); + let mut inv = invocation(wasm_path, "test"); + inv.base_dir = base.to_path_buf(); + inv.files = vec!["link.txt".to_string()]; + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::SymlinkFile { .. }) + )); } #[test] @@ -844,20 +864,7 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "ok".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(result.is_ok()); + assert!(run_plugin(invocation(wasm_path, "ok")).is_ok()); } #[test] @@ -865,21 +872,8 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "error".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); assert!(matches!( - result, + run_plugin(invocation(wasm_path, "error")), Err(RunPluginError::PluginFailed { ref message }) if message == "deliberate failure" )); } @@ -891,20 +885,9 @@ mod tests { }; // The "spin" fixture busy-loops forever; a 1-second limit keeps the // test fast while still exercising the epoch-interruption trap. - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "spin".to_string(), - 1, - None, - ); - let err = result.expect_err("spinning plugin should hit the compute limit"); + let mut inv = invocation(wasm_path, "spin"); + inv.compute_limit_secs = 1; + let err = run_plugin(inv).expect_err("spinning plugin should hit the compute limit"); // The trap surfaces through the CallExec source chain, so walk it and // assert the message names both the limit and the override env var. let mut chain = err.to_string(); @@ -926,19 +909,9 @@ mod tests { }; let (tx, mut rx) = tokio::sync::mpsc::channel::(16); let result = tokio::task::block_in_place(|| { - run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "print".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - Some(tx), - ) + let mut inv = invocation(wasm_path, "print"); + inv.stdio = Some(tx); + run_plugin(inv) }); assert!(result.is_ok()); let msg = rx.try_recv().expect("expected stdout message on channel"); @@ -952,36 +925,10 @@ mod tests { let Some(wasm_path) = option_env!("TEST_PLUGIN_V1_WASM") else { return; }; - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "ok".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); - assert!(result.is_ok()); + assert!(run_plugin(invocation(wasm_path, "ok")).is_ok()); // Its error surface flows through the same machinery as v0.2.0 plugins. - let result = run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "error".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - None, - ); assert!(matches!( - result, + run_plugin(invocation(wasm_path, "error")), Err(RunPluginError::PluginFailed { ref message }) if message == "deliberate v1 failure" )); } @@ -993,19 +940,9 @@ mod tests { }; let (tx, mut rx) = tokio::sync::mpsc::channel::(16); let result = tokio::task::block_in_place(|| { - run_plugin( - wasm_path.into(), - ".".into(), - vec![], - vec![], - anon(), - dummy_agent(), - None, - anon(), - "hello".to_string(), - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, - Some(tx), - ) + let mut inv = invocation(wasm_path, "hello"); + inv.stdio = Some(tx); + run_plugin(inv) }); let lines = result.expect("plugin should succeed"); assert_eq!(lines, vec!["hello".to_string()]); diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 9e7ffb297..8212e5090 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -14,6 +14,24 @@ interface types { content: string, } + /// An entry in the project's canister ID mapping table: a canister name + /// and the textual principal it resolves to in the environment being synced. + record canister-id-entry { + /// The canister's fully-qualified project key: the subproject it belongs + /// to (a path relative to the app root) joined to its local name by a + /// single colon, e.g. "services/open-accounts:backend". A canister + /// defined directly in the project being synced has no subproject prefix + /// and appears as its bare local name, e.g. "backend". + /// + /// Every canister in the same subproject as the canister being synced is + /// additionally listed under its bare local name (a duplicate entry with + /// the same `id`), so a plugin can address a sibling by the same local + /// name the manifest uses. + name: string, + /// Textual principal the name resolves to for this environment. + id: string, + } + /// Input passed by the runtime to the plugin's exec() export. record sync-exec-input { /// Textual principal of the canister being synced. @@ -32,6 +50,12 @@ interface types { /// Textual principal of the proxy canister, if one was configured via /// `--proxy`. None when no proxy is in use. proxy-canister-id: option, + /// Name→principal mapping for every named canister in the project for + /// the environment being synced, sorted by name. Informational: the + /// plugin may use it to resolve canister names it knows about. Being + /// listed here does not grant permission to call a canister — that + /// still requires declaring it as a dependency (see `call-target`). + canister-ids: list, } /// A request to call a method on the target canister. @@ -59,7 +83,7 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, file-input}; + use types.{sync-exec-input, canister-call-request, canister-id-entry, file-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index 519a75b71..93dfdd0bd 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -19,6 +19,10 @@ use script::{HostScripts, ScriptInvocation, ScriptRunError, ScriptRunner}; pub struct Params { pub path: PathBuf, pub cid: Principal, + /// Fully-qualified store key of the canister being synced (e.g. `backend`, + /// or `services/open-crm:backend` for a dependency canister). Its namespace + /// prefix identifies which other canisters are in the same subproject. + pub name: String, /// Name of the environment being synced (e.g. "local", "production"). /// Passed to sync plugin steps via `SyncExecInput`. pub environment: String, @@ -165,6 +169,7 @@ mod tests { let params = Params { path: "/work/backend".into(), cid, + name: "backend".to_owned(), environment: "production".to_owned(), network: "ic".to_owned(), canister_ids: BTreeMap::from([( diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 97056d64d..49c636c35 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,8 +1,11 @@ +use std::collections::BTreeMap; + use camino::Utf8PathBuf; use candid::Principal; use ic_agent::Agent; use icp_sync_plugin::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, RunPluginError, run_plugin, + DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, + run_plugin, }; use snafu::prelude::*; use tokio::sync::mpsc::Sender; @@ -91,7 +94,10 @@ pub(super) async fn sync( let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); - // 3. Run the plugin (blocking call — signal Tokio that this thread will block). + // 3. Build the canister ID table exposed to the plugin. + let canister_ids = exposed_canister_ids(params); + + // 4. Run the plugin (blocking call — signal Tokio that this thread will block). let identity_principal = agent .get_principal() .map_err(|err| PluginError::GetIdentityPrincipal { err })?; @@ -101,23 +107,44 @@ pub(super) async fn sync( let stdio_clone = stdio.clone(); tokio::task::block_in_place(|| { - run_plugin( + run_plugin(PluginInvocation { wasm_path, base_dir, dirs, files, - params.cid, - agent_clone, + target_canister_id: params.cid, + agent: agent_clone, proxy, identity_principal, - environment_owned, + environment: environment_owned, compute_limit_secs, - stdio_clone, - ) + canister_ids, + stdio: stdio_clone, + }) }) .context(RunSnafu) } +/// The canister ID table exposed to a sync plugin: every named canister in the +/// project, plus — for canisters in the same subproject as the one being synced +/// — a duplicate entry under the bare local name. A store key is +/// `:` for a dependency canister and a bare local name for a +/// canister defined directly in the project (see the WIT `canister-id-entry` +/// docs), so the syncing canister's namespace is the prefix of its own key. +fn exposed_canister_ids(params: &Params) -> BTreeMap { + let syncing_namespace = params.name.split_once(':').map(|(namespace, _)| namespace); + + let mut table = params.canister_ids.clone(); + for (key, id) in ¶ms.canister_ids { + if let Some((namespace, local)) = key.split_once(':') + && Some(namespace) == syncing_namespace + { + table.entry(local.to_owned()).or_insert(*id); + } + } + table +} + #[cfg(test)] mod tests { use super::*; @@ -140,4 +167,64 @@ mod tests { ); } } + + fn principal(byte: u8) -> Principal { + Principal::from_slice(&[byte; 4]) + } + + fn params_named(name: &str, ids: &[(&str, Principal)]) -> Params { + Params { + path: "/work".into(), + cid: principal(0), + name: name.to_owned(), + environment: "demo".to_owned(), + network: "ic".to_owned(), + canister_ids: ids.iter().map(|(n, p)| ((*n).to_owned(), *p)).collect(), + proxy: None, + } + } + + /// Canisters sharing the syncing canister's subproject are additionally + /// exposed under their bare local name; canisters in other subprojects are + /// not. + #[test] + fn exposed_ids_add_bare_names_for_same_subproject() { + let backend = principal(1); + let frontend = principal(2); + let foreign = principal(3); + let params = params_named( + "services/open-accounts:backend", + &[ + ("services/open-accounts:backend", backend), + ("services/open-accounts:frontend", frontend), + ("services/open-crm:backend", foreign), + ], + ); + + let table = exposed_canister_ids(¶ms); + + // Same-subproject canisters gain a bare-local duplicate... + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + // ...while the fully-qualified keys are still present for everyone. + assert_eq!( + table.get("services/open-accounts:frontend"), + Some(&frontend) + ); + assert_eq!(table.get("services/open-crm:backend"), Some(&foreign)); + // The other subproject's canister is not reachable by a bare name; the + // bare "backend" belongs to the syncing canister's own subproject. + assert_eq!(table.get("backend"), Some(&backend)); + } + + /// A single-project layout keys canisters by bare local name already, so no + /// duplicates are added. + #[test] + fn exposed_ids_unchanged_without_a_subproject() { + let backend = principal(1); + let params = params_named("backend", &[("backend", backend)]); + let table = exposed_canister_ids(¶ms); + assert_eq!(table.len(), 1); + assert_eq!(table.get("backend"), Some(&backend)); + } } diff --git a/crates/icp/src/canister/sync/script.rs b/crates/icp/src/canister/sync/script.rs index 7c9d741d7..e26d73171 100644 --- a/crates/icp/src/canister/sync/script.rs +++ b/crates/icp/src/canister/sync/script.rs @@ -145,6 +145,7 @@ mod tests { Params { path: "/work/backend".into(), cid: principal(1), + name: "backend".to_owned(), environment: "production".to_owned(), network: "ic".to_owned(), canister_ids: canister_ids diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index f8310deea..30c1b0c89 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work against a single canister. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced, and lets it make canister calls and read declared files — nothing more. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work against a single canister. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -35,6 +35,7 @@ icp sync ├─ exec(sync-exec-input) called │ canister-id = │ identity-principal = + │ canister-ids = │ dirs / files = what you declared in the manifest │ └─ plugin makes canister-call(...) to the target canister (× N) @@ -68,6 +69,9 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `files` | The files you declared in `files:`, each as a `(name, content)` pair read by the host | | `identity-principal` | Textual principal of the signing identity used for canister calls | | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | +| `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | + +Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister in the project being synced, or a `subproject:canister` key for a dependency canister. Canisters in the same subproject as the one being synced are additionally listed under their bare local name. ### Calling the canister — `canister-call` From d815aec360768b73851a5b47de38cbd0efc9dc49 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 10:03:15 -0700 Subject: [PATCH 31/51] Document the actual canister-ids restriction The `canister-ids` table is informational; `canister-call` always targets the canister being synced. The field and rationale docs described a call-target / dependency-declaration permission mechanism that does not exist in this interface, which could mislead plugin authors into expecting they can call other canisters from the table. State the real restriction instead. --- crates/icp-sync-plugin/DESIGN.md | 5 +++-- crates/icp-sync-plugin/sync-plugin.wit | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 0f636dd0b..2a5787b6e 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -33,8 +33,9 @@ docs; the *reasons* behind those choices are recorded here. than a policy the plugin could bypass. - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve - canister names it knows about. It is informational only; calling still - requires a declaration. + canister names it knows about. It is informational only: `canister-call` + still targets the canister being synced, so the table grants no ability to + call other canisters. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 8212e5090..baa2d582c 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -53,8 +53,8 @@ interface types { /// Name→principal mapping for every named canister in the project for /// the environment being synced, sorted by name. Informational: the /// plugin may use it to resolve canister names it knows about. Being - /// listed here does not grant permission to call a canister — that - /// still requires declaring it as a dependency (see `call-target`). + /// listed here does not let the plugin call a canister — the + /// `canister-call` import always targets the canister being synced. canister-ids: list, } From e5d672a48a471439789105be9d141e5d143b13ce Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 11:39:46 -0700 Subject: [PATCH 32/51] copilot --- crates/icp-sync-plugin/sync-plugin.wit | 7 ++-- crates/icp/src/canister/sync/plugin.rs | 57 ++++++++++++++++++++++++-- docs/concepts/sync-plugins.md | 2 +- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index baa2d582c..e201568b5 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -20,13 +20,14 @@ interface types { /// The canister's fully-qualified project key: the subproject it belongs /// to (a path relative to the app root) joined to its local name by a /// single colon, e.g. "services/open-accounts:backend". A canister - /// defined directly in the project being synced has no subproject prefix - /// and appears as its bare local name, e.g. "backend". + /// defined directly in the app root has no subproject prefix and appears + /// as its bare local name, e.g. "backend". /// /// Every canister in the same subproject as the canister being synced is /// additionally listed under its bare local name (a duplicate entry with /// the same `id`), so a plugin can address a sibling by the same local - /// name the manifest uses. + /// name the manifest uses. A bare name always means the sibling, so an + /// app-root canister sharing that local name is not listed. name: string, /// Textual principal the name resolves to for this environment. id: string, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 49c636c35..22f749724 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -129,17 +129,22 @@ pub(super) async fn sync( /// project, plus — for canisters in the same subproject as the one being synced /// — a duplicate entry under the bare local name. A store key is /// `:` for a dependency canister and a bare local name for a -/// canister defined directly in the project (see the WIT `canister-id-entry` +/// canister defined directly in the app root (see the WIT `canister-id-entry` /// docs), so the syncing canister's namespace is the prefix of its own key. +/// +/// A local name never contains a colon but a subproject directory may, so keys +/// split on their *last* colon. The bare-name aliases take precedence over an +/// app-root canister of the same local name: a plugin resolving a bare name is +/// naming what the syncing canister's own manifest calls it. fn exposed_canister_ids(params: &Params) -> BTreeMap { - let syncing_namespace = params.name.split_once(':').map(|(namespace, _)| namespace); + let syncing_namespace = params.name.rsplit_once(':').map(|(namespace, _)| namespace); let mut table = params.canister_ids.clone(); for (key, id) in ¶ms.canister_ids { - if let Some((namespace, local)) = key.split_once(':') + if let Some((namespace, local)) = key.rsplit_once(':') && Some(namespace) == syncing_namespace { - table.entry(local.to_owned()).or_insert(*id); + table.insert(local.to_owned(), *id); } } table @@ -217,6 +222,50 @@ mod tests { assert_eq!(table.get("backend"), Some(&backend)); } + /// An app-root canister sharing a local name with a sibling of the syncing + /// canister does not keep the bare name: the syncing subproject's own + /// canister is what that name means to the plugin. + #[test] + fn exposed_ids_sibling_alias_overrides_the_app_root_name() { + let root_backend = principal(1); + let sibling_backend = principal(2); + let params = params_named( + "services/open-accounts:frontend", + &[ + ("backend", root_backend), + ("services/open-accounts:backend", sibling_backend), + ("services/open-accounts:frontend", principal(3)), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("backend"), Some(&sibling_backend)); + // The app-root canister's only key was that bare name, so it drops out + // of the table entirely rather than answering to a sibling's name. + assert!(!table.values().any(|id| *id == root_backend)); + } + + /// A subproject directory may itself contain a colon, so keys are split on + /// their last one — the same rule bundling uses. + #[test] + fn exposed_ids_split_subproject_prefix_at_the_last_colon() { + let backend = principal(1); + let frontend = principal(2); + let params = params_named( + "services/odd:name:backend", + &[ + ("services/odd:name:backend", backend), + ("services/odd:name:frontend", frontend), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("backend"), Some(&backend)); + assert_eq!(table.get("frontend"), Some(&frontend)); + } + /// A single-project layout keys canisters by bare local name already, so no /// duplicates are added. #[test] diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 30c1b0c89..a7a6eb3f9 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -71,7 +71,7 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | | `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | -Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister in the project being synced, or a `subproject:canister` key for a dependency canister. Canisters in the same subproject as the one being synced are additionally listed under their bare local name. +Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister that came from a dependency. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. ### Calling the canister — `canister-call` From 128154985e3e946c123a35b8c31f72a23be352a3 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 11:13:42 -0700 Subject: [PATCH 33/51] Fix docs --- crates/icp-sync-plugin/src/runtime.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 2cf39ad3b..50406b383 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -39,9 +39,8 @@ use wasmtime_wasi::{DirPerms, FilePerms}; // module so their generated type names don't collide. `run_plugin` reads the // interface version from the component's own metadata (see `detect_plugin_abi`) // and drives it through the matching module, so plugins built against either -// interface load. The two interfaces are currently structurally identical; the -// split exists so later breaking changes to the current interface can land -// without dropping support for already-built plugins. +// interface load. The split exists so breaking changes to the current interface +// can land without dropping support for already-built plugins. mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", From a6f6d6637498342abf16702a71c4c0561858e47e Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 04:52:18 -0700 Subject: [PATCH 34/51] Implement cross-canister plugin targeting --- crates/icp-cli/src/operations/bundle.rs | 1 + crates/icp-sync-plugin/DESIGN.md | 49 ++++-- crates/icp-sync-plugin/src/lib.rs | 4 +- crates/icp-sync-plugin/src/runtime.rs | 186 ++++++++++++++++++--- crates/icp-sync-plugin/sync-plugin.wit | 35 +++- crates/icp/src/canister/sync/plugin.rs | 96 ++++++++++- crates/icp/src/manifest/adapter/plugin.rs | 51 ++++++ crates/icp/src/manifest/canister.rs | 2 + docs/concepts/sync-plugins.md | 19 ++- docs/guides/writing-sync-plugins.md | 3 +- docs/reference/configuration.md | 8 +- docs/schemas/canister-yaml-schema.json | 10 ++ docs/schemas/icp-yaml-schema.json | 10 ++ examples/icp-sync-plugin/plugin/src/lib.rs | 2 + 14 files changed, 411 insertions(+), 65 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 6786fc7d5..4b07c5f5d 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -788,6 +788,7 @@ async fn prepare_plugin_step( sha256: Some(plugin_sha256), dirs: bundle_dirs, files: bundle_files, + canisters: None, })) } diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 2a5787b6e..40646741f 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -27,10 +27,13 @@ docs; the *reasons* behind those choices are recorded here. - **Raw Candid bytes at the boundary** — `canister-call-request.arg` is `list`. The plugin owns Candid encoding/decoding; the host forwards bytes unchanged. This keeps the host free of any per-canister type knowledge. -- **`canister-call` takes no canister ID** — the host always calls the canister - from `sync-exec-input.canister-id`. There is deliberately no field for a - different target, so the single-canister restriction is *structural* rather - than a policy the plugin could bypass. +- **`canister-call` takes an explicit `target`** — the plugin selects the + canister being synced (`host`) or a canister it declared as a dependency, by + name or principal. The host resolves the target and *enforces* the + declaration: a target absent from the step's `canisters:` list is rejected + without a call. (In the earlier `@0.1.0` interface `canister-call` had no + target and always reached the canister being synced; see *Interface + versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only: `canister-call` @@ -71,9 +74,12 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin ``` `PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `dirs`, `files`, -`target_canister_id` (the canister being synced), `agent`, `proxy`, -`identity_principal`, `environment`, `compute_limit_secs`, and the exposed -`canister_ids` table, plus `stdio`. +`host_canister_id` (the canister being synced), `agent`, `proxy`, +`identity_principal`, `environment`, `compute_limit_secs`, the exposed +`canister_ids` table, the `callable: CallableCanisters` enforcement set, and +`stdio`. The CLI resolves the manifest's declared `canisters:` into +`CallableCanisters` before calling; this crate stays free of any manifest +knowledge. `dirs` and `files` are the manifest-relative path strings, straight from the adapter. The runtime owns *all* filesystem access anchored at `base_dir`: it @@ -112,7 +118,8 @@ mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugi mod v1 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugin-v1.wit" }); } struct HostState { - target_canister_id: Principal, + host_canister_id: Principal, + callable: CallableCanisters, // by_name + by_id, from the manifest agent: Arc, proxy: Option, wasi_ctx: wasmtime_wasi::WasiCtx, @@ -121,16 +128,18 @@ struct HostState { } // Implemented for both v1::SyncPluginImports and v2::SyncPluginImports; both -// delegate to one shared `do_canister_call(...)`. +// delegate to one shared `do_canister_call(target, ...)`. ``` `HostState` implements `WasiView` so wasmtime_wasi can access the WASI context. `canister_call` uses `tokio::runtime::Handle::current().block_on(...)` because the caller already wraps the synchronous `run_plugin` in -`tokio::task::block_in_place`. Both interface versions call the canister being -synced. When a proxy is configured and the call is a non-`direct` update, it is -encoded as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise -it goes straight to the target via `ic-agent`. +`tokio::task::block_in_place`. For a v0.2.0 plugin the target is resolved from +the request's `call-target` by `resolve_call_target`, which enforces the +`callable` set; for a v0.1.0 plugin the target is always `host_canister_id`. +When a proxy is configured and the call is a non-`direct` update, it is encoded +as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise it goes +straight to the resolved target via `ic-agent`. ### Interface versioning (parallel v0.1.0 / v0.2.0 support) @@ -174,21 +183,27 @@ Deserializes the `canister.yaml` fields into: ```rust pub struct Adapter { - pub source: SourceField, // path: or url: + pub source: SourceField, // path: or url: pub sha256: Option, pub dirs: Option>, pub files: Option>, + pub canisters: Option>, // extra callable canisters } ``` -`Deserialize` is hand-written to reject a `url` source without a `sha256`. +`CanisterRef` is an untagged `Principal | Name` (anything that parses as a +principal is one; everything else is a name), written in the manifest as a plain +string. `Deserialize` is hand-written to reject a `url` source without a +`sha256`. ### `crates/icp/src/canister/sync/plugin.rs` Resolves the wasm (local read or remote HTTP fetch into the package cache), -verifies sha256, builds the exposed canister ID table, then calls +verifies sha256, builds the exposed canister ID table and the `CallableCanisters` +enforcement set (resolving `canisters:` against the project's IDs), then calls `icp_sync_plugin::run_plugin(...)` with a `PluginInvocation`. The runtime — not the CLI — opens the declared paths and enforces the path-safety checks, so the CLI no longer touches the plugin's input files itself. `exposed_canister_ids` adds a bare-local-name duplicate for every canister in the same subproject as -the one being synced. +the one being synced; `resolve_callable` fails the step if a declared dependency +name does not resolve. diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index 84c5f4e41..053be28a3 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -2,6 +2,6 @@ mod path; mod runtime; pub use runtime::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, - run_plugin, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + PluginInvocation, RunPluginError, run_plugin, }; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 50406b383..f45ece92c 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1,5 +1,5 @@ // Host-side Component Model runtime for sync plugins. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex as StdMutex; @@ -39,8 +39,7 @@ use wasmtime_wasi::{DirPerms, FilePerms}; // module so their generated type names don't collide. `run_plugin` reads the // interface version from the component's own metadata (see `detect_plugin_abi`) // and drives it through the matching module, so plugins built against either -// interface load. The split exists so breaking changes to the current interface -// can land without dropping support for already-built plugins. +// interface load. mod v2 { wasmtime::component::bindgen!({ world: "sync-plugin", @@ -55,11 +54,62 @@ mod v1 { }); } -use v2::icp::sync_plugin::types::{CallType, CanisterIdEntry}; +use v2::icp::sync_plugin::types::{CallTarget, CallType, CanisterIdEntry}; + +/// The canisters a sync plugin is permitted to call, beyond the canister being +/// synced (which is always reachable via [`CallTarget::Host`]). +/// +/// Built by the CLI from the plugin step's declared `canisters` dependencies, +/// resolved against the project's canister ID table. Keeping the resolution on +/// the CLI side keeps this runtime crate free of any manifest knowledge. +#[derive(Clone, Debug, Default)] +pub struct CallableCanisters { + /// Dependencies callable by name ([`CallTarget::Name`]). Maps the name — as + /// it appears in the canister ID table — to the principal it resolves to. + pub by_name: BTreeMap, + /// Every principal callable by [`CallTarget::Id`]. Includes the principals + /// of the `by_name` entries, so an author may target the same canister + /// either way. + pub by_id: BTreeSet, +} + +/// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing +/// that the plugin declared it as a dependency. The canister being synced +/// (`host`) is always permitted. +fn resolve_call_target( + target: &CallTarget, + host_canister_id: Principal, + callable: &CallableCanisters, +) -> Result { + match target { + CallTarget::Host => Ok(host_canister_id), + CallTarget::Name(name) => callable.by_name.get(name).copied().ok_or_else(|| { + format!( + "plugin is not permitted to call canister '{name}': declare it in the sync step's \ + `canisters` list to allow it" + ) + }), + CallTarget::Id(text) => { + let principal = Principal::from_text(text) + .map_err(|e| format!("invalid target principal '{text}': {e}"))?; + if principal == host_canister_id || callable.by_id.contains(&principal) { + Ok(principal) + } else { + Err(format!( + "plugin is not permitted to call canister '{principal}': declare it in the \ + sync step's `canisters` list to allow it" + )) + } + } + } +} // HostState holds everything the plugin's import functions need. struct HostState { - target_canister_id: Principal, + /// The canister being synced — the target of [`CallTarget::Host`] calls. + host_canister_id: Principal, + /// Canisters the plugin declared as dependencies and may also call. + callable: CallableCanisters, agent: Arc, /// Proxy canister to route update calls through, if configured. proxy: Option, @@ -84,10 +134,12 @@ impl wasmtime_wasi::WasiView for HostState { } impl HostState { - /// Perform a canister call to the canister being synced. Shared by both - /// interface versions. + /// Perform a canister call to an already-resolved target principal. Shared + /// by both interface versions: the v0.1.0 import always passes the canister + /// being synced; the v0.2.0 import passes the resolved `call-target`. fn do_canister_call( &mut self, + target: Principal, method: String, arg_bytes: Vec, call_type: CallType, @@ -96,7 +148,6 @@ impl HostState { ) -> Result, String> { use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; - let cid = self.target_canister_id; let agent = Arc::clone(&self.agent); let proxy = if direct { None } else { self.proxy }; @@ -108,7 +159,7 @@ impl HostState { CallType::Update => { if let Some(proxy_cid) = proxy { let proxy_args = ProxyArgs { - canister_id: cid, + canister_id: target, method: method.clone(), args: arg_bytes, cycles: candid::Nat::from(cycles), @@ -128,14 +179,14 @@ impl HostState { } } else { agent - .update(&cid, &method) + .update(&target, &method) .with_arg(arg_bytes) .await .map_err(|e| format!("canister call failed: {e}")) } } CallType::Query => agent - .query(&cid, &method) + .query(&target, &method) .with_arg(arg_bytes) .call() .await @@ -151,7 +202,7 @@ impl HostState { } } -// -- v0.2.0 interface. --------------------------------------------------------- +// -- v0.2.0 interface: the plugin chooses the target via `call-target`. -------- // `types::Host` is an empty marker trait generated for the `types` interface. impl v2::icp::sync_plugin::types::Host for HostState {} @@ -161,11 +212,19 @@ impl v2::SyncPluginImports for HostState { &mut self, req: v2::icp::sync_plugin::types::CanisterCallRequest, ) -> Result, String> { - self.do_canister_call(req.method, req.arg, req.call_type, req.direct, req.cycles) + let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; + self.do_canister_call( + target, + req.method, + req.arg, + req.call_type, + req.direct, + req.cycles, + ) } } -// -- v0.1.0 interface. --------------------------------------------------------- +// -- v0.1.0 interface: calls always go to the canister being synced. ----------- impl v1::icp::sync_plugin::types::Host for HostState {} @@ -174,12 +233,16 @@ impl v1::SyncPluginImports for HostState { &mut self, req: v1::icp::sync_plugin::types::CanisterCallRequest, ) -> Result, String> { + // The legacy interface has no target field; always call the host canister. + let target = self.host_canister_id; // v1's `call-type` is a distinct generated enum; map it to the shared one. let call_type = match req.call_type { v1::icp::sync_plugin::types::CallType::Update => CallType::Update, v1::icp::sync_plugin::types::CallType::Query => CallType::Query, }; - self.do_canister_call(req.method, req.arg, call_type, req.direct, req.cycles) + self.do_canister_call( + target, req.method, req.arg, call_type, req.direct, req.cycles, + ) } } @@ -265,9 +328,11 @@ pub enum RunPluginError { /// Which version of the sync-plugin interface a component was built against. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum PluginAbi { - /// Current interface (`icp:sync-plugin@0.2.x`). + /// Current interface (`icp:sync-plugin@0.2.x`): `canister-call` chooses a + /// target and `sync-exec-input` carries the canister ID table. V2, - /// Legacy interface (`icp:sync-plugin@0.1.x`). + /// Legacy interface (`icp:sync-plugin@0.1.x`): calls always reach the + /// canister being synced. V1, } @@ -339,8 +404,8 @@ pub struct PluginInvocation { pub dirs: Vec, /// Manifest-relative files to read and pass inline. pub files: Vec, - /// The canister being synced. - pub target_canister_id: Principal, + /// The canister being synced. Reachable via `call-target::host`. + pub host_canister_id: Principal, /// Agent used for canister calls. pub agent: Agent, /// Proxy canister to route update calls through, if configured. @@ -355,6 +420,10 @@ pub struct PluginInvocation { /// plugin. Same-project canisters appear both under their fully-qualified /// key and their bare local name (see the WIT `canister-id-entry` docs). pub canister_ids: BTreeMap, + /// Canisters the plugin declared as dependencies and may call, beyond the + /// canister being synced. Ignored by v0.1.0 plugins, which can only reach + /// the canister being synced. + pub callable: CallableCanisters, /// Channel for live rolling-view output, if any. pub stdio: Option>, } @@ -365,13 +434,14 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin base_dir, dirs, files, - target_canister_id, + host_canister_id, agent, proxy, identity_principal, environment, compute_limit_secs, canister_ids, + callable, stdio, } = invocation; @@ -463,7 +533,8 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let epoch_extension = Arc::new(AtomicU64::new(0)); let host_state = HostState { - target_canister_id, + host_canister_id, + callable, agent: Arc::new(agent), proxy, wasi_ctx: wasi_builder.build(), @@ -485,13 +556,15 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin } }); - let canister_id_text = target_canister_id.to_text(); + let canister_id_text = host_canister_id.to_text(); let identity_text = identity_principal.to_text(); let proxy_text = proxy.map(|p| p.to_text()); // Which interface the plugin was built against is read from the component's // own declared metadata (see `detect_plugin_abi`) rather than probed by - // trial instantiation, then driven through the matching bindgen world. + // trial instantiation. Both are served in parallel: v0.2.0 plugins choose a + // call target and receive the canister ID table; v0.1.0 plugins get neither + // and always call the canister being synced. let call_result = match detect_plugin_abi(&engine, &component, &wasm_path)? { PluginAbi::V2 => { let mut linker: Linker = Linker::new(&engine); @@ -743,7 +816,7 @@ mod tests { } /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister - /// and identity, no proxy, an empty canister ID table, the default compute + /// and identity, no proxy, no declared dependencies, the default compute /// limit, and the current directory as the base. Individual tests override /// the few fields they care about. fn invocation(wasm_path: &str, environment: &str) -> PluginInvocation { @@ -752,17 +825,80 @@ mod tests { base_dir: ".".into(), dirs: vec![], files: vec![], - target_canister_id: anon(), + host_canister_id: anon(), agent: dummy_agent(), proxy: None, identity_principal: anon(), environment: environment.to_string(), compute_limit_secs: DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, canister_ids: BTreeMap::new(), + callable: CallableCanisters::default(), stdio: None, } } + // ------------------------------------------------------------------------- + // Call-target resolution (enforcement) — pure logic, no fixture WASM needed + // ------------------------------------------------------------------------- + + #[test] + fn resolve_target_host_is_always_allowed() { + let host = Principal::from_slice(&[1; 4]); + let callable = CallableCanisters::default(); + assert_eq!( + resolve_call_target(&CallTarget::Host, host, &callable).unwrap(), + host + ); + } + + #[test] + fn resolve_target_name_requires_declaration() { + let host = Principal::from_slice(&[1; 4]); + let dep = Principal::from_slice(&[2; 4]); + let callable = CallableCanisters { + by_name: BTreeMap::from([("backend".to_string(), dep)]), + by_id: BTreeSet::from([dep]), + }; + assert_eq!( + resolve_call_target(&CallTarget::Name("backend".into()), host, &callable).unwrap(), + dep + ); + let err = resolve_call_target(&CallTarget::Name("frontend".into()), host, &callable) + .expect_err("undeclared name must be rejected"); + assert!( + err.contains("not permitted") && err.contains("frontend"), + "got: {err}" + ); + } + + #[test] + fn resolve_target_id_allows_host_and_declared_only() { + let host = Principal::from_slice(&[1; 4]); + let dep = Principal::from_slice(&[2; 4]); + let other = Principal::from_slice(&[3; 4]); + let callable = CallableCanisters { + by_name: BTreeMap::new(), + by_id: BTreeSet::from([dep]), + }; + // A declared principal is allowed; so is the host, implicitly. + assert_eq!( + resolve_call_target(&CallTarget::Id(dep.to_text()), host, &callable).unwrap(), + dep + ); + assert_eq!( + resolve_call_target(&CallTarget::Id(host.to_text()), host, &callable).unwrap(), + host + ); + // An undeclared principal is rejected. + let err = resolve_call_target(&CallTarget::Id(other.to_text()), host, &callable) + .expect_err("undeclared principal must be rejected"); + assert!(err.contains("not permitted"), "got: {err}"); + // Garbage text is a distinct, clearer error. + let err = resolve_call_target(&CallTarget::Id("not a principal".into()), host, &callable) + .expect_err("invalid principal text must be rejected"); + assert!(err.contains("invalid target principal"), "got: {err}"); + } + // ------------------------------------------------------------------------- // Error-path tests — no fixture WASM needed // ------------------------------------------------------------------------- diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index e201568b5..bd7b13d67 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -33,6 +33,25 @@ interface types { id: string, } + /// Which canister a `canister-call-request` targets. + /// + /// A plugin may target the canister being synced, or any canister it + /// declared as a dependency in the sync step's `canisters` list — by that + /// canister's name or by its textual principal. Targeting a canister that + /// was not declared as a dependency is rejected by the host. + variant call-target { + /// The canister being synced (`sync-exec-input.canister-id`). Always + /// permitted, whether or not it also appears in `canisters`. + host, + /// A declared-dependency canister identified by name, spelled exactly as + /// it appears in `sync-exec-input.canister-ids` — a bare local name for a + /// canister in the same subproject, or a `subproject:local` key + /// otherwise. The host resolves it against that mapping table. + name(string), + /// A declared-dependency canister identified by its textual principal. + id(string), + } + /// Input passed by the runtime to the plugin's exec() export. record sync-exec-input { /// Textual principal of the canister being synced. @@ -59,8 +78,12 @@ interface types { canister-ids: list, } - /// A request to call a method on the target canister. + /// A request to call a method on a canister. record canister-call-request { + /// Which canister to call. `host` targets the canister being synced; + /// `name`/`id` target a canister declared as a dependency in the sync + /// step's `canisters` list. + target: call-target, /// The canister method to call. method: string, /// Candid-encoded argument bytes. The plugin is responsible for @@ -84,15 +107,17 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, canister-id-entry, file-input}; + use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin // ------------------------------------------------------------------------- - /// Make an update or query call to the canister being synced. - /// The host always calls the canister from sync-exec-input.canister-id; - /// the plugin does not choose the target. + /// Make an update or query call to a canister. + /// The `req.target` selects the canister: the one being synced (`host`), or + /// a canister declared as a dependency in the sync step's `canisters` list, + /// by name or principal. A target that was not declared as a dependency is + /// rejected without making a call. /// Returns the raw Candid-encoded response bytes on success or an error /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 22f749724..ad50d1d0e 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,16 +1,20 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use camino::Utf8PathBuf; use candid::Principal; use ic_agent::Agent; use icp_sync_plugin::{ - DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, - run_plugin, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + PluginInvocation, RunPluginError, run_plugin, }; use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{canister::wasm, manifest::adapter::plugin::Adapter, package::PackageCache}; +use crate::{ + canister::wasm, + manifest::adapter::plugin::{Adapter, CanisterRef}, + package::PackageCache, +}; use super::Params; @@ -29,6 +33,12 @@ pub enum PluginError { #[snafu(display("failed to run plugin"))] Run { source: RunPluginError }, + + #[snafu(display( + "sync plugin declares a dependency on canister '{name}', but no canister by that name \ + is known in environment '{environment}'" + ))] + UnknownDependency { name: String, environment: String }, } /// Resolve the plugin compute-time limit, honoring the @@ -94,8 +104,10 @@ pub(super) async fn sync( let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); - // 3. Build the canister ID table exposed to the plugin. + // 3. Build the canister ID table exposed to the plugin, then resolve the + // plugin's declared callable canisters against it. let canister_ids = exposed_canister_ids(params); + let callable = resolve_callable(adapter, &canister_ids, environment)?; // 4. Run the plugin (blocking call — signal Tokio that this thread will block). let identity_principal = agent @@ -112,13 +124,14 @@ pub(super) async fn sync( base_dir, dirs, files, - target_canister_id: params.cid, + host_canister_id: params.cid, agent: agent_clone, proxy, identity_principal, environment: environment_owned, compute_limit_secs, canister_ids, + callable, stdio: stdio_clone, }) }) @@ -150,6 +163,38 @@ fn exposed_canister_ids(params: &Params) -> BTreeMap { table } +/// Resolve the canisters a plugin declared it may call into a [`CallableCanisters`] +/// enforcement set. Named dependencies are looked up in `canister_ids`; a name +/// that does not resolve is a manifest error. +fn resolve_callable( + adapter: &Adapter, + canister_ids: &BTreeMap, + environment: &str, +) -> Result { + let mut by_name = BTreeMap::new(); + let mut by_id = BTreeSet::new(); + for canister in adapter.canisters.iter().flatten() { + match canister { + CanisterRef::Principal(principal) => { + by_id.insert(*principal); + } + CanisterRef::Name(name) => { + let principal = + canister_ids + .get(name) + .copied() + .context(UnknownDependencySnafu { + name: name.clone(), + environment: environment.to_owned(), + })?; + by_name.insert(name.clone(), principal); + by_id.insert(principal); + } + } + } + Ok(CallableCanisters { by_name, by_id }) +} + #[cfg(test)] mod tests { use super::*; @@ -173,6 +218,8 @@ mod tests { } } + use crate::manifest::adapter::prebuilt::{LocalSource, SourceField}; + fn principal(byte: u8) -> Principal { Principal::from_slice(&[byte; 4]) } @@ -189,6 +236,18 @@ mod tests { } } + fn adapter_with(canisters: Option>) -> Adapter { + Adapter { + source: SourceField::Local(LocalSource { + path: "plugin.wasm".into(), + }), + sha256: None, + dirs: None, + files: None, + canisters, + } + } + /// Canisters sharing the syncing canister's subproject are additionally /// exposed under their bare local name; canisters in other subprojects are /// not. @@ -276,4 +335,29 @@ mod tests { assert_eq!(table.len(), 1); assert_eq!(table.get("backend"), Some(&backend)); } + + #[test] + fn resolve_callable_resolves_names_and_principals() { + let dep = principal(1); + let raw = principal(2); + let table = BTreeMap::from([("backend".to_owned(), dep)]); + let adapter = adapter_with(Some(vec![ + CanisterRef::Name("backend".to_owned()), + CanisterRef::Principal(raw), + ])); + + let callable = resolve_callable(&adapter, &table, "demo").unwrap(); + + assert_eq!(callable.by_name.get("backend"), Some(&dep)); + assert!(callable.by_id.contains(&dep)); + assert!(callable.by_id.contains(&raw)); + } + + #[test] + fn resolve_callable_rejects_unknown_name() { + let adapter = adapter_with(Some(vec![CanisterRef::Name("nope".to_owned())])); + let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") + .expect_err("an undeclared name must fail"); + assert!(matches!(err, PluginError::UnknownDependency { .. })); + } } diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 915b03c1d..5aef5ccfa 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,8 +1,24 @@ +use candid::Principal; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; use super::prebuilt::SourceField; +/// A canister a sync plugin is permitted to call, beyond the canister being +/// synced. Written in the manifest either as a textual principal (e.g. +/// `aaaaa-aa`) or as a canister name resolved against the project's canister ID +/// table for the environment being synced (e.g. `backend`, or a namespaced +/// dependency canister such as `services/open-crm:backend`). Anything that +/// parses as a principal is taken as one; everything else is a name. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum CanisterRef { + /// An explicit principal (e.g. `aaaaa-aa`). + Principal(Principal), + /// A canister name from this project's ID table (e.g. `backend`). + Name(String), +} + /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -45,6 +61,14 @@ pub struct Adapter { /// Files (relative to canister directory) the host reads and passes to /// the plugin as part of `sync-exec-input.files`. pub files: Option>, + + /// Canisters this plugin may call in addition to the canister being synced. + /// Each entry is a canister name (resolved against the project's canister ID + /// table) or a textual principal. The plugin picks a target per call via the + /// `call-target` in its `canister-call` request; a target not listed here is + /// rejected by the host. + #[schemars(with = "Option>")] + pub canisters: Option>, } impl<'de> Deserialize<'de> for Adapter { @@ -56,6 +80,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, + canisters: Option>, } let h = AdapterHelper::deserialize(d)?; @@ -69,6 +94,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: h.sha256, dirs: h.dirs, files: h.files, + canisters: h.canisters, }) } } @@ -94,6 +120,7 @@ mod tests { sha256: None, dirs: None, files: None, + canisters: None, }, ); } @@ -120,6 +147,7 @@ mod tests { sha256: Some("abc123".to_string()), dirs: Some(vec!["assets/seed-data".to_string(), "config".to_string()]), files: Some(vec!["config.txt".to_string()]), + canisters: None, }, ); } @@ -139,6 +167,28 @@ mod tests { ); } + #[test] + fn canisters_parse_as_names_and_principals() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + canisters: + - backend + - services/open-crm:backend + - aaaaa-aa + "#, + ) + .expect("failed to deserialize Adapter with canisters"); + assert_eq!( + adapter.canisters, + Some(vec![ + CanisterRef::Name("backend".to_string()), + CanisterRef::Name("services/open-crm:backend".to_string()), + CanisterRef::Principal(Principal::from_text("aaaaa-aa").unwrap()), + ]), + ); + } + #[test] fn remote_url_with_sha256() { assert_eq!( @@ -156,6 +206,7 @@ mod tests { sha256: Some("a665a45920422f9d417e".to_string()), dirs: None, files: None, + canisters: None, }, ); } diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index b032be7e5..8d2838a5b 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -793,6 +793,7 @@ mod tests { sha256: None, dirs: Some(vec!["assets/seed-data/".to_string()]), files: None, + canisters: None, } )] }), @@ -837,6 +838,7 @@ mod tests { ), dirs: None, files: None, + canisters: None, })] }), }, diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index a7a6eb3f9..aca070959 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work against a single canister. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it declares as dependencies. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -15,7 +15,7 @@ Sync plugins fill that gap. A plugin is: - **Portable** — written in any language that compiles to `wasm32-wasip2`, distributed as one `.wasm` file (local path or remote URL + `sha256`). - **Sandboxed** — it cannot open network sockets, spawn subprocesses, or touch the filesystem outside the directories you explicitly grant it. -- **Scoped to one canister** — it can call update and query methods, but only on the canister being synced. The target is fixed by the host; the plugin cannot choose a different one. +- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister it declares as a dependency in the manifest's `canisters:` list. A call to a canister that was not declared is rejected by the host. The most common way to get a sync plugin is through a [recipe](recipes.md). For example, the `@dfinity/asset-canister` recipe emits a `plugin` sync step (starting with `v2.2.1`) that uploads your built static files to the asset canister — so for everyday frontend deployment you never write a plugin yourself. @@ -38,7 +38,9 @@ icp sync │ canister-ids = │ dirs / files = what you declared in the manifest │ - └─ plugin makes canister-call(...) to the target canister (× N) + └─ plugin makes canister-call({ target, ... }) (× N) + target = host (the canister being synced), or a + declared-dependency canister by name or principal ``` ## The Plugin Interface @@ -47,7 +49,7 @@ The interface is defined as a [WIT](https://component-model.bytecodealliance.org ```wit world sync-plugin { - // Host import: call the canister being synced. + // Host import: call the canister being synced or a declared dependency. import canister-call: func(req: canister-call-request) -> result, string>; // Plugin export: run the sync step. @@ -73,19 +75,20 @@ The authoritative interface, including all record fields, lives in [`sync-plugin Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister that came from a dependency. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. -### Calling the canister — `canister-call` +### Calling a canister — `canister-call` -The plugin calls methods on the target canister through the `canister-call` import. It supplies the method name, **Candid-encoded argument bytes** (the host forwards them unchanged), and a few routing options: +The plugin calls methods through the `canister-call` import. It picks a `target`, supplies the method name, **Candid-encoded argument bytes** (the host forwards them unchanged), and a few routing options: | Request field | Meaning | |---------------|---------| +| `target` | Which canister to call: `host` (the canister being synced), or a canister declared in `canisters:` addressed by `name` or by `id` (principal) | | `method` | The canister method to call | | `arg` | Candid-encoded argument bytes (the plugin encodes; the host forwards as-is) | | `call-type` | `update` or `query` | | `direct` | When `false` (default), update calls are routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, the call always goes directly to the target. Query calls always go directly regardless. | | `cycles` | Cycles to attach to a proxied update call; only meaningful when `direct` is `false`, a proxy is configured, and `call-type` is `update` | -The host always calls the canister named in `sync-exec-input.canister-id`. There is no field for a different canister ID — the single-canister restriction is structural, not a policy the plugin can opt out of. +The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name`/`id` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. ### Logging — stdout and stderr @@ -114,7 +117,7 @@ The plugin runs with a deliberately narrow capability surface. | Read declared `dirs:` | yes | read-only preopens | | Clocks, RNG, `wasi:io` | yes | Rust's `HashMap`, `chrono`, etc. work normally | | `process::exit` / panics | yes | abort the guest cleanly; the host surfaces the error | -| Canister calls | yes | only to the canister being synced | +| Canister calls | yes | to the canister being synced, and to canisters declared in `canisters:` | | Environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | | Network sockets / DNS | blocked | treat the network as unavailable | | Filesystem writes | blocked | no writable preopens | diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 979a43081..0c3004752 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -65,6 +65,7 @@ impl Guest for Plugin { // Call a method on the canister being synced. canister_call(&CanisterCallRequest { + target: CallTarget::Host, // the canister being synced method: "set_uploader".to_string(), arg, call_type: icp::sync_plugin::types::CallType::Update, @@ -84,7 +85,7 @@ export!(Plugin); A few things to note: - **You encode the arguments.** `arg` is raw Candid bytes. Encode with `candid::Encode!`; decode any response (`Vec`) with `candid::Decode!`. -- **The target is fixed.** `canister_call` always reaches the canister in `input.canister_id` — there is no field to target another canister. +- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` or `CallTarget::Id(principal_text)` — a name matches the entries in `input.canister_ids`. The host rejects a target you did not declare. - **`direct` and `cycles` control proxy routing.** With `direct: false`, update calls go through the [proxy canister](proxy-canister.md) when one is configured, and `cycles` can fund the forwarded call. With `direct: true`, the call always goes straight to the target. See [The Plugin Interface](../concepts/sync-plugins.md#the-plugin-interface) for the full semantics. ## Read Declared Files and Directories diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 219eff450..7e594facd 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -151,6 +151,9 @@ sync: - config files: # files read by the host and passed inline - config.txt + canisters: # extra canisters the plugin may call + - ledger # by name (resolved for the environment) + - ryjl3-tyaaa-aaaaa-aaaba-cai # or by principal # Remote plugin (downloaded and verified before execution) - type: plugin @@ -165,10 +168,13 @@ sync: | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | | `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | | `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | +| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name (resolved against the project's canister IDs for the environment) or a textual principal | Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. -The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. +A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a dependency canister. A name that does not resolve to a known canister for the environment fails the sync step. + +The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. ## Recipes diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 7559c5cb2..4dcd4d0b8 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -69,6 +69,16 @@ ], "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { + "canisters": { + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "dirs": { "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", "items": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index b83a4e7a9..1c4758c8e 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -69,6 +69,16 @@ ], "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { + "canisters": { + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, "dirs": { "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", "items": { diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index 6f8d25508..f6d7534bd 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -24,6 +24,7 @@ impl Guest for Plugin { .map_err(|e| format!("invalid identity principal: {e}"))?; let arg = Encode!(&uploader).map_err(|e| format!("encode set_uploader arg: {e}"))?; canister_call(&CanisterCallRequest { + target: CallTarget::Host, method: "set_uploader".to_string(), arg, call_type: icp::sync_plugin::types::CallType::Update, @@ -68,6 +69,7 @@ fn register_dir(dir: &Path) -> Result { let arg = Encode!(&path_str, &content_trimmed) .map_err(|e| format!("encode register arg: {e}"))?; canister_call(&CanisterCallRequest { + target: CallTarget::Host, method: "register".to_string(), arg, call_type: icp::sync_plugin::types::CallType::Update, From 077bc213e8477147e8eb705a1fd9f1be144b9330 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 10:14:38 -0700 Subject: [PATCH 35/51] Document canister-ids call-target permission on the targeting interface With cross-canister targeting present, the `canister-ids` table's field doc and the DESIGN rationale should describe the real permission model: the table is informational, and calling a listed canister requires declaring it as a dependency (`call-target`). The mappings-branch wording ("canister-call always targets the canister being synced") was correct only before this interface added targeting. --- crates/icp-sync-plugin/DESIGN.md | 5 ++--- crates/icp-sync-plugin/sync-plugin.wit | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 40646741f..fcc8a82fa 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -36,9 +36,8 @@ docs; the *reasons* behind those choices are recorded here. versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve - canister names it knows about. It is informational only: `canister-call` - still targets the canister being synced, so the table grants no ability to - call other canisters. + canister names it knows about. It is informational only; calling still + requires a declaration. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index bd7b13d67..751e263ad 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -73,8 +73,8 @@ interface types { /// Name→principal mapping for every named canister in the project for /// the environment being synced, sorted by name. Informational: the /// plugin may use it to resolve canister names it knows about. Being - /// listed here does not let the plugin call a canister — the - /// `canister-call` import always targets the canister being synced. + /// listed here does not grant permission to call a canister — that + /// still requires declaring it as a dependency (see `call-target`). canister-ids: list, } From ea543e8d53d534002bf488c2c2ff975dd3027efa Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 10:34:51 -0700 Subject: [PATCH 36/51] copilot --- crates/icp-cli/src/operations/bundle.rs | 29 +++++- crates/icp-cli/tests/bundle_tests.rs | 130 ++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 4b07c5f5d..af861f99b 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -17,7 +17,9 @@ use icp::{ ArgsFormat, BuildStep, BuildSteps, CanisterManifest, DependencyManifest, EnvironmentManifest, Instructions, Item, LoadManifestFromPathError, ManagedMode, ManifestInitArgs, Mode, NetworkManifest, PROJECT_MANIFEST, ProjectManifest, SyncStep, - SyncSteps, load_manifest_from_path, plugin, prebuilt, + SyncSteps, load_manifest_from_path, plugin, + plugin::CanisterRef, + prebuilt, prebuilt::{LocalSource, SourceField}, }, package::PackageCache, @@ -650,6 +652,7 @@ async fn prepare_canister( canister_path, &path_name, idx, + local_names, pkg_cache, out, ) @@ -710,6 +713,27 @@ fn localize_controllers( settings } +/// Rewrite a plugin's declared call targets from workspace store keys back to the +/// local names of the instance being written, on the same grounds as +/// [`localize_controllers`]. Principals are already absolute and pass through. +fn localize_call_targets( + canisters: Option<&[CanisterRef]>, + local_names: &HashMap<&str, &str>, +) -> Option> { + canisters.map(|canisters| { + canisters + .iter() + .map(|target| match target { + CanisterRef::Name(name) => match local_names.get(name.as_str()) { + Some(local) => CanisterRef::Name((*local).to_owned()), + None => target.clone(), + }, + CanisterRef::Principal(_) => target.clone(), + }) + .collect() + }) +} + #[allow(clippy::too_many_arguments)] async fn prepare_plugin_step( adapter: &plugin::Adapter, @@ -718,6 +742,7 @@ async fn prepare_plugin_step( canister_path: &Path, path_name: &str, idx: usize, + local_names: &HashMap<&str, &str>, pkg_cache: &PackageCache, out: &mut BundleArtifacts, ) -> Result { @@ -788,7 +813,7 @@ async fn prepare_plugin_step( sha256: Some(plugin_sha256), dirs: bundle_dirs, files: bundle_files, - canisters: None, + canisters: localize_call_targets(adapter.canisters.as_deref(), local_names), })) } diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 74748e4e4..9619fb10a 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1088,6 +1088,136 @@ fn bundle_packages_plugin_sync_steps() { ); } +/// A plugin's declared call targets must survive bundling — dropping them would turn a +/// working project into a bundle whose cross-canister calls are all rejected. Names of +/// the writing instance's own canisters come back out as local names; principals and +/// names that already resolved against the workspace are left alone. +#[test] +fn bundle_preserves_plugin_call_targets() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let build_step = formatdoc! {r#" + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + "#}; + + // Bundling only repackages the plugin wasm bytes, so any non-empty content works. + write(&project_dir.join("plugin.wasm"), b"\x00asm\x01\x00\x00\x00") + .expect("failed to write plugin wasm"); + + let dep_dir = project_dir.join("vendor/openemail"); + create_dir_all(&dep_dir).expect("failed to create dependency dir"); + write(&dep_dir.join("plugin.wasm"), b"\x00asm\x01\x00\x00\x00") + .expect("failed to write dependency plugin wasm"); + + // The dependency's plugin names its own sibling, both bare and by store key. + write_string( + &dep_dir.join("icp.yaml"), + &formatdoc! {r#" + canisters: + - name: backend + {build_step} + sync: + steps: + - type: plugin + path: plugin.wasm + canisters: + - helper + - vendor/openemail:helper + - name: helper + {build_step} + "#}, + ) + .expect("failed to write dependency manifest"); + + // The root's plugin names a root sibling, a dependency canister by store key, and a + // literal principal. + write_string( + &project_dir.join("icp.yaml"), + &formatdoc! {r#" + canisters: + - name: frontend + {build_step} + sync: + steps: + - type: plugin + path: plugin.wasm + canisters: + - api + - vendor/openemail:backend + - aaaaa-aa + - name: api + {build_step} + + dependencies: + - name: openemail + path: ./vendor/openemail + canisters: [backend] + "#}, + ) + .expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut manifests: std::collections::HashMap = std::collections::HashMap::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path.ends_with("icp.yaml") { + let mut yaml = String::new(); + entry + .read_to_string(&mut yaml) + .expect("failed to read manifest"); + manifests.insert(path, yaml); + } + } + + let plugin_targets = |yaml: &str, canister: &str| -> Vec { + let parsed: serde_yaml::Value = + serde_yaml::from_str(yaml).expect("manifest yaml is invalid"); + let canisters = parsed["canisters"] + .as_sequence() + .expect("manifest has no canisters"); + let entry = canisters + .iter() + .find(|c| c["name"].as_str() == Some(canister)) + .unwrap_or_else(|| panic!("{canister} not found in bundled manifest: {yaml}")); + entry["sync"]["steps"][0]["canisters"] + .as_sequence() + .unwrap_or_else(|| panic!("{canister} plugin step lost its canisters: {yaml}")) + .iter() + .map(|t| t.as_str().expect("call target is not a string").to_owned()) + .collect() + }; + + assert_eq!( + plugin_targets(&manifests["icp.yaml"], "frontend"), + ["api", "vendor/openemail:backend", "aaaaa-aa"], + ); + // Both spellings of the dependency's own sibling come out as its local name. + assert_eq!( + plugin_targets(&manifests["vendor/openemail/icp.yaml"], "backend"), + ["helper", "helper"], + ); +} + /// An `icp_appmanifest.yaml` next to the project manifest must be included in the bundle, with its /// top-level `images` paths relocated under a top-level `images/` folder and the /// referenced image files copied alongside. Unrelated metadata is preserved. From 6caedb4127311a927d92fdc8a52cf957600f91ab Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 10:05:59 -0700 Subject: [PATCH 37/51] Remove id target --- crates/icp-cli/src/operations/bundle.rs | 19 +++---- crates/icp-cli/tests/bundle_tests.rs | 6 +-- crates/icp-sync-plugin/DESIGN.md | 23 ++++---- crates/icp-sync-plugin/src/runtime.rs | 47 +--------------- crates/icp-sync-plugin/sync-plugin.wit | 14 +++-- crates/icp/src/canister/sync/plugin.rs | 65 ++++++++++------------- crates/icp/src/manifest/adapter/plugin.rs | 38 ++++--------- docs/concepts/sync-plugins.md | 6 +-- docs/guides/writing-sync-plugins.md | 2 +- docs/reference/configuration.md | 6 +-- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- 12 files changed, 75 insertions(+), 155 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index af861f99b..bd460d576 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -17,9 +17,7 @@ use icp::{ ArgsFormat, BuildStep, BuildSteps, CanisterManifest, DependencyManifest, EnvironmentManifest, Instructions, Item, LoadManifestFromPathError, ManagedMode, ManifestInitArgs, Mode, NetworkManifest, PROJECT_MANIFEST, ProjectManifest, SyncStep, - SyncSteps, load_manifest_from_path, plugin, - plugin::CanisterRef, - prebuilt, + SyncSteps, load_manifest_from_path, plugin, prebuilt, prebuilt::{LocalSource, SourceField}, }, package::PackageCache, @@ -715,20 +713,17 @@ fn localize_controllers( /// Rewrite a plugin's declared call targets from workspace store keys back to the /// local names of the instance being written, on the same grounds as -/// [`localize_controllers`]. Principals are already absolute and pass through. +/// [`localize_controllers`]. fn localize_call_targets( - canisters: Option<&[CanisterRef]>, + canisters: Option<&[String]>, local_names: &HashMap<&str, &str>, -) -> Option> { +) -> Option> { canisters.map(|canisters| { canisters .iter() - .map(|target| match target { - CanisterRef::Name(name) => match local_names.get(name.as_str()) { - Some(local) => CanisterRef::Name((*local).to_owned()), - None => target.clone(), - }, - CanisterRef::Principal(_) => target.clone(), + .map(|target| match local_names.get(target.as_str()) { + Some(local) => (*local).to_owned(), + None => target.clone(), }) .collect() }) diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 9619fb10a..67b8a014f 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1134,8 +1134,7 @@ fn bundle_preserves_plugin_call_targets() { ) .expect("failed to write dependency manifest"); - // The root's plugin names a root sibling, a dependency canister by store key, and a - // literal principal. + // The root's plugin names a root sibling and a dependency canister by store key. write_string( &project_dir.join("icp.yaml"), &formatdoc! {r#" @@ -1149,7 +1148,6 @@ fn bundle_preserves_plugin_call_targets() { canisters: - api - vendor/openemail:backend - - aaaaa-aa - name: api {build_step} @@ -1209,7 +1207,7 @@ fn bundle_preserves_plugin_call_targets() { assert_eq!( plugin_targets(&manifests["icp.yaml"], "frontend"), - ["api", "vendor/openemail:backend", "aaaaa-aa"], + ["api", "vendor/openemail:backend"], ); // Both spellings of the dependency's own sibling come out as its local name. assert_eq!( diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index fcc8a82fa..e9270dc79 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -29,11 +29,13 @@ docs; the *reasons* behind those choices are recorded here. unchanged. This keeps the host free of any per-canister type knowledge. - **`canister-call` takes an explicit `target`** — the plugin selects the canister being synced (`host`) or a canister it declared as a dependency, by - name or principal. The host resolves the target and *enforces* the - declaration: a target absent from the step's `canisters:` list is rejected - without a call. (In the earlier `@0.1.0` interface `canister-call` had no - target and always reached the canister being synced; see *Interface - versioning* below.) + name. The host resolves the target and *enforces* the declaration: a target + absent from the step's `canisters:` list is rejected without a call. Names are + the only way to address a dependency: the name→principal mapping is the host's + to make, since it varies per environment, and a plugin that hardcodes a + principal is pinned to one deployment. (In the earlier `@0.1.0` interface + `canister-call` had no target and always reached the canister being synced; see + *Interface versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still @@ -118,7 +120,7 @@ mod v1 { wasmtime::component::bindgen!({ world: "sync-plugin", path: "sync-plugi struct HostState { host_canister_id: Principal, - callable: CallableCanisters, // by_name + by_id, from the manifest + callable: CallableCanisters, // name → principal, from the manifest agent: Arc, proxy: Option, wasi_ctx: wasmtime_wasi::WasiCtx, @@ -186,14 +188,13 @@ pub struct Adapter { pub sha256: Option, pub dirs: Option>, pub files: Option>, - pub canisters: Option>, // extra callable canisters + pub canisters: Option>, // extra callable canisters, by name } ``` -`CanisterRef` is an untagged `Principal | Name` (anything that parses as a -principal is one; everything else is a name), written in the manifest as a plain -string. `Deserialize` is hand-written to reject a `url` source without a -`sha256`. +Each `canisters:` entry is a canister name resolved against the project's ID +table for the environment being synced. `Deserialize` is hand-written to reject a +`url` source without a `sha256`. ### `crates/icp/src/canister/sync/plugin.rs` diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index f45ece92c..6c75caae2 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1,5 +1,5 @@ // Host-side Component Model runtime for sync plugins. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex as StdMutex; @@ -67,10 +67,6 @@ pub struct CallableCanisters { /// Dependencies callable by name ([`CallTarget::Name`]). Maps the name — as /// it appears in the canister ID table — to the principal it resolves to. pub by_name: BTreeMap, - /// Every principal callable by [`CallTarget::Id`]. Includes the principals - /// of the `by_name` entries, so an author may target the same canister - /// either way. - pub by_id: BTreeSet, } /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing @@ -89,18 +85,6 @@ fn resolve_call_target( `canisters` list to allow it" ) }), - CallTarget::Id(text) => { - let principal = Principal::from_text(text) - .map_err(|e| format!("invalid target principal '{text}': {e}"))?; - if principal == host_canister_id || callable.by_id.contains(&principal) { - Ok(principal) - } else { - Err(format!( - "plugin is not permitted to call canister '{principal}': declare it in the \ - sync step's `canisters` list to allow it" - )) - } - } } } @@ -857,7 +841,6 @@ mod tests { let dep = Principal::from_slice(&[2; 4]); let callable = CallableCanisters { by_name: BTreeMap::from([("backend".to_string(), dep)]), - by_id: BTreeSet::from([dep]), }; assert_eq!( resolve_call_target(&CallTarget::Name("backend".into()), host, &callable).unwrap(), @@ -871,34 +854,6 @@ mod tests { ); } - #[test] - fn resolve_target_id_allows_host_and_declared_only() { - let host = Principal::from_slice(&[1; 4]); - let dep = Principal::from_slice(&[2; 4]); - let other = Principal::from_slice(&[3; 4]); - let callable = CallableCanisters { - by_name: BTreeMap::new(), - by_id: BTreeSet::from([dep]), - }; - // A declared principal is allowed; so is the host, implicitly. - assert_eq!( - resolve_call_target(&CallTarget::Id(dep.to_text()), host, &callable).unwrap(), - dep - ); - assert_eq!( - resolve_call_target(&CallTarget::Id(host.to_text()), host, &callable).unwrap(), - host - ); - // An undeclared principal is rejected. - let err = resolve_call_target(&CallTarget::Id(other.to_text()), host, &callable) - .expect_err("undeclared principal must be rejected"); - assert!(err.contains("not permitted"), "got: {err}"); - // Garbage text is a distinct, clearer error. - let err = resolve_call_target(&CallTarget::Id("not a principal".into()), host, &callable) - .expect_err("invalid principal text must be rejected"); - assert!(err.contains("invalid target principal"), "got: {err}"); - } - // ------------------------------------------------------------------------- // Error-path tests — no fixture WASM needed // ------------------------------------------------------------------------- diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 751e263ad..cace1e32c 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -36,9 +36,9 @@ interface types { /// Which canister a `canister-call-request` targets. /// /// A plugin may target the canister being synced, or any canister it - /// declared as a dependency in the sync step's `canisters` list — by that - /// canister's name or by its textual principal. Targeting a canister that - /// was not declared as a dependency is rejected by the host. + /// declared as a dependency in the sync step's `canisters` list — always by + /// that canister's name. Targeting a canister that was not declared as a + /// dependency is rejected by the host. variant call-target { /// The canister being synced (`sync-exec-input.canister-id`). Always /// permitted, whether or not it also appears in `canisters`. @@ -48,8 +48,6 @@ interface types { /// canister in the same subproject, or a `subproject:local` key /// otherwise. The host resolves it against that mapping table. name(string), - /// A declared-dependency canister identified by its textual principal. - id(string), } /// Input passed by the runtime to the plugin's exec() export. @@ -81,7 +79,7 @@ interface types { /// A request to call a method on a canister. record canister-call-request { /// Which canister to call. `host` targets the canister being synced; - /// `name`/`id` target a canister declared as a dependency in the sync + /// `name` targets a canister declared as a dependency in the sync /// step's `canisters` list. target: call-target, /// The canister method to call. @@ -116,8 +114,8 @@ world sync-plugin { /// Make an update or query call to a canister. /// The `req.target` selects the canister: the one being synced (`host`), or /// a canister declared as a dependency in the sync step's `canisters` list, - /// by name or principal. A target that was not declared as a dependency is - /// rejected without making a call. + /// by name. A target that was not declared as a dependency is rejected + /// without making a call. /// Returns the raw Candid-encoded response bytes on success or an error /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index ad50d1d0e..1e525dc22 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use camino::Utf8PathBuf; use candid::Principal; @@ -10,11 +10,7 @@ use icp_sync_plugin::{ use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{ - canister::wasm, - manifest::adapter::plugin::{Adapter, CanisterRef}, - package::PackageCache, -}; +use crate::{canister::wasm, manifest::adapter::plugin::Adapter, package::PackageCache}; use super::Params; @@ -164,7 +160,7 @@ fn exposed_canister_ids(params: &Params) -> BTreeMap { } /// Resolve the canisters a plugin declared it may call into a [`CallableCanisters`] -/// enforcement set. Named dependencies are looked up in `canister_ids`; a name +/// enforcement set. Each declared name is looked up in `canister_ids`; a name /// that does not resolve is a manifest error. fn resolve_callable( adapter: &Adapter, @@ -172,27 +168,17 @@ fn resolve_callable( environment: &str, ) -> Result { let mut by_name = BTreeMap::new(); - let mut by_id = BTreeSet::new(); - for canister in adapter.canisters.iter().flatten() { - match canister { - CanisterRef::Principal(principal) => { - by_id.insert(*principal); - } - CanisterRef::Name(name) => { - let principal = - canister_ids - .get(name) - .copied() - .context(UnknownDependencySnafu { - name: name.clone(), - environment: environment.to_owned(), - })?; - by_name.insert(name.clone(), principal); - by_id.insert(principal); - } - } + for name in adapter.canisters.iter().flatten() { + let principal = canister_ids + .get(name) + .copied() + .context(UnknownDependencySnafu { + name: name.clone(), + environment: environment.to_owned(), + })?; + by_name.insert(name.clone(), principal); } - Ok(CallableCanisters { by_name, by_id }) + Ok(CallableCanisters { by_name }) } #[cfg(test)] @@ -236,7 +222,7 @@ mod tests { } } - fn adapter_with(canisters: Option>) -> Adapter { + fn adapter_with(canisters: Option>) -> Adapter { Adapter { source: SourceField::Local(LocalSource { path: "plugin.wasm".into(), @@ -337,25 +323,30 @@ mod tests { } #[test] - fn resolve_callable_resolves_names_and_principals() { + fn resolve_callable_resolves_names() { let dep = principal(1); - let raw = principal(2); - let table = BTreeMap::from([("backend".to_owned(), dep)]); + let sibling = principal(2); + let table = BTreeMap::from([ + ("backend".to_owned(), sibling), + ("services/open-crm:backend".to_owned(), dep), + ]); let adapter = adapter_with(Some(vec![ - CanisterRef::Name("backend".to_owned()), - CanisterRef::Principal(raw), + "backend".to_owned(), + "services/open-crm:backend".to_owned(), ])); let callable = resolve_callable(&adapter, &table, "demo").unwrap(); - assert_eq!(callable.by_name.get("backend"), Some(&dep)); - assert!(callable.by_id.contains(&dep)); - assert!(callable.by_id.contains(&raw)); + assert_eq!(callable.by_name.get("backend"), Some(&sibling)); + assert_eq!( + callable.by_name.get("services/open-crm:backend"), + Some(&dep) + ); } #[test] fn resolve_callable_rejects_unknown_name() { - let adapter = adapter_with(Some(vec![CanisterRef::Name("nope".to_owned())])); + let adapter = adapter_with(Some(vec!["nope".to_owned()])); let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") .expect_err("an undeclared name must fail"); assert!(matches!(err, PluginError::UnknownDependency { .. })); diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 5aef5ccfa..68a76c686 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,24 +1,8 @@ -use candid::Principal; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; use super::prebuilt::SourceField; -/// A canister a sync plugin is permitted to call, beyond the canister being -/// synced. Written in the manifest either as a textual principal (e.g. -/// `aaaaa-aa`) or as a canister name resolved against the project's canister ID -/// table for the environment being synced (e.g. `backend`, or a namespaced -/// dependency canister such as `services/open-crm:backend`). Anything that -/// parses as a principal is taken as one; everything else is a name. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CanisterRef { - /// An explicit principal (e.g. `aaaaa-aa`). - Principal(Principal), - /// A canister name from this project's ID table (e.g. `backend`). - Name(String), -} - /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -63,12 +47,12 @@ pub struct Adapter { pub files: Option>, /// Canisters this plugin may call in addition to the canister being synced. - /// Each entry is a canister name (resolved against the project's canister ID - /// table) or a textual principal. The plugin picks a target per call via the - /// `call-target` in its `canister-call` request; a target not listed here is - /// rejected by the host. - #[schemars(with = "Option>")] - pub canisters: Option>, + /// Each entry is a canister name resolved against the project's canister ID + /// table for the environment being synced (e.g. `backend`, or a namespaced + /// dependency canister such as `services/open-crm:backend`). The plugin + /// picks a target per call via the `call-target` in its `canister-call` + /// request; a target not listed here is rejected by the host. + pub canisters: Option>, } impl<'de> Deserialize<'de> for Adapter { @@ -80,7 +64,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, - canisters: Option>, + canisters: Option>, } let h = AdapterHelper::deserialize(d)?; @@ -168,23 +152,21 @@ mod tests { } #[test] - fn canisters_parse_as_names_and_principals() { + fn canisters_parse_as_names() { let adapter = serde_yaml::from_str::( r#" path: plugins/my-sync.wasm canisters: - backend - services/open-crm:backend - - aaaaa-aa "#, ) .expect("failed to deserialize Adapter with canisters"); assert_eq!( adapter.canisters, Some(vec![ - CanisterRef::Name("backend".to_string()), - CanisterRef::Name("services/open-crm:backend".to_string()), - CanisterRef::Principal(Principal::from_text("aaaaa-aa").unwrap()), + "backend".to_string(), + "services/open-crm:backend".to_string(), ]), ); } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index aca070959..4dae4da32 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -40,7 +40,7 @@ icp sync │ └─ plugin makes canister-call({ target, ... }) (× N) target = host (the canister being synced), or a - declared-dependency canister by name or principal + declared-dependency canister by name ``` ## The Plugin Interface @@ -81,14 +81,14 @@ The plugin calls methods through the `canister-call` import. It picks a `target` | Request field | Meaning | |---------------|---------| -| `target` | Which canister to call: `host` (the canister being synced), or a canister declared in `canisters:` addressed by `name` or by `id` (principal) | +| `target` | Which canister to call: `host` (the canister being synced), or a canister declared in `canisters:` addressed by `name` | | `method` | The canister method to call | | `arg` | Candid-encoded argument bytes (the plugin encodes; the host forwards as-is) | | `call-type` | `update` or `query` | | `direct` | When `false` (default), update calls are routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, the call always goes directly to the target. Query calls always go directly regardless. | | `cycles` | Cycles to attach to a proxied update call; only meaningful when `direct` is `false`, a proxy is configured, and `call-type` is `update` | -The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name`/`id` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. +The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. A name is the only way to address another canister — the host owns the name→principal mapping, which differs per environment. ### Logging — stdout and stderr diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 0c3004752..80cf1f46e 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -85,7 +85,7 @@ export!(Plugin); A few things to note: - **You encode the arguments.** `arg` is raw Candid bytes. Encode with `candid::Encode!`; decode any response (`Vec`) with `candid::Decode!`. -- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` or `CallTarget::Id(principal_text)` — a name matches the entries in `input.canister_ids`. The host rejects a target you did not declare. +- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` — the name matches the entries in `input.canister_ids`. Names are the only way to reach another canister; the host resolves them per environment. The host rejects a target you did not declare. - **`direct` and `cycles` control proxy routing.** With `direct: false`, update calls go through the [proxy canister](proxy-canister.md) when one is configured, and `cycles` can fund the forwarded call. With `direct: true`, the call always goes straight to the target. See [The Plugin Interface](../concepts/sync-plugins.md#the-plugin-interface) for the full semantics. ## Read Declared Files and Directories diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 7e594facd..7d885a292 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -151,9 +151,9 @@ sync: - config files: # files read by the host and passed inline - config.txt - canisters: # extra canisters the plugin may call + canisters: # extra canisters the plugin may call, - ledger # by name (resolved for the environment) - - ryjl3-tyaaa-aaaaa-aaaba-cai # or by principal + - services/open-crm:backend # Remote plugin (downloaded and verified before execution) - type: plugin @@ -168,7 +168,7 @@ sync: | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | | `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | | `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | -| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name (resolved against the project's canister IDs for the environment) or a textual principal | +| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 4dcd4d0b8..c361b4562 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 1c4758c8e..5f27fa902 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name (resolved against the project's canister ID\ntable) or a textual principal. The plugin picks a target per call via the\n`call-target` in its `canister-call` request; a target not listed here is\nrejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, From 371af525c634feec677797836768e63b60471629 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 10:13:56 -0700 Subject: [PATCH 38/51] Remove project dependencies from sandboxing logic --- crates/icp-sync-plugin/DESIGN.md | 18 +++++++-------- crates/icp-sync-plugin/src/runtime.rs | 22 +++++++++--------- crates/icp-sync-plugin/sync-plugin.wit | 27 +++++++++++------------ crates/icp/src/canister/sync/mod.rs | 2 +- crates/icp/src/canister/sync/plugin.rs | 23 ++++++++++--------- crates/icp/src/manifest/adapter/plugin.rs | 2 +- docs/concepts/sync-plugins.md | 10 ++++----- docs/reference/configuration.md | 2 +- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index e9270dc79..9d57338c9 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -28,18 +28,18 @@ docs; the *reasons* behind those choices are recorded here. `list`. The plugin owns Candid encoding/decoding; the host forwards bytes unchanged. This keeps the host free of any per-canister type knowledge. - **`canister-call` takes an explicit `target`** — the plugin selects the - canister being synced (`host`) or a canister it declared as a dependency, by - name. The host resolves the target and *enforces* the declaration: a target - absent from the step's `canisters:` list is rejected without a call. Names are - the only way to address a dependency: the name→principal mapping is the host's - to make, since it varies per environment, and a plugin that hardcodes a - principal is pinned to one deployment. (In the earlier `@0.1.0` interface + canister being synced (`host`) or a canister from the step's `canisters:` + list, by name. The host resolves the target and *enforces* the list: a target + absent from it is rejected without a call. Names are the only way to address + another canister: the name→principal mapping is the host's to make, since it + varies per environment, and a plugin that hardcodes a principal is pinned to + one deployment. (In the earlier `@0.1.0` interface `canister-call` had no target and always reached the canister being synced; see *Interface versioning* below.) - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still - requires a declaration. + requires an entry in `canisters:`. - **Filesystem access via WASI, not a host import** — plugins use standard language APIs (`std::fs`); the host preopens the declared `dirs` read-only. No bespoke `read-file`/`list-dir` import is needed. @@ -205,5 +205,5 @@ enforcement set (resolving `canisters:` against the project's IDs), then calls the CLI — opens the declared paths and enforces the path-safety checks, so the CLI no longer touches the plugin's input files itself. `exposed_canister_ids` adds a bare-local-name duplicate for every canister in the same subproject as -the one being synced; `resolve_callable` fails the step if a declared dependency -name does not resolve. +the one being synced; `resolve_callable` fails the step if a name in +`canisters:` does not resolve. diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 6c75caae2..4632a1ab9 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -59,19 +59,19 @@ use v2::icp::sync_plugin::types::{CallTarget, CallType, CanisterIdEntry}; /// The canisters a sync plugin is permitted to call, beyond the canister being /// synced (which is always reachable via [`CallTarget::Host`]). /// -/// Built by the CLI from the plugin step's declared `canisters` dependencies, -/// resolved against the project's canister ID table. Keeping the resolution on -/// the CLI side keeps this runtime crate free of any manifest knowledge. +/// Built by the CLI from the plugin step's `canisters` list, resolved against +/// the project's canister ID table. Keeping the resolution on the CLI side +/// keeps this runtime crate free of any manifest knowledge. #[derive(Clone, Debug, Default)] pub struct CallableCanisters { - /// Dependencies callable by name ([`CallTarget::Name`]). Maps the name — as - /// it appears in the canister ID table — to the principal it resolves to. + /// Canisters callable by name ([`CallTarget::Name`]). Maps the name — as it + /// appears in the canister ID table — to the principal it resolves to. pub by_name: BTreeMap, } /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing -/// that the plugin declared it as a dependency. The canister being synced -/// (`host`) is always permitted. +/// that the plugin listed it in `canisters`. The canister being synced (`host`) +/// is always permitted. fn resolve_call_target( target: &CallTarget, host_canister_id: Principal, @@ -92,7 +92,7 @@ fn resolve_call_target( struct HostState { /// The canister being synced — the target of [`CallTarget::Host`] calls. host_canister_id: Principal, - /// Canisters the plugin declared as dependencies and may also call. + /// Canisters the plugin declared in `canisters` and may also call. callable: CallableCanisters, agent: Arc, /// Proxy canister to route update calls through, if configured. @@ -404,7 +404,7 @@ pub struct PluginInvocation { /// plugin. Same-project canisters appear both under their fully-qualified /// key and their bare local name (see the WIT `canister-id-entry` docs). pub canister_ids: BTreeMap, - /// Canisters the plugin declared as dependencies and may call, beyond the + /// Canisters the plugin declared in `canisters` and may call, beyond the /// canister being synced. Ignored by v0.1.0 plugins, which can only reach /// the canister being synced. pub callable: CallableCanisters, @@ -800,8 +800,8 @@ mod tests { } /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister - /// and identity, no proxy, no declared dependencies, the default compute - /// limit, and the current directory as the base. Individual tests override + /// and identity, no proxy, no declared callable canisters, the default + /// compute limit, and the current directory as the base. Tests override /// the few fields they care about. fn invocation(wasm_path: &str, environment: &str) -> PluginInvocation { PluginInvocation { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index cace1e32c..c2d51fde2 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -35,18 +35,18 @@ interface types { /// Which canister a `canister-call-request` targets. /// - /// A plugin may target the canister being synced, or any canister it - /// declared as a dependency in the sync step's `canisters` list — always by - /// that canister's name. Targeting a canister that was not declared as a - /// dependency is rejected by the host. + /// A plugin may target the canister being synced, or any canister listed in + /// the sync step's `canisters` list — always by that canister's name. + /// Targeting a canister that was not listed is rejected by the host. variant call-target { /// The canister being synced (`sync-exec-input.canister-id`). Always /// permitted, whether or not it also appears in `canisters`. host, - /// A declared-dependency canister identified by name, spelled exactly as - /// it appears in `sync-exec-input.canister-ids` — a bare local name for a - /// canister in the same subproject, or a `subproject:local` key - /// otherwise. The host resolves it against that mapping table. + /// A canister from the `canisters` list, identified by name, spelled + /// exactly as it appears in `sync-exec-input.canister-ids` — a bare + /// local name for a canister in the same subproject, or a + /// `subproject:local` key otherwise. The host resolves it against that + /// mapping table. name(string), } @@ -72,15 +72,15 @@ interface types { /// the environment being synced, sorted by name. Informational: the /// plugin may use it to resolve canister names it knows about. Being /// listed here does not grant permission to call a canister — that - /// still requires declaring it as a dependency (see `call-target`). + /// still requires listing it in `canisters` (see `call-target`). canister-ids: list, } /// A request to call a method on a canister. record canister-call-request { /// Which canister to call. `host` targets the canister being synced; - /// `name` targets a canister declared as a dependency in the sync - /// step's `canisters` list. + /// `name` targets a canister listed in the sync step's `canisters` + /// list. target: call-target, /// The canister method to call. method: string, @@ -113,9 +113,8 @@ world sync-plugin { /// Make an update or query call to a canister. /// The `req.target` selects the canister: the one being synced (`host`), or - /// a canister declared as a dependency in the sync step's `canisters` list, - /// by name. A target that was not declared as a dependency is rejected - /// without making a call. + /// a canister listed in the sync step's `canisters` list, by name. A target + /// that was not listed is rejected without making a call. /// Returns the raw Candid-encoded response bytes on success or an error /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index 93dfdd0bd..a90ff93be 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -20,7 +20,7 @@ pub struct Params { pub path: PathBuf, pub cid: Principal, /// Fully-qualified store key of the canister being synced (e.g. `backend`, - /// or `services/open-crm:backend` for a dependency canister). Its namespace + /// or `services/open-crm:backend` for a canister in a subproject). Its namespace /// prefix identifies which other canisters are in the same subproject. pub name: String, /// Name of the environment being synced (e.g. "local", "production"). diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 1e525dc22..73a111306 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -31,10 +31,10 @@ pub enum PluginError { Run { source: RunPluginError }, #[snafu(display( - "sync plugin declares a dependency on canister '{name}', but no canister by that name \ + "sync plugin lists canister '{name}' as callable, but no canister by that name \ is known in environment '{environment}'" ))] - UnknownDependency { name: String, environment: String }, + UnknownCallableCanister { name: String, environment: String }, } /// Resolve the plugin compute-time limit, honoring the @@ -101,7 +101,7 @@ pub(super) async fn sync( let files: Vec = adapter.files.clone().unwrap_or_default(); // 3. Build the canister ID table exposed to the plugin, then resolve the - // plugin's declared callable canisters against it. + // step's `canisters` list against it. let canister_ids = exposed_canister_ids(params); let callable = resolve_callable(adapter, &canister_ids, environment)?; @@ -137,9 +137,10 @@ pub(super) async fn sync( /// The canister ID table exposed to a sync plugin: every named canister in the /// project, plus — for canisters in the same subproject as the one being synced /// — a duplicate entry under the bare local name. A store key is -/// `:` for a dependency canister and a bare local name for a -/// canister defined directly in the app root (see the WIT `canister-id-entry` -/// docs), so the syncing canister's namespace is the prefix of its own key. +/// `:` for a canister in a subproject and a bare local name +/// for a canister defined directly in the app root (see the WIT +/// `canister-id-entry` docs), so the syncing canister's namespace is the prefix +/// of its own key. /// /// A local name never contains a colon but a subproject directory may, so keys /// split on their *last* colon. The bare-name aliases take precedence over an @@ -159,9 +160,9 @@ fn exposed_canister_ids(params: &Params) -> BTreeMap { table } -/// Resolve the canisters a plugin declared it may call into a [`CallableCanisters`] -/// enforcement set. Each declared name is looked up in `canister_ids`; a name -/// that does not resolve is a manifest error. +/// Resolve the step's `canisters` list into a [`CallableCanisters`] enforcement +/// set. Each listed name is looked up in `canister_ids`; a name that does not +/// resolve is a manifest error. fn resolve_callable( adapter: &Adapter, canister_ids: &BTreeMap, @@ -172,7 +173,7 @@ fn resolve_callable( let principal = canister_ids .get(name) .copied() - .context(UnknownDependencySnafu { + .context(UnknownCallableCanisterSnafu { name: name.clone(), environment: environment.to_owned(), })?; @@ -349,6 +350,6 @@ mod tests { let adapter = adapter_with(Some(vec!["nope".to_owned()])); let err = resolve_callable(&adapter, &BTreeMap::new(), "demo") .expect_err("an undeclared name must fail"); - assert!(matches!(err, PluginError::UnknownDependency { .. })); + assert!(matches!(err, PluginError::UnknownCallableCanister { .. })); } } diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 68a76c686..099825626 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -49,7 +49,7 @@ pub struct Adapter { /// Canisters this plugin may call in addition to the canister being synced. /// Each entry is a canister name resolved against the project's canister ID /// table for the environment being synced (e.g. `backend`, or a namespaced - /// dependency canister such as `services/open-crm:backend`). The plugin + /// subproject canister such as `services/open-crm:backend`). The plugin /// picks a target per call via the `call-target` in its `canister-call` /// request; a target not listed here is rejected by the host. pub canisters: Option>, diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 4dae4da32..144c30ec6 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it declares as dependencies. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -15,7 +15,7 @@ Sync plugins fill that gap. A plugin is: - **Portable** — written in any language that compiles to `wasm32-wasip2`, distributed as one `.wasm` file (local path or remote URL + `sha256`). - **Sandboxed** — it cannot open network sockets, spawn subprocesses, or touch the filesystem outside the directories you explicitly grant it. -- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister it declares as a dependency in the manifest's `canisters:` list. A call to a canister that was not declared is rejected by the host. +- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister listed in the manifest's `canisters:` list. A call to a canister that was not listed is rejected by the host. The most common way to get a sync plugin is through a [recipe](recipes.md). For example, the `@dfinity/asset-canister` recipe emits a `plugin` sync step (starting with `v2.2.1`) that uploads your built static files to the asset canister — so for everyday frontend deployment you never write a plugin yourself. @@ -40,7 +40,7 @@ icp sync │ └─ plugin makes canister-call({ target, ... }) (× N) target = host (the canister being synced), or a - declared-dependency canister by name + canister from `canisters:` by name ``` ## The Plugin Interface @@ -49,7 +49,7 @@ The interface is defined as a [WIT](https://component-model.bytecodealliance.org ```wit world sync-plugin { - // Host import: call the canister being synced or a declared dependency. + // Host import: call the canister being synced or one listed in `canisters:`. import canister-call: func(req: canister-call-request) -> result, string>; // Plugin export: run the sync step. @@ -73,7 +73,7 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | | `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | -Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister that came from a dependency. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. +Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister defined in a subproject. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. ### Calling a canister — `canister-call` diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 7d885a292..3ba6ccc62 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -172,7 +172,7 @@ sync: Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. -A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a dependency canister. A name that does not resolve to a known canister for the environment fails the sync step. +A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a canister defined in a subproject. A name that does not resolve to a known canister for the environment fails the sync step. The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index c361b4562..9b9db63d9 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 5f27fa902..87186951b 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\ndependency canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, From 755a686f124665ef0caeeeedf918655911e0f61c Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Wed, 19 Aug 2026 07:37:27 -0700 Subject: [PATCH 39/51] Add kv fields to plugin input --- crates/icp-cli/src/operations/bundle.rs | 1 + crates/icp-sync-plugin/DESIGN.md | 5 ++- crates/icp-sync-plugin/src/runtime.rs | 43 +++++++++++++++++++ crates/icp-sync-plugin/sync-plugin.wit | 15 ++++++- .../tests/fixtures/test-plugin/src/lib.rs | 17 ++++++++ crates/icp/src/canister/sync/plugin.rs | 8 ++++ crates/icp/src/manifest/adapter/plugin.rs | 33 ++++++++++++++ crates/icp/src/manifest/canister.rs | 2 + docs/concepts/sync-plugins.md | 5 ++- docs/guides/writing-sync-plugins.md | 14 +++++- docs/reference/configuration.md | 5 ++- docs/schemas/canister-yaml-schema.json | 12 +++++- docs/schemas/icp-yaml-schema.json | 12 +++++- 13 files changed, 163 insertions(+), 9 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index bd460d576..dd90010b4 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -809,6 +809,7 @@ async fn prepare_plugin_step( dirs: bundle_dirs, files: bundle_files, canisters: localize_call_targets(adapter.canisters.as_deref(), local_names), + fields: adapter.fields.clone(), })) } diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 9d57338c9..56b563944 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -75,7 +75,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin ``` `PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `dirs`, `files`, -`host_canister_id` (the canister being synced), `agent`, `proxy`, +`fields`, `host_canister_id` (the canister being synced), `agent`, `proxy`, `identity_principal`, `environment`, `compute_limit_secs`, the exposed `canister_ids` table, the `callable: CallableCanisters` enforcement set, and `stdio`. The CLI resolves the manifest's declared `canisters:` into @@ -188,7 +188,8 @@ pub struct Adapter { pub sha256: Option, pub dirs: Option>, pub files: Option>, - pub canisters: Option>, // extra callable canisters, by name + pub fields: Option>, // inline key-value fields + pub canisters: Option>, // extra callable canisters, by name } ``` diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 4632a1ab9..46c02cda5 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -388,6 +388,9 @@ pub struct PluginInvocation { pub dirs: Vec, /// Manifest-relative files to read and pass inline. pub files: Vec, + /// Key-value fields to pass inline. Passed to v0.2.0 plugins; ignored by + /// v0.1.0 plugins, whose interface has no `fields`. + pub fields: BTreeMap, /// The canister being synced. Reachable via `call-target::host`. pub host_canister_id: Principal, /// Agent used for canister calls. @@ -418,6 +421,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin base_dir, dirs, files, + fields, host_canister_id, agent, proxy, @@ -573,6 +577,10 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin .into_iter() .map(|(name, content)| v2::FileInput { name, content }) .collect(), + fields: fields + .into_iter() + .map(|(name, value)| v2::FieldInput { name, value }) + .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, canister_ids: canister_ids @@ -809,6 +817,7 @@ mod tests { base_dir: ".".into(), dirs: vec![], files: vec![], + fields: BTreeMap::new(), host_canister_id: anon(), agent: dummy_agent(), proxy: None, @@ -1008,6 +1017,40 @@ mod tests { assert!(msg.contains("stdout from plugin"), "got: {msg}"); } + #[tokio::test(flavor = "multi_thread")] + async fn plugin_fields_are_passed_through() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let (tx, mut rx) = tokio::sync::mpsc::channel::(16); + let result = tokio::task::block_in_place(|| { + let mut inv = invocation(wasm_path, "fields"); + inv.fields = BTreeMap::from([ + ("greeting".to_string(), "hi".to_string()), + ("audience".to_string(), "world".to_string()), + ]); + inv.stdio = Some(tx); + run_plugin(inv) + }); + assert!(result.is_ok()); + let echoed = rx + .try_recv() + .expect("expected the plugin to echo its fields"); + assert_eq!(echoed, "audience=world,greeting=hi"); + } + + #[test] + fn plugin_missing_expected_field_fails() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + // The "fields" fixture requires a `greeting` field; passing none fails. + assert!(matches!( + run_plugin(invocation(wasm_path, "fields")), + Err(RunPluginError::PluginFailed { ref message }) if message == "missing 'greeting' field" + )); + } + #[test] fn legacy_v1_plugin_is_detected_and_driven() { // A plugin built against the v0.1.0 interface must still load: the host diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index c2d51fde2..a19c609f2 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -14,6 +14,16 @@ interface types { content: string, } + /// A key-value field declared in the manifest step's `fields` setting. + /// The host passes the plugin's declared fields inline; how they are + /// interpreted is up to the plugin. + record field-input { + /// Field name, as spelled in the manifest. + name: string, + /// Field value. + value: string, + } + /// An entry in the project's canister ID mapping table: a canister name /// and the textual principal it resolves to in the environment being synced. record canister-id-entry { @@ -63,6 +73,9 @@ interface types { /// Files declared in the manifest step's `files` setting, read by /// the host and passed inline. The plugin decides how to use them. files: list, + /// Key-value fields declared in the manifest step's `fields` setting, + /// passed inline. The plugin decides how to use them. + fields: list, /// Textual principal of the signing identity used for canister calls. identity-principal: string, /// Textual principal of the proxy canister, if one was configured via @@ -105,7 +118,7 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input}; + use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 47b82c26a..4093ef226 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -19,6 +19,23 @@ impl Guest for TestPlugin { println!("stdout from plugin"); Ok(()) } + // Echo the fields back so the host can assert on ordering and + // values; fail if the expected field is missing. + "fields" => { + let rendered = input + .fields + .iter() + .map(|f| format!("{}={}", f.name, f.value)) + .collect::>() + .join(","); + match input.fields.iter().find(|f| f.name == "greeting") { + Some(_) => { + eprintln!("{rendered}"); + Ok(()) + } + None => Err("missing 'greeting' field".to_string()), + } + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 73a111306..da28b7288 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -99,6 +99,12 @@ pub(super) async fn sync( let base_dir = Utf8PathBuf::from(params.path.as_str()); let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); + let fields: BTreeMap = adapter + .fields + .clone() + .unwrap_or_default() + .into_iter() + .collect(); // 3. Build the canister ID table exposed to the plugin, then resolve the // step's `canisters` list against it. @@ -120,6 +126,7 @@ pub(super) async fn sync( base_dir, dirs, files, + fields, host_canister_id: params.cid, agent: agent_clone, proxy, @@ -231,6 +238,7 @@ mod tests { sha256: None, dirs: None, files: None, + fields: None, canisters, } } diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 099825626..e0daaa664 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; @@ -20,6 +22,8 @@ use super::prebuilt::SourceField; /// - assets/seed-data /// files: # files read by the host and passed inline /// - config.txt +/// fields: # key-value fields passed inline +/// api_url: https://example.com /// ``` /// /// Example (remote URL — `sha256` is required): @@ -46,6 +50,10 @@ pub struct Adapter { /// the plugin as part of `sync-exec-input.files`. pub files: Option>, + /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. + /// Values are strings; the plugin decides how to interpret them. + pub fields: Option>, + /// Canisters this plugin may call in addition to the canister being synced. /// Each entry is a canister name resolved against the project's canister ID /// table for the environment being synced (e.g. `backend`, or a namespaced @@ -64,6 +72,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, + fields: Option>, canisters: Option>, } @@ -78,6 +87,7 @@ impl<'de> Deserialize<'de> for Adapter { sha256: h.sha256, dirs: h.dirs, files: h.files, + fields: h.fields, canisters: h.canisters, }) } @@ -104,6 +114,7 @@ mod tests { sha256: None, dirs: None, files: None, + fields: None, canisters: None, }, ); @@ -131,11 +142,32 @@ mod tests { sha256: Some("abc123".to_string()), dirs: Some(vec!["assets/seed-data".to_string(), "config".to_string()]), files: Some(vec!["config.txt".to_string()]), + fields: None, canisters: None, }, ); } + #[test] + fn fields_parse_as_a_string_map() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + fields: + api_url: https://example.com + token: abc123 + "#, + ) + .expect("failed to deserialize Adapter with fields"); + assert_eq!( + adapter.fields, + Some(HashMap::from([ + ("api_url".to_string(), "https://example.com".to_string()), + ("token".to_string(), "abc123".to_string()), + ])), + ); + } + #[test] fn remote_url_without_sha256_is_rejected() { let err = serde_yaml::from_str::( @@ -188,6 +220,7 @@ mod tests { sha256: Some("a665a45920422f9d417e".to_string()), dirs: None, files: None, + fields: None, canisters: None, }, ); diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 8d2838a5b..28a576f9b 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -793,6 +793,7 @@ mod tests { sha256: None, dirs: Some(vec!["assets/seed-data/".to_string()]), files: None, + fields: None, canisters: None, } )] @@ -838,6 +839,7 @@ mod tests { ), dirs: None, files: None, + fields: None, canisters: None, })] }), diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 144c30ec6..a4b027dbb 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -25,7 +25,7 @@ When a `plugin` sync step executes for a canister, icp-cli: 1. Resolves the wasm — reads the local `path`, or downloads the `url` to the package cache. 2. Verifies the `sha256` checksum if one is given (required for `url`). -3. Reads any files listed in `files:` and preopens any directories listed in `dirs:` read-only. +3. Reads any files listed in `files:`, preopens any directories listed in `dirs:` read-only, and collects any key-value pairs listed in `fields:`. 4. Instantiates the component in a WASI sandbox and calls its `exec()` export. 5. Forwards the plugin's output to the CLI and reports success or the returned error. @@ -36,7 +36,7 @@ icp sync │ canister-id = │ identity-principal = │ canister-ids = - │ dirs / files = what you declared in the manifest + │ dirs/files/fields = what you declared in the manifest │ └─ plugin makes canister-call({ target, ... }) (× N) target = host (the canister being synced), or a @@ -69,6 +69,7 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `environment` | Name of the environment being synced (e.g. `local`, `production`) | | `dirs` | The directories you declared in `dirs:`; the host preopened each one read-only | | `files` | The files you declared in `files:`, each as a `(name, content)` pair read by the host | +| `fields` | The key-value fields you declared in `fields:`, each as a `(name, value)` pair; values are strings | | `identity-principal` | Textual principal of the signing identity used for canister calls | | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | | `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 80cf1f46e..0a4428397 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -114,6 +114,16 @@ for file in &input.files { Writes, paths outside a preopen, and `..` traversal are all rejected by the sandbox. See [The Sandbox](../concepts/sync-plugins.md#the-sandbox) for the full capability list and resource limits. +## Read Declared Fields + +Key-value pairs declared in the manifest's `fields:` are passed inline as string values. Use them for small configuration a plugin needs without shipping a file: + +```rust +for field in &input.fields { + println!("{} = {}", field.name, field.value); +} +``` + ## Build ```bash @@ -124,7 +134,7 @@ The output `.wasm` (under `target/wasm32-wasip2/release/`) is loaded directly by ## Wire It Into the Manifest -Reference the built wasm from a `plugin` sync step and declare the files and directories the plugin needs: +Reference the built wasm from a `plugin` sync step and declare the files, directories, and fields the plugin needs: ```yaml sync: @@ -135,6 +145,8 @@ sync: - seed-data files: - config.txt + fields: + api_url: https://example.com ``` Then run the sync phase: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 3ba6ccc62..95539c6f8 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -151,7 +151,9 @@ sync: - config files: # files read by the host and passed inline - config.txt - canisters: # extra canisters the plugin may call, + fields: # key-value fields passed inline + api_url: https://example.com + canisters: # extra canisters the plugin may call - ledger # by name (resolved for the environment) - services/open-crm:backend @@ -168,6 +170,7 @@ sync: | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | | `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | | `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | +| `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | | `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 9b9db63d9..711169c87 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -89,6 +89,16 @@ "null" ] }, + "fields": { + "additionalProperties": { + "type": "string" + }, + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "type": [ + "object", + "null" + ] + }, "files": { "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", "items": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 87186951b..303db6e37 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -89,6 +89,16 @@ "null" ] }, + "fields": { + "additionalProperties": { + "type": "string" + }, + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "type": [ + "object", + "null" + ] + }, "files": { "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", "items": { From bd47ac2f88386e5d94baedd34cc3244d33de4367 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 08:34:39 -0700 Subject: [PATCH 40/51] improvements --- crates/icp-cli/tests/bundle_tests.rs | 65 ++++++++ crates/icp-sync-plugin/DESIGN.md | 20 ++- crates/icp-sync-plugin/src/runtime.rs | 29 ++-- .../tests/fixtures/test-plugin/src/lib.rs | 15 +- crates/icp/src/canister/sync/plugin.rs | 7 +- crates/icp/src/manifest/adapter/plugin.rs | 153 +++++++++++++++++- docs/guides/writing-sync-plugins.md | 3 + docs/reference/configuration.md | 3 + docs/schemas/canister-yaml-schema.json | 10 +- docs/schemas/icp-yaml-schema.json | 10 +- 10 files changed, 267 insertions(+), 48 deletions(-) diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 67b8a014f..1500ec340 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -768,6 +768,71 @@ fn bundle_normalizes_dotdot_within_project() { ); } +/// Unlike `dirs`/`files`, a plugin step's `fields` reference nothing on disk, so bundling must +/// carry them into the rewritten manifest verbatim — a deploy from the bundle sees the same +/// configuration the original project declared. +#[test] +fn bundle_preserves_plugin_fields() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + write(&project_dir.join("plugin.wasm"), b"\x00asm\x01\x00\x00\x00") + .expect("failed to write plugin wasm"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: plugin.wasm + fields: + api_url: https://example.com + port: 8080 + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut manifest_yaml = String::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path == "icp.yaml" { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + } + + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let fields = &parsed["canisters"][0]["sync"]["steps"][0]["fields"]; + assert_eq!(fields["api_url"].as_str(), Some("https://example.com")); + // `port` was written unquoted; loading stringifies it, so the rewritten + // manifest carries a string too. + assert_eq!(fields["port"].as_str(), Some("8080")); +} + /// A plugin sync step whose `dirs` entry resolves *outside* the project directory must be /// rejected. Bundles can only reference files inside the project so the produced archive is portable. #[test] diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 56b563944..8737dffab 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -153,7 +153,10 @@ instantiates the matching `bindgen!` world and builds the matching trial instantiation: it is unambiguous and needs no throwaway `Store`. A component with no recognized `icp:sync-plugin/types@` import, or an unsupported version, is rejected with `UnsupportedInterface`. Both `.wit` files -are checked in; `sync-plugin-v1.wit` is the frozen v0.1.0 contract. +are checked in; `sync-plugin-v1.wit` is the frozen v0.1.0 contract. Inputs the +v0.1.0 `sync-exec-input` has no field for — `canister-ids` and `fields` — are +simply dropped for a v1 plugin; a v1 plugin cannot observe them, so declaring +`fields:` alongside one has no effect. ### Compute budget (epoch interruption) @@ -188,14 +191,25 @@ pub struct Adapter { pub sha256: Option, pub dirs: Option>, pub files: Option>, - pub fields: Option>, // inline key-value fields + pub fields: Option>, // inline key-value fields pub canisters: Option>, // extra callable canisters, by name } ``` Each `canisters:` entry is a canister name resolved against the project's ID table for the environment being synced. `Deserialize` is hand-written to reject a -`url` source without a `sha256`. +`url` source without a `sha256`. `fields` is a `BTreeMap` rather than a `HashMap` +so that re-serializing the adapter (the bundler writes a consolidated manifest) +is byte-stable; the WIT interface itself makes no promise about the order fields +arrive in. + +Each `fields` value deserializes through `FieldValue`, which takes any YAML +scalar and stringifies it, so `retries: 3` need not be quoted. `serde_yaml` does +that coercion itself when reading YAML *text*, but a canister's build/sync +section reaches the adapter as an already-parsed `serde_yaml::Value` (see +`CanisterManifest`'s hand-written `Deserialize`), and re-deserializing from a +`Value` keeps a number a number — hence the explicit visitor. Lists, mappings, +and empty values are rejected: there is no string to hand the plugin. ### `crates/icp/src/canister/sync/plugin.rs` diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 46c02cda5..708dc241f 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -1017,26 +1017,21 @@ mod tests { assert!(msg.contains("stdout from plugin"), "got: {msg}"); } - #[tokio::test(flavor = "multi_thread")] - async fn plugin_fields_are_passed_through() { + #[test] + fn plugin_fields_are_passed_through() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; - let (tx, mut rx) = tokio::sync::mpsc::channel::(16); - let result = tokio::task::block_in_place(|| { - let mut inv = invocation(wasm_path, "fields"); - inv.fields = BTreeMap::from([ - ("greeting".to_string(), "hi".to_string()), - ("audience".to_string(), "world".to_string()), - ]); - inv.stdio = Some(tx); - run_plugin(inv) - }); - assert!(result.is_ok()); - let echoed = rx - .try_recv() - .expect("expected the plugin to echo its fields"); - assert_eq!(echoed, "audience=world,greeting=hi"); + let mut inv = invocation(wasm_path, "fields"); + inv.fields = BTreeMap::from([ + ("greeting".to_string(), "hi".to_string()), + ("audience".to_string(), "world".to_string()), + ]); + // The "fields" fixture echoes what it received to stderr, which + // run_plugin returns. The interface promises no field order, but the + // BTreeMap makes the host's order name-sorted in practice. + let lines = run_plugin(inv).expect("plugin should succeed"); + assert_eq!(lines, vec!["audience=world,greeting=hi".to_string()]); } #[test] diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 4093ef226..63fc360fd 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -19,22 +19,19 @@ impl Guest for TestPlugin { println!("stdout from plugin"); Ok(()) } - // Echo the fields back so the host can assert on ordering and - // values; fail if the expected field is missing. "fields" => { + if !input.fields.iter().any(|f| f.name == "greeting") { + return Err("missing 'greeting' field".to_string()); + } + // Echo the fields back so the host can assert on what arrived. let rendered = input .fields .iter() .map(|f| format!("{}={}", f.name, f.value)) .collect::>() .join(","); - match input.fields.iter().find(|f| f.name == "greeting") { - Some(_) => { - eprintln!("{rendered}"); - Ok(()) - } - None => Err("missing 'greeting' field".to_string()), - } + eprintln!("{rendered}"); + Ok(()) } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index da28b7288..eee54a1b4 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -99,12 +99,7 @@ pub(super) async fn sync( let base_dir = Utf8PathBuf::from(params.path.as_str()); let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); let files: Vec = adapter.files.clone().unwrap_or_default(); - let fields: BTreeMap = adapter - .fields - .clone() - .unwrap_or_default() - .into_iter() - .collect(); + let fields: BTreeMap = adapter.fields.clone().unwrap_or_default(); // 3. Build the canister ID table exposed to the plugin, then resolve the // step's `canisters` list against it. diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index e0daaa664..c84ab6e3c 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,10 +1,83 @@ -use std::collections::HashMap; +use std::{collections::BTreeMap, fmt}; use schemars::JsonSchema; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{ + Deserialize, Deserializer, Serialize, + de::{self, Visitor}, +}; use super::prebuilt::SourceField; +/// One `fields:` value on its way in from the manifest. A plugin always receives +/// a string, but writing `port: 8080` should not have to be quoted, so any YAML +/// scalar is accepted and stringified. Lists, mappings, and empty values are +/// rejected: there is no string to hand the plugin. +/// +/// Note this cannot be left to serde's own `String` handling. `serde_yaml` +/// coerces scalars when deserializing straight from YAML text, but the manifest +/// is parsed into a `serde_yaml::Value` first (see `CanisterManifest`'s +/// `Deserialize`), and re-deserializing from a `Value` keeps a number a number. +struct FieldValue(String); + +impl<'de> Deserialize<'de> for FieldValue { + fn deserialize>(d: D) -> Result { + struct ScalarVisitor; + + impl Visitor<'_> for ScalarVisitor { + type Value = FieldValue; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, number, or boolean") + } + + fn visit_str(self, v: &str) -> Result { + Ok(FieldValue(v.to_owned())) + } + + fn visit_i64(self, v: i64) -> Result { + Ok(FieldValue(v.to_string())) + } + + fn visit_u64(self, v: u64) -> Result { + Ok(FieldValue(v.to_string())) + } + + fn visit_f64(self, v: f64) -> Result { + Ok(FieldValue(v.to_string())) + } + + fn visit_bool(self, v: bool) -> Result { + Ok(FieldValue(v.to_string())) + } + } + + d.deserialize_any(ScalarVisitor) + } +} + +impl JsonSchema for FieldValue { + fn schema_name() -> std::borrow::Cow<'static, str> { + "FieldValue".into() + } + + fn inline_schema() -> bool { + true + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": ["string", "number", "boolean"], + }) + } +} + +fn deserialize_fields<'de, D: Deserializer<'de>>( + d: D, +) -> Result>, D::Error> { + let fields = Option::>::deserialize(d)?; + Ok(fields.map(|fields| fields.into_iter().map(|(k, v)| (k, v.0)).collect())) +} + /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -24,6 +97,7 @@ use super::prebuilt::SourceField; /// - config.txt /// fields: # key-value fields passed inline /// api_url: https://example.com +/// retries: 3 /// ``` /// /// Example (remote URL — `sha256` is required): @@ -51,8 +125,10 @@ pub struct Adapter { pub files: Option>, /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. - /// Values are strings; the plugin decides how to interpret them. - pub fields: Option>, + /// A plugin receives every value as a string; a number or boolean written + /// unquoted arrives as its text form. The plugin decides how to interpret them. + #[schemars(with = "Option>")] + pub fields: Option>, /// Canisters this plugin may call in addition to the canister being synced. /// Each entry is a canister name resolved against the project's canister ID @@ -72,7 +148,8 @@ impl<'de> Deserialize<'de> for Adapter { sha256: Option, dirs: Option>, files: Option>, - fields: Option>, + #[serde(default, deserialize_with = "deserialize_fields")] + fields: Option>, canisters: Option>, } @@ -95,6 +172,8 @@ impl<'de> Deserialize<'de> for Adapter { #[cfg(test)] mod tests { + use indoc::indoc; + use super::*; use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; @@ -148,9 +227,19 @@ mod tests { ); } + /// Parse an adapter the way the manifest loader does: YAML text into a + /// `serde_yaml::Value`, then that value into the typed adapter. Going + /// through the value matters for `fields` — deserializing straight from + /// text lets `serde_yaml` coerce scalars to strings on its own, which + /// would hide whether `FieldValue` accepts them. + fn adapter_via_value(yaml: &str) -> Result { + let value: serde_yaml::Value = serde_yaml::from_str(yaml).expect("invalid yaml"); + serde_yaml::from_value(value) + } + #[test] fn fields_parse_as_a_string_map() { - let adapter = serde_yaml::from_str::( + let adapter = adapter_via_value( r#" path: plugins/my-sync.wasm fields: @@ -161,13 +250,63 @@ mod tests { .expect("failed to deserialize Adapter with fields"); assert_eq!( adapter.fields, - Some(HashMap::from([ + Some(BTreeMap::from([ ("api_url".to_string(), "https://example.com".to_string()), ("token".to_string(), "abc123".to_string()), ])), ); } + #[test] + fn scalar_field_values_are_stringified() { + let adapter = adapter_via_value( + r#" + path: plugins/my-sync.wasm + fields: + port: 8080 + enabled: true + ratio: 1.5 + "#, + ) + .expect("failed to deserialize Adapter with scalar fields"); + assert_eq!( + adapter.fields, + Some(BTreeMap::from([ + ("port".to_string(), "8080".to_string()), + ("enabled".to_string(), "true".to_string()), + ("ratio".to_string(), "1.5".to_string()), + ])), + ); + } + + #[test] + fn non_scalar_field_values_are_rejected() { + for yaml in [ + // A plugin can only receive a string, so there is nothing sensible + // to hand it for a nested mapping... + indoc! {r#" + path: plugins/my-sync.wasm + fields: + nested: + a: b + "#}, + // ...or for a key written with no value at all. + indoc! {r#" + path: plugins/my-sync.wasm + fields: + blank: + "#}, + ] { + let err = + adapter_via_value(yaml).expect_err("non-scalar field value should be rejected"); + assert!( + err.to_string() + .contains("expected a string, number, or boolean"), + "unexpected error: {err}" + ); + } + } + #[test] fn remote_url_without_sha256_is_rejected() { let err = serde_yaml::from_str::( diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 0a4428397..9b6f28282 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -124,6 +124,8 @@ for field in &input.fields { } ``` +A value always arrives as a string, so parse the ones you want as another type — a manifest may write `retries: 3` unquoted, and the plugin receives `"3"`. + ## Build ```bash @@ -147,6 +149,7 @@ sync: - config.txt fields: api_url: https://example.com + retries: 3 ``` Then run the sync phase: diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 95539c6f8..0518287d9 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -153,6 +153,7 @@ sync: - config.txt fields: # key-value fields passed inline api_url: https://example.com + retries: 3 canisters: # extra canisters the plugin may call - ledger # by name (resolved for the environment) - services/open-crm:backend @@ -175,6 +176,8 @@ sync: Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. +A plugin receives every `fields:` value as a string. Numbers and booleans need no quoting — `port: 8080` arrives as `"8080"` — but a value may not be a list, a mapping, or empty. + A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a canister defined in a subproject. A name that does not resolve to a known canister for the environment fails the sync step. The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 711169c87..4632f9cb0 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -91,9 +91,13 @@ }, "fields": { "additionalProperties": { - "type": "string" + "type": [ + "string", + "number", + "boolean" + ] }, - "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nA plugin receives every value as a string; a number or boolean written\nunquoted arrives as its text form. The plugin decides how to interpret them.", "type": [ "object", "null" diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 303db6e37..3121bf499 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -91,9 +91,13 @@ }, "fields": { "additionalProperties": { - "type": "string" + "type": [ + "string", + "number", + "boolean" + ] }, - "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nValues are strings; the plugin decides how to interpret them.", + "description": "Key-value fields passed to the plugin as part of `sync-exec-input.fields`.\nA plugin receives every value as a string; a number or boolean written\nunquoted arrives as its text form. The plugin decides how to interpret them.", "type": [ "object", "null" From bbab5a468d5bad86d7723624a942a60fcd00ccc9 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 06:32:52 -0700 Subject: [PATCH 41/51] Add key names to files/directories for plugins --- crates/icp-cli/src/operations/bundle.rs | 38 ++- crates/icp-cli/tests/sync_tests.rs | 69 +++++ crates/icp-sync-plugin/DESIGN.md | 21 +- crates/icp-sync-plugin/src/lib.rs | 2 +- crates/icp-sync-plugin/src/runtime.rs | 107 +++++-- crates/icp-sync-plugin/sync-plugin.wit | 23 +- .../tests/fixtures/test-plugin/src/lib.rs | 11 + crates/icp/src/canister/sync/plugin.rs | 25 +- crates/icp/src/manifest/adapter/plugin.rs | 278 +++++++++++++++++- crates/icp/src/manifest/canister.rs | 9 +- docs/concepts/sync-plugins.md | 6 +- docs/guides/writing-sync-plugins.md | 10 +- docs/reference/configuration.md | 18 +- docs/schemas/canister-yaml-schema.json | 67 +++-- docs/schemas/icp-yaml-schema.json | 67 +++-- examples/icp-sync-plugin/plugin/src/lib.rs | 2 +- 16 files changed, 654 insertions(+), 99 deletions(-) diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index dd90010b4..94eef4268 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -766,39 +766,49 @@ async fn prepare_plugin_step( // Plugin preopened dirs go under a `dirs/` subdir so a user-supplied dir literally named // `files` cannot collide with the `files/` area used for plugin input files. + // The declared paths are rewritten to their archive locations; each entry's + // map key is carried through unchanged. let bundle_dirs = adapter.dirs.as_ref().map(|dirs| { - dirs.iter() + dirs.entries() + .iter() .map(|d| { let manifest_path = format!( "plugins/{path_name}/{idx}/dirs/{}", - normalize_archive_dir(d) + normalize_archive_dir(&d.path) ); out.plugin_dirs.push(DirEntry { - src_path: canister_path.join(d), + src_path: canister_path.join(&d.path), archive_prefix: archive_join(prefix, &manifest_path), }); - manifest_path + plugin::NamedPath { + key: d.key.clone(), + path: manifest_path, + } }) - .collect::>() + .collect::() }); let bundle_files = adapter.files.as_ref().map(|files| { files + .entries() .iter() .map(|f| { let manifest_path = format!( "plugins/{path_name}/{idx}/files/{}", - normalize_archive_dir(f) + normalize_archive_dir(&f.path) ); out.plugin_files.push(PluginFile { - src_path: canister_path.join(f), + src_path: canister_path.join(&f.path), archive_path: archive_join(prefix, &manifest_path), canister_name: canister.name.clone(), - orig_file: f.clone(), + orig_file: f.path.clone(), }); - manifest_path + plugin::NamedPath { + key: f.key.clone(), + path: manifest_path, + } }) - .collect::>() + .collect::() }); Ok(SyncStep::Plugin(plugin::Adapter { @@ -1301,8 +1311,8 @@ fn validate_source_paths( SyncStep::Script(_) => {} SyncStep::Plugin(adapter) => { if let Some(dirs) = &adapter.dirs { - for d in dirs { - let src = canister_path.join(d); + for d in dirs.entries() { + let src = canister_path.join(&d.path); let resolved = resolve_within_project( &src, project_dir, @@ -1313,8 +1323,8 @@ fn validate_source_paths( } } if let Some(files) = &adapter.files { - for f in files { - let src = canister_path.join(f); + for f in files.entries() { + let src = canister_path.join(&f.path); resolve_within_project( &src, project_dir, diff --git a/crates/icp-cli/tests/sync_tests.rs b/crates/icp-cli/tests/sync_tests.rs index 31f72ef48..888dabdb6 100644 --- a/crates/icp-cli/tests/sync_tests.rs +++ b/crates/icp-cli/tests/sync_tests.rs @@ -465,6 +465,75 @@ async fn sync_plugin_registers_seed_data() { ); } +/// `dirs:` may be written as a map (name → path, or name → list of paths) +/// instead of a plain list. The declared paths are still preopened and traversed +/// the same way, so registration works end-to-end; this proves the map form +/// deserializes and reaches the runtime. +#[tokio::test] +async fn sync_plugin_accepts_map_form_dirs() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + + let (canister_wasm, plugin_wasm) = build_sync_plugin_example(); + + // Two directories, declared under one map key as a list. + let fruit = project_dir.join("fruit"); + let veg = project_dir.join("veg"); + create_dir_all(&fruit).expect("failed to create fruit dir"); + create_dir_all(&veg).expect("failed to create veg dir"); + write_string(&fruit.join("fruit-01.txt"), "apple").expect("failed to write fruit-01.txt"); + write_string(&veg.join("veg-01.txt"), "carrot").expect("failed to write veg-01.txt"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{canister_wasm}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: {plugin_wasm} + dirs: + produce: + - fruit + - veg + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let _g = ctx.start_network_in(&project_dir, "random-network").await; + ctx.ping_until_healthy(&project_dir, "random-network"); + + clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) + .mint_cycles(10 * TRILLION); + + ctx.icp() + .current_dir(&project_dir) + .args(["deploy", "--environment", "random-environment"]) + .assert() + .success(); + + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "call", + "my-canister", + "show", + "()", + "--query", + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout(contains("apple").and(contains("carrot"))); +} + /// A malformed `ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS` must abort the sync with an /// actionable error rather than being silently ignored. This also exercises the /// end-to-end wiring: it proves the override is actually read on the real plugin diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 8737dffab..326a9b6a5 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -82,10 +82,11 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin `CallableCanisters` before calling; this crate stays free of any manifest knowledge. -`dirs` and `files` are the manifest-relative path strings, straight from the -adapter. The runtime owns *all* filesystem access anchored at `base_dir`: it -preopens each `dir` from `base_dir.join(dir)` and reads each `file` from -`base_dir.join(file)`, passing the contents inline in `SyncExecInput`. Keeping +`dirs` and `files` are the manifest-relative paths (as `KeyedPath`s carrying the +map key each was declared under, if any), straight from the adapter. The runtime +owns *all* filesystem access anchored at `base_dir`: it preopens each `dir` from +`base_dir.join(dir.path)` and reads each `file` from `base_dir.join(file.path)`, +passing the contents — and the keys — inline in `SyncExecInput`. Keeping both inside the runtime means the path-safety logic (below) lives in one place and stays private to this crate — the CLI just forwards strings. The returned `Vec` is the plugin's persistent stderr lines (see stdio capture below); @@ -189,13 +190,21 @@ Deserializes the `canister.yaml` fields into: pub struct Adapter { pub source: SourceField, // path: or url: pub sha256: Option, - pub dirs: Option>, - pub files: Option>, + pub dirs: Option, + pub files: Option, pub fields: Option>, // inline key-value fields pub canisters: Option>, // extra callable canisters, by name } ``` +`NamedPaths` deserializes `dirs:`/`files:` from either a plain list of paths or a +map of name → path (or list of paths), flattening to an ordered list of +`(key, path)` entries: `key` is `None` for a list entry and `Some(name)` for a +map entry, and is *non-unique* — a map key that resolves to a list of paths +produces one entry per path, all sharing the key. The CLI passes these to the +runtime as `KeyedPath`s (this crate stays free of manifest types), which surface +in `sync-exec-input.dirs`/`files` as each entry's `key`. + Each `canisters:` entry is a canister name resolved against the project's ID table for the environment being synced. `Deserialize` is hand-written to reject a `url` source without a `sha256`. `fields` is a `BTreeMap` rather than a `HashMap` diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index 053be28a3..d2fa023b4 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -2,6 +2,6 @@ mod path; mod runtime; pub use runtime::{ - CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, }; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 708dc241f..719a920a9 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -56,6 +56,20 @@ mod v1 { use v2::icp::sync_plugin::types::{CallTarget, CallType, CanisterIdEntry}; +/// A manifest path passed to a plugin, tagged with the map key it was declared +/// under. Both `dirs` and `files` are lists of these. +/// +/// The key is `None` when the manifest wrote the setting as a plain list, and +/// `Some(name)` when it wrote a map. It is *non-unique*: several paths share a +/// key when a map key resolves to a list of paths. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KeyedPath { + /// The map key this path was declared under, or `None` for a plain-list entry. + pub key: Option, + /// Manifest-relative path, anchored at the invocation's `base_dir`. + pub path: String, +} + /// The canisters a sync plugin is permitted to call, beyond the canister being /// synced (which is always reachable via [`CallTarget::Host`]). /// @@ -384,10 +398,12 @@ pub struct PluginInvocation { pub wasm_path: Utf8PathBuf, /// Directory the declared `dirs`/`files` are anchored at (the canister dir). pub base_dir: Utf8PathBuf, - /// Manifest-relative directories to preopen read-only. - pub dirs: Vec, - /// Manifest-relative files to read and pass inline. - pub files: Vec, + /// Manifest-relative directories to preopen read-only, each tagged with the + /// map key it was declared under (if any). + pub dirs: Vec, + /// Manifest-relative files to read and pass inline, each tagged with the map + /// key it was declared under (if any). + pub files: Vec, /// Key-value fields to pass inline. Passed to v0.2.0 plugins; ignored by /// v0.1.0 plugins, whose interface has no `fields`. pub fields: BTreeMap, @@ -475,7 +491,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin // Preopen each declared directory read-only. The guest sees it at the // same relative path it used in the manifest. let mut wasi_builder = wasmtime_wasi::WasiCtxBuilder::new(); - for dir in &dirs { + for KeyedPath { path: dir, .. } in &dirs { ensure!(!crate::path::escapes_base(dir), UnsafeDirSnafu { dir }); // Reject symlinks in the declared path: neither the final entry nor any // intermediate component may be a symlink, so the preopen cannot escape @@ -498,10 +514,11 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin // Read each declared file on the host and pass its content inline. The same // path-safety checks as `dirs` apply: reject escaping or symlinked paths so // a read cannot leave `base_dir`. - // Held as plain (name, content) pairs so they can be converted to whichever - // interface version's `file-input` record the plugin turns out to use. - let mut file_contents: Vec<(String, String)> = Vec::with_capacity(files.len()); - for name in &files { + // Held as plain (key, name, content) triples so they can be converted to + // whichever interface version's `file-input` record the plugin turns out to + // use (v0.1.0 has no `key`, so it is dropped there). + let mut file_contents: Vec<(Option, String, String)> = Vec::with_capacity(files.len()); + for KeyedPath { key, path: name } in &files { ensure!(!crate::path::escapes_base(name), UnsafeFileSnafu { name }); if let Some(link) = crate::path::first_symlink_component(&base_dir, name) { return SymlinkFileSnafu { name, link }.fail(); @@ -509,7 +526,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let path = base_dir.join(name); let content = std::fs::read_to_string(path.as_std_path()).context(ReadFileSnafu { path })?; - file_contents.push((name.clone(), content)); + file_contents.push((key.clone(), name.clone(), content)); } let persistent_stderr: Arc>> = Arc::default(); @@ -572,10 +589,13 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let input = v2::SyncExecInput { canister_id: canister_id_text, environment, - dirs, + dirs: dirs + .into_iter() + .map(|KeyedPath { key, path }| v2::DirInput { key, path }) + .collect(), files: file_contents .into_iter() - .map(|(name, content)| v2::FileInput { name, content }) + .map(|(key, name, content)| v2::FileInput { key, name, content }) .collect(), fields: fields .into_iter() @@ -611,10 +631,14 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let input = v1::SyncExecInput { canister_id: canister_id_text, environment, - dirs, + // The v0.1.0 interface has no per-entry key; pass just the paths. + dirs: dirs + .into_iter() + .map(|KeyedPath { path, .. }| path) + .collect(), files: file_contents .into_iter() - .map(|(name, content)| v1::FileInput { name, content }) + .map(|(_key, name, content)| v1::FileInput { name, content }) .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, @@ -807,6 +831,17 @@ mod tests { Principal::anonymous() } + /// Plain (unkeyed) [`KeyedPath`]s, as a plain-list manifest entry produces. + fn unkeyed(paths: &[&str]) -> Vec { + paths + .iter() + .map(|p| KeyedPath { + key: None, + path: (*p).to_string(), + }) + .collect() + } + /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister /// and identity, no proxy, no declared callable canisters, the default /// compute limit, and the current directory as the base. Tests override @@ -896,7 +931,7 @@ mod tests { return; }; let mut inv = invocation(wasm_path, "test"); - inv.dirs = vec!["nonexistent_dir".to_string()]; + inv.dirs = unkeyed(&["nonexistent_dir"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::PreopenDir { .. }) @@ -917,7 +952,7 @@ mod tests { let mut inv = invocation(wasm_path, "test"); inv.base_dir = base.to_path_buf(); - inv.dirs = vec!["link".to_string()]; + inv.dirs = unkeyed(&["link"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::SymlinkDir { .. }) @@ -930,7 +965,7 @@ mod tests { return; }; let mut inv = invocation(wasm_path, "test"); - inv.files = vec!["nonexistent_file.txt".to_string()]; + inv.files = unkeyed(&["nonexistent_file.txt"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::ReadFile { .. }) @@ -951,7 +986,7 @@ mod tests { let mut inv = invocation(wasm_path, "test"); inv.base_dir = base.to_path_buf(); - inv.files = vec!["link.txt".to_string()]; + inv.files = unkeyed(&["link.txt"]); assert!(matches!( run_plugin(inv), Err(RunPluginError::SymlinkFile { .. }) @@ -1046,6 +1081,42 @@ mod tests { )); } + #[tokio::test(flavor = "multi_thread")] + async fn dir_and_file_keys_reach_the_plugin() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + // A real dir and file must exist: the host preopens the dir and reads + // the file before calling exec(). + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let base = tmp.path(); + std::fs::create_dir_all(base.join("seeds")).expect("create dir"); + std::fs::write(base.join("cfg.txt"), b"data").expect("write file"); + + let (tx, mut rx) = tokio::sync::mpsc::channel::(16); + let result = tokio::task::block_in_place(|| { + let mut inv = invocation(wasm_path, "keys"); + inv.base_dir = base.to_path_buf(); + inv.dirs = vec![KeyedPath { + key: Some("assets".to_string()), + path: "seeds".to_string(), + }]; + inv.files = vec![KeyedPath { + key: None, + path: "cfg.txt".to_string(), + }]; + inv.stdio = Some(tx); + run_plugin(inv) + }); + let lines = result.expect("plugin should succeed"); + assert_eq!( + lines, + vec!["dir assets=seeds".to_string(), "file -=cfg.txt".to_string()], + ); + // The same lines are forwarded live to the rolling-view channel. + assert!(rx.try_recv().is_ok()); + } + #[test] fn legacy_v1_plugin_is_detected_and_driven() { // A plugin built against the v0.1.0 interface must still load: the host diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index a19c609f2..d5a4481cc 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -5,8 +5,23 @@ interface types { /// Whether a canister call is an update or a query. enum call-type { update, query } + /// A directory the host preopened on behalf of the plugin. + record dir-input { + /// The map key this directory was declared under in the manifest, or + /// `none` when `dirs` was written as a plain list. Several entries share + /// one key when a key maps to a list of directories. + key: option, + /// Path of the directory as declared in the manifest (relative to the + /// canister directory). The host preopens it at this same path. + path: string, + } + /// A file the host read on behalf of the plugin. record file-input { + /// The map key this file was declared under in the manifest, or `none` + /// when `files` was written as a plain list. Several entries share one + /// key when a key maps to a list of files. + key: option, /// Path of the file as declared in the manifest (relative to /// the canister directory). name: string, @@ -69,9 +84,13 @@ interface types { /// Directories declared in the manifest step's `dirs` setting. /// The host preopens each entry via WASI; the plugin can traverse /// them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). - dirs: list, + /// Each entry carries the map key it was declared under, if any (see + /// `dir-input`). + dirs: list, /// Files declared in the manifest step's `files` setting, read by /// the host and passed inline. The plugin decides how to use them. + /// Each entry carries the map key it was declared under, if any (see + /// `file-input`). files: list, /// Key-value fields declared in the manifest step's `fields` setting, /// passed inline. The plugin decides how to use them. @@ -118,7 +137,7 @@ interface types { /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, file-input, field-input}; + use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, dir-input, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 63fc360fd..2c7db0cb7 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -33,6 +33,17 @@ impl Guest for TestPlugin { eprintln!("{rendered}"); Ok(()) } + // Echo each dir/file entry as `kind key=path`, using "-" for an + // absent key, so the host can assert keys survive the boundary. + "keys" => { + for dir in &input.dirs { + eprintln!("dir {}={}", dir.key.as_deref().unwrap_or("-"), dir.path); + } + for file in &input.files { + eprintln!("file {}={}", file.key.as_deref().unwrap_or("-"), file.name); + } + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index eee54a1b4..60ce522bc 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -4,13 +4,30 @@ use camino::Utf8PathBuf; use candid::Principal; use ic_agent::Agent; use icp_sync_plugin::{ - CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, PLUGIN_COMPUTE_LIMIT_ENV, + CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, }; use snafu::prelude::*; use tokio::sync::mpsc::Sender; -use crate::{canister::wasm, manifest::adapter::plugin::Adapter, package::PackageCache}; +use crate::{ + canister::wasm, + manifest::adapter::plugin::{Adapter, NamedPaths}, + package::PackageCache, +}; + +/// Convert a manifest [`NamedPaths`] (or its absence) into the runtime's +/// key-tagged path list. A missing setting yields an empty list. +fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { + paths + .into_iter() + .flat_map(NamedPaths::entries) + .map(|entry| KeyedPath { + key: entry.key.clone(), + path: entry.path.clone(), + }) + .collect() +} use super::Params; @@ -97,8 +114,8 @@ pub(super) async fn sync( // subject to the runtime's path-safety checks (no escaping or symlinked // paths). let base_dir = Utf8PathBuf::from(params.path.as_str()); - let dirs: Vec = adapter.dirs.clone().unwrap_or_default(); - let files: Vec = adapter.files.clone().unwrap_or_default(); + let dirs = keyed_paths(adapter.dirs.as_ref()); + let files = keyed_paths(adapter.files.as_ref()); let fields: BTreeMap = adapter.fields.clone().unwrap_or_default(); // 3. Build the canister ID table exposed to the plugin, then resolve the diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index c84ab6e3c..56a0fb8c5 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,9 +1,14 @@ -use std::{collections::BTreeMap, fmt}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt, +}; +use indexmap::IndexMap; use schemars::JsonSchema; use serde::{ - Deserialize, Deserializer, Serialize, + Deserialize, Deserializer, Serialize, Serializer, de::{self, Visitor}, + ser::SerializeMap, }; use super::prebuilt::SourceField; @@ -78,6 +83,156 @@ fn deserialize_fields<'de, D: Deserializer<'de>>( Ok(fields.map(|fields| fields.into_iter().map(|(k, v)| (k, v.0)).collect())) } +/// A single manifest path together with the map key it was declared under. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NamedPath { + /// The map key this path was declared under, or `None` for a plain-list + /// entry. Non-unique: several paths share a key when the key maps to a list. + pub key: Option, + /// The path itself, relative to the canister directory. + pub path: String, +} + +/// A set of manifest paths declared either as a plain list or as a map of +/// name → path(s). Used for a plugin step's `dirs` and `files`. +/// +/// In `canister.yaml` this accepts three shapes: +/// ```yaml +/// # a plain list — entries carry no key +/// files: +/// - config.txt +/// - data.json +/// # a map whose keys each name a single path... +/// files: +/// main: config.txt +/// # ...or a list of paths, which all share that key +/// files: +/// seeds: +/// - a.json +/// - b.json +/// ``` +/// +/// Order is preserved: list entries in written order; map entries in written +/// key order, each key's paths in written order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NamedPaths(Vec); + +/// A list of paths, or a map of name → path (or list of paths). The map form +/// tags each path with its key for the plugin; a key may map to several paths. +/// +/// This type exists only to describe [`NamedPaths`] in the generated JSON schema +/// (see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`] +/// owns the actual (de)serialization. +#[derive(JsonSchema)] +#[serde(untagged)] +#[allow(dead_code)] +enum NamedPathsSchema { + List(Vec), + Map(HashMap), +} + +/// One map value in [`NamedPathsSchema`]: a single path, or a list of paths that +/// share the key. +#[derive(JsonSchema)] +#[serde(untagged)] +#[allow(dead_code)] +enum PathOrListSchema { + One(String), + Many(Vec), +} + +impl NamedPaths { + /// Build from an ordered list of key-tagged paths. + pub fn from_entries(entries: Vec) -> Self { + NamedPaths(entries) + } + + /// The declared paths, in order, each tagged with its map key (if any). + pub fn entries(&self) -> &[NamedPath] { + &self.0 + } + + /// Consume into the ordered list of key-tagged paths. + pub fn into_entries(self) -> Vec { + self.0 + } +} + +impl FromIterator for NamedPaths { + fn from_iter>(iter: I) -> Self { + NamedPaths(iter.into_iter().collect()) + } +} + +impl<'de> Deserialize<'de> for NamedPaths { + fn deserialize>(d: D) -> Result { + /// A map value: a single path or a list of paths sharing the key. + #[derive(Deserialize)] + #[serde(untagged)] + enum PathOrList { + One(String), + Many(Vec), + } + + #[derive(Deserialize)] + #[serde(untagged)] + enum Repr { + List(Vec), + Map(IndexMap), + } + + let entries = match Repr::deserialize(d)? { + Repr::List(paths) => paths + .into_iter() + .map(|path| NamedPath { key: None, path }) + .collect(), + Repr::Map(map) => map + .into_iter() + .flat_map(|(key, value)| { + let paths = match value { + PathOrList::One(path) => vec![path], + PathOrList::Many(paths) => paths, + }; + paths.into_iter().map(move |path| NamedPath { + key: Some(key.clone()), + path, + }) + }) + .collect(), + }; + Ok(NamedPaths(entries)) + } +} + +impl Serialize for NamedPaths { + fn serialize(&self, s: S) -> Result { + // Deserialization yields either all-unkeyed (list form) or all-keyed + // (map form) entries; serialize back to whichever it was. + if self.0.iter().all(|e| e.key.is_none()) { + let paths: Vec<&str> = self.0.iter().map(|e| e.path.as_str()).collect(); + paths.serialize(s) + } else { + // Group paths by key, preserving order. A key with one path + // serializes as a scalar; multiple as a list. + let mut groups: IndexMap<&str, Vec<&str>> = IndexMap::new(); + for e in &self.0 { + groups + .entry(e.key.as_deref().unwrap_or_default()) + .or_default() + .push(&e.path); + } + let mut map = s.serialize_map(Some(groups.len()))?; + for (key, paths) in groups { + match paths.as_slice() { + [one] => map.serialize_entry(key, one)?, + many => map.serialize_entry(key, many)?, + } + } + map.end() + } + } +} + /// Configuration for a sync plugin step. /// /// A sync plugin is a WebAssembly module invoked during `icp sync` for a @@ -100,6 +255,18 @@ fn deserialize_fields<'de, D: Deserializer<'de>>( /// retries: 3 /// ``` /// +/// `dirs` and `files` may instead be written as a map, tagging each entry with a +/// `key` surfaced to the plugin; a key may map to a single path or a list: +/// ```yaml +/// - type: plugin +/// path: ./plugins/populate-data.wasm +/// dirs: +/// seed: assets/seed-data # keyed single path +/// migrations: # keyed list — entries share the key +/// - migrations/2025 +/// - migrations/2026 +/// ``` +/// /// Example (remote URL — `sha256` is required): /// ```yaml /// - type: plugin @@ -117,12 +284,18 @@ pub struct Adapter { /// Directories (relative to canister directory) the plugin may read from. /// Each entry must be a directory; it is preopened via WASI so the plugin - /// can traverse it using standard filesystem APIs. - pub dirs: Option>, + /// can traverse it using standard filesystem APIs. Written as a plain list + /// of paths, or as a map of name → path (or list of paths); the name is + /// surfaced to the plugin as each entry's `key`. + #[schemars(with = "Option")] + pub dirs: Option, /// Files (relative to canister directory) the host reads and passes to - /// the plugin as part of `sync-exec-input.files`. - pub files: Option>, + /// the plugin as part of `sync-exec-input.files`. Written as a plain list + /// of paths, or as a map of name → path (or list of paths); the name is + /// surfaced to the plugin as each entry's `key`. + #[schemars(with = "Option")] + pub files: Option, /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. /// A plugin receives every value as a string; a number or boolean written @@ -146,8 +319,8 @@ impl<'de> Deserialize<'de> for Adapter { #[serde(flatten)] source: SourceField, sha256: Option, - dirs: Option>, - files: Option>, + dirs: Option, + files: Option, #[serde(default, deserialize_with = "deserialize_fields")] fields: Option>, canisters: Option>, @@ -175,6 +348,27 @@ mod tests { use indoc::indoc; use super::*; + + /// [`NamedPaths`] with no keys, as a plain-list manifest entry produces. + fn unkeyed(paths: [&str; N]) -> NamedPaths { + NamedPaths::from_entries( + paths + .into_iter() + .map(|path| NamedPath { + key: None, + path: path.to_string(), + }) + .collect(), + ) + } + + /// A single key-tagged [`NamedPath`]. + fn keyed(key: &str, path: &str) -> NamedPath { + NamedPath { + key: Some(key.to_string()), + path: path.to_string(), + } + } use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; #[test] @@ -219,8 +413,8 @@ mod tests { path: "plugins/my-sync.wasm".into(), }), sha256: Some("abc123".to_string()), - dirs: Some(vec!["assets/seed-data".to_string(), "config".to_string()]), - files: Some(vec!["config.txt".to_string()]), + dirs: Some(unkeyed(["assets/seed-data", "config"])), + files: Some(unkeyed(["config.txt"])), fields: None, canisters: None, }, @@ -237,6 +431,70 @@ mod tests { serde_yaml::from_value(value) } + /// The list form leaves every entry keyless. + #[test] + fn dirs_and_files_as_plain_lists_have_no_keys() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + dirs: + - assets + files: + - a.txt + - b.txt + "#, + ) + .expect("failed to deserialize Adapter with list dirs/files"); + assert_eq!(adapter.dirs, Some(unkeyed(["assets"]))); + assert_eq!(adapter.files, Some(unkeyed(["a.txt", "b.txt"]))); + } + + /// The map form tags each entry with its key. A key mapping to a list yields + /// several entries sharing that (non-unique) key, in written order. + #[test] + fn dirs_and_files_as_maps_carry_keys() { + let adapter = serde_yaml::from_str::( + r#" + path: plugins/my-sync.wasm + dirs: + seed: assets/seed-data + extra: + - one + - two + files: + main: config.txt + "#, + ) + .expect("failed to deserialize Adapter with map dirs/files"); + assert_eq!( + adapter.dirs.map(NamedPaths::into_entries), + Some(vec![ + keyed("seed", "assets/seed-data"), + keyed("extra", "one"), + keyed("extra", "two"), + ]), + ); + assert_eq!( + adapter.files.map(NamedPaths::into_entries), + Some(vec![keyed("main", "config.txt")]), + ); + } + + /// The list and map forms round-trip through serialization back to their + /// natural YAML shape. + #[test] + fn named_paths_round_trip() { + for yaml in [ + "- a.txt\n- b.txt\n", + "single: one.txt\nmany:\n- x.txt\n- y.txt\n", + ] { + let parsed: NamedPaths = + serde_yaml::from_str(yaml).expect("failed to parse NamedPaths"); + let reserialized = serde_yaml::to_string(&parsed).expect("failed to serialize"); + assert_eq!(reserialized, yaml, "round-trip changed the YAML shape"); + } + } + #[test] fn fields_parse_as_a_string_map() { let adapter = adapter_via_value( diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 28a576f9b..23c9ca853 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -791,7 +791,14 @@ mod tests { path: "./plugins/my-sync.wasm".into(), }), sha256: None, - dirs: Some(vec!["assets/seed-data/".to_string()]), + dirs: Some( + crate::manifest::adapter::plugin::NamedPaths::from_entries( + vec![crate::manifest::adapter::plugin::NamedPath { + key: None, + path: "assets/seed-data/".to_string(), + }], + ) + ), files: None, fields: None, canisters: None, diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index a4b027dbb..853cab13e 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -67,8 +67,8 @@ The authoritative interface, including all record fields, lives in [`sync-plugin |-------|-------------| | `canister-id` | Textual principal of the canister being synced | | `environment` | Name of the environment being synced (e.g. `local`, `production`) | -| `dirs` | The directories you declared in `dirs:`; the host preopened each one read-only | -| `files` | The files you declared in `files:`, each as a `(name, content)` pair read by the host | +| `dirs` | The directories you declared in `dirs:`; the host preopened each one read-only. Each entry carries its `key` (see below) and `path` | +| `files` | The files you declared in `files:`, each with its `key`, `name` (path), and `content` read by the host | | `fields` | The key-value fields you declared in `fields:`, each as a `(name, value)` pair; values are strings | | `identity-principal` | Textual principal of the signing identity used for canister calls | | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | @@ -76,6 +76,8 @@ The authoritative interface, including all record fields, lives in [`sync-plugin Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister defined in a subproject. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. +`dirs` and `files` each carry a `key`: the map key the entry was declared under in the manifest, or absent when `dirs:`/`files:` was written as a plain list. A key that maps to a list of paths produces several entries sharing that key, so the key is not unique. Use it to group or label declared paths — e.g. distinguish `seed:` directories from `migrations:` directories — without hardcoding paths in the plugin. + ### Calling a canister — `canister-call` The plugin calls methods through the `canister-call` import. It picks a `target`, supplies the method name, **Candid-encoded argument bytes** (the host forwards them unchanged), and a few routing options: diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 9b6f28282..8e466b585 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -92,19 +92,19 @@ A few things to note: A plugin can't see the filesystem freely — only what you grant it in the manifest's `dirs:` and `files:`. -Directories in `dirs:` are preopened read-only at the same relative path. Traverse them with standard `std::fs`: +Directories in `dirs:` are preopened read-only at the same relative path. Each entry gives you its `path` plus a `key` (the map key it was declared under, or `None` for a plain-list entry). Traverse them with standard `std::fs`: ```rust for dir in &input.dirs { - for entry in std::fs::read_dir(dir).map_err(|e| e.to_string())? { + for entry in std::fs::read_dir(&dir.path).map_err(|e| e.to_string())? { let path = entry.map_err(|e| e.to_string())?.path(); let content = std::fs::read_to_string(&path).map_err(|e| e.to_string())?; - // ... encode and send to the canister ... + // ... encode and send to the canister; dir.key groups related dirs ... } } ``` -Files in `files:` are read by the host up front and passed inline — read them from the input struct, not from disk: +Files in `files:` are read by the host up front and passed inline — read them from the input struct, not from disk. Each entry carries its `key`, `name` (the path), and `content`: ```rust for file in &input.files { @@ -112,6 +112,8 @@ for file in &input.files { } ``` +Declaring `dirs:`/`files:` as a map instead of a list tags each entry with a `key`, so a plugin can group or label paths (for example, tell `seed:` directories from `migrations:`) without hardcoding paths. A key that maps to a list of paths yields several entries sharing that key. + Writes, paths outside a preopen, and `..` traversal are all rejected by the sandbox. See [The Sandbox](../concepts/sync-plugins.md#the-sandbox) for the full capability list and resource limits. ## Read Declared Fields diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0518287d9..d91e58c77 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -169,11 +169,25 @@ sync: | `path` | string | One of `path` or `url` | Local path to the wasm, relative to the canister directory | | `url` | string | One of `path` or `url` | URL to download the wasm from | | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | -| `dirs` | array of string | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | -| `files` | array of string | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | +| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | +| `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | | `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | | `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | +`dirs:` and `files:` each accept either a plain list of paths or a map. As a map, each key names a single path or a list of paths, and the key is surfaced to the plugin as that entry's `key` (a key mapping to a list produces several entries sharing it). A plain-list entry has no key. For example: + +```yaml + - type: plugin + path: ./plugins/populate-data.wasm + dirs: + seed: assets/seed-data # one path under a key + migrations: # several paths sharing a key + - migrations/2025 + - migrations/2026 + files: + - config.txt # a plain list is still fine +``` + Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. A plugin receives every `fields:` value as a string. Numbers and booleans need no quoting — `port: 8080` arrives as `"8080"` — but a value may not be a list, a mapping, or empty. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 4632f9cb0..a5c7e262d 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -80,14 +80,15 @@ ] }, "dirs": { - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "fields": { "additionalProperties": { @@ -104,14 +105,15 @@ ] }, "files": { - "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "sha256": { "description": "Optional sha256 checksum of the wasm file.\nOptional for `path`; required for `url`.", @@ -339,6 +341,37 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, + "NamedPathsSchema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "additionalProperties": { + "$ref": "#/$defs/PathOrListSchema" + }, + "type": "object" + } + ], + "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + }, + "PathOrListSchema": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + }, "Recipe": { "properties": { "configuration": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 3121bf499..4121c9bb4 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -80,14 +80,15 @@ ] }, "dirs": { - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "fields": { "additionalProperties": { @@ -104,14 +105,15 @@ ] }, "files": { - "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`.", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] + "anyOf": [ + { + "$ref": "#/$defs/NamedPathsSchema" + }, + { + "type": "null" + } + ], + "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." }, "sha256": { "description": "Optional sha256 checksum of the wasm file.\nOptional for `path`; required for `url`.", @@ -795,6 +797,23 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, + "NamedPathsSchema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "additionalProperties": { + "$ref": "#/$defs/PathOrListSchema" + }, + "type": "object" + } + ], + "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + }, "NetworkManifest": { "description": "A network definition for the project", "oneOf": [ @@ -835,6 +854,20 @@ ], "type": "object" }, + "PathOrListSchema": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + }, "Recipe": { "properties": { "configuration": { diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index f6d7534bd..c3fdee356 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -38,7 +38,7 @@ impl Guest for Plugin { // uploader principal, which is the current identity — not the proxy. let mut registered = 0u32; for dir in &input.dirs { - registered += register_dir(Path::new(dir))?; + registered += register_dir(Path::new(&dir.path))?; } // Persisted after the step completes; use stderr. From 91be38ac449bba93783fec30f387ccd62e72042f Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 20 Aug 2026 07:21:32 -0700 Subject: [PATCH 42/51] fixes --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/icp-cli/src/operations/bundle.rs | 73 +++---- crates/icp-cli/tests/bundle_tests.rs | 102 +++++++++ crates/icp-sync-plugin/DESIGN.md | 16 +- crates/icp-sync-plugin/src/runtime.rs | 26 ++- crates/icp/src/canister/sync/plugin.rs | 8 +- crates/icp/src/manifest/adapter/plugin.rs | 242 +++++++++------------- crates/icp/src/manifest/canister.rs | 41 ++-- docs/schemas/canister-yaml-schema.json | 18 +- docs/schemas/icp-yaml-schema.json | 18 +- 11 files changed, 304 insertions(+), 243 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 330024485..9ca77dd4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6357,6 +6357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", + "indexmap", "ref-cast", "schemars_derive", "serde", diff --git a/Cargo.toml b/Cargo.toml index 30710a835..60dc41e1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,7 @@ pkcs8 = { version = "0.10.2", features = ["encryption", "std"] } rand = "0.10.1" regex = "1.12.2" reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "json", "stream"] } -schemars = { version = "1.0.4", features = ["derive", "url2"] } +schemars = { version = "1.0.4", features = ["derive", "indexmap2", "url2"] } scrypt = "0.11.0" sec1 = { version = "0.7.3", features = ["pkcs8"] } send_ctrlc = "0.6" diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 94eef4268..5bd881d9e 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -769,49 +769,36 @@ async fn prepare_plugin_step( // The declared paths are rewritten to their archive locations; each entry's // map key is carried through unchanged. let bundle_dirs = adapter.dirs.as_ref().map(|dirs| { - dirs.entries() - .iter() - .map(|d| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/dirs/{}", - normalize_archive_dir(&d.path) - ); - out.plugin_dirs.push(DirEntry { - src_path: canister_path.join(&d.path), - archive_prefix: archive_join(prefix, &manifest_path), - }); - plugin::NamedPath { - key: d.key.clone(), - path: manifest_path, - } - }) - .collect::() + dirs.map_paths(|dir| { + let manifest_path = format!( + "plugins/{path_name}/{idx}/dirs/{}", + normalize_archive_dir(dir) + ); + out.plugin_dirs.push(DirEntry { + src_path: canister_path.join(dir), + archive_prefix: archive_join(prefix, &manifest_path), + }); + manifest_path + }) }); let bundle_files = adapter.files.as_ref().map(|files| { - files - .entries() - .iter() - .map(|f| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/files/{}", - normalize_archive_dir(&f.path) - ); - out.plugin_files.push(PluginFile { - src_path: canister_path.join(&f.path), - archive_path: archive_join(prefix, &manifest_path), - canister_name: canister.name.clone(), - orig_file: f.path.clone(), - }); - plugin::NamedPath { - key: f.key.clone(), - path: manifest_path, - } - }) - .collect::() + files.map_paths(|file| { + let manifest_path = format!( + "plugins/{path_name}/{idx}/files/{}", + normalize_archive_dir(file) + ); + out.plugin_files.push(PluginFile { + src_path: canister_path.join(file), + archive_path: archive_join(prefix, &manifest_path), + canister_name: canister.name.clone(), + orig_file: file.to_string(), + }); + manifest_path + }) }); - Ok(SyncStep::Plugin(plugin::Adapter { + Ok(SyncStep::Plugin(Box::new(plugin::Adapter { source: SourceField::Local(LocalSource { path: plugin_wasm_path.as_str().into(), }), @@ -820,7 +807,7 @@ async fn prepare_plugin_step( files: bundle_files, canisters: localize_call_targets(adapter.canisters.as_deref(), local_names), fields: adapter.fields.clone(), - })) + }))) } async fn inline_networks( @@ -1311,8 +1298,8 @@ fn validate_source_paths( SyncStep::Script(_) => {} SyncStep::Plugin(adapter) => { if let Some(dirs) = &adapter.dirs { - for d in dirs.entries() { - let src = canister_path.join(&d.path); + for dir in dirs.entries() { + let src = canister_path.join(dir.path); let resolved = resolve_within_project( &src, project_dir, @@ -1323,8 +1310,8 @@ fn validate_source_paths( } } if let Some(files) = &adapter.files { - for f in files.entries() { - let src = canister_path.join(&f.path); + for file in files.entries() { + let src = canister_path.join(file.path); resolve_within_project( &src, project_dir, diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 1500ec340..66feb2c84 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1281,6 +1281,108 @@ fn bundle_preserves_plugin_call_targets() { ); } +/// Map-form `dirs:`/`files:` must survive bundling: the paths are rewritten to their +/// archive locations, but each stays under the key it was declared with, so a plugin sees +/// the same keys whether it runs from the project or from the bundle. +#[test] +fn bundle_preserves_plugin_path_keys() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let plugin_bytes: &[u8] = b"\x00asm\x01\x00\x00\x00plugin"; + write(&project_dir.join("plugin.wasm"), plugin_bytes).expect("failed to write plugin"); + + for (dir, file) in [("seed", "s.txt"), ("m2025", "a.txt"), ("m2026", "b.txt")] { + let path = project_dir.join(dir); + create_dir_all(&path).expect("failed to create dir"); + write_string(&path.join(file), "data").expect("failed to write file"); + } + write_string(&project_dir.join("config.toml"), "key=value").expect("failed to write config"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: plugin.wasm + dirs: + seed: seed + migrations: + - m2025 + - m2026 + files: + main: config.toml + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut archived: Vec = Vec::new(); + let mut manifest_yaml = String::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path == "icp.yaml" { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + archived.push(path); + } + + for expected in [ + "plugins/my-canister/0/dirs/seed/s.txt", + "plugins/my-canister/0/dirs/m2025/a.txt", + "plugins/my-canister/0/dirs/m2026/b.txt", + "plugins/my-canister/0/files/config.toml", + ] { + assert!( + archived.iter().any(|path| path == expected), + "{expected} not found in bundle; archive holds {archived:?}" + ); + } + + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let step = &parsed["canisters"][0]["sync"]["steps"][0]; + assert_eq!( + step["dirs"]["seed"].as_str(), + Some("plugins/my-canister/0/dirs/seed") + ); + assert_eq!( + step["dirs"]["migrations"][0].as_str(), + Some("plugins/my-canister/0/dirs/m2025") + ); + assert_eq!( + step["dirs"]["migrations"][1].as_str(), + Some("plugins/my-canister/0/dirs/m2026") + ); + assert_eq!( + step["files"]["main"].as_str(), + Some("plugins/my-canister/0/files/config.toml") + ); +} + /// An `icp_appmanifest.yaml` next to the project manifest must be included in the bundle, with its /// top-level `images` paths relocated under a top-level `images/` folder and the /// referenced image files copied alongside. Unrelated metadata is preserved. diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 326a9b6a5..27617fc14 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -197,13 +197,15 @@ pub struct Adapter { } ``` -`NamedPaths` deserializes `dirs:`/`files:` from either a plain list of paths or a -map of name → path (or list of paths), flattening to an ordered list of -`(key, path)` entries: `key` is `None` for a list entry and `Some(name)` for a -map entry, and is *non-unique* — a map key that resolves to a list of paths -produces one entry per path, all sharing the key. The CLI passes these to the -runtime as `KeyedPath`s (this crate stays free of manifest types), which surface -in `sync-exec-input.dirs`/`files` as each entry's `key`. +`NamedPaths` is an untagged `List(Vec) | Map(IndexMap)` +— the two shapes `dirs:`/`files:` may be written in — keeping the written form +exact, so bundling can rewrite the paths (`map_paths`) and serialize the step +back out unchanged in shape. `entries()` flattens either form to ordered +`(key, path)` pairs: `key` is `None` for a list entry and `Some(name)` for a map +entry, and is *non-unique* — a map key holding a list of paths yields one entry +per path, all sharing the key. The CLI passes those to the runtime as +`KeyedPath`s (this crate stays free of manifest types), which surface in +`sync-exec-input.dirs`/`files` as each entry's `key`. Each `canisters:` entry is a canister name resolved against the project's ID table for the environment being synced. `Deserialize` is hand-written to reject a diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 719a920a9..ae3d76f91 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -70,6 +70,16 @@ pub struct KeyedPath { pub path: String, } +/// A declared file the host read: the key and path it was declared under, plus +/// its content. Held version-agnostically so it can be converted to whichever +/// interface version's `file-input` record the plugin turns out to use — the +/// v0.1.0 record has no `key`, so it is dropped there. +struct FileContent { + key: Option, + name: String, + content: String, +} + /// The canisters a sync plugin is permitted to call, beyond the canister being /// synced (which is always reachable via [`CallTarget::Host`]). /// @@ -514,10 +524,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin // Read each declared file on the host and pass its content inline. The same // path-safety checks as `dirs` apply: reject escaping or symlinked paths so // a read cannot leave `base_dir`. - // Held as plain (key, name, content) triples so they can be converted to - // whichever interface version's `file-input` record the plugin turns out to - // use (v0.1.0 has no `key`, so it is dropped there). - let mut file_contents: Vec<(Option, String, String)> = Vec::with_capacity(files.len()); + let mut file_contents: Vec = Vec::with_capacity(files.len()); for KeyedPath { key, path: name } in &files { ensure!(!crate::path::escapes_base(name), UnsafeFileSnafu { name }); if let Some(link) = crate::path::first_symlink_component(&base_dir, name) { @@ -526,7 +533,11 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let path = base_dir.join(name); let content = std::fs::read_to_string(path.as_std_path()).context(ReadFileSnafu { path })?; - file_contents.push((key.clone(), name.clone(), content)); + file_contents.push(FileContent { + key: key.clone(), + name: name.clone(), + content, + }); } let persistent_stderr: Arc>> = Arc::default(); @@ -595,7 +606,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin .collect(), files: file_contents .into_iter() - .map(|(key, name, content)| v2::FileInput { key, name, content }) + .map(|FileContent { key, name, content }| v2::FileInput { key, name, content }) .collect(), fields: fields .into_iter() @@ -636,9 +647,10 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin .into_iter() .map(|KeyedPath { path, .. }| path) .collect(), + // The v0.1.0 `file-input` record has no `key`; drop it. files: file_contents .into_iter() - .map(|(_key, name, content)| v1::FileInput { name, content }) + .map(|FileContent { name, content, .. }| v1::FileInput { name, content }) .collect(), identity_principal: identity_text, proxy_canister_id: proxy_text, diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 60ce522bc..fbde4dde4 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -16,6 +16,8 @@ use crate::{ package::PackageCache, }; +use super::Params; + /// Convert a manifest [`NamedPaths`] (or its absence) into the runtime's /// key-tagged path list. A missing setting yields an empty list. fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { @@ -23,14 +25,12 @@ fn keyed_paths(paths: Option<&NamedPaths>) -> Vec { .into_iter() .flat_map(NamedPaths::entries) .map(|entry| KeyedPath { - key: entry.key.clone(), - path: entry.path.clone(), + key: entry.key.map(str::to_string), + path: entry.path.to_string(), }) .collect() } -use super::Params; - #[derive(Debug, Snafu)] pub enum PluginError { #[snafu(transparent)] diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 56a0fb8c5..d3814f7d3 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -1,14 +1,11 @@ -use std::{ - collections::{BTreeMap, HashMap}, - fmt, -}; +use std::{collections::BTreeMap, fmt}; use indexmap::IndexMap; +use itertools::Either; use schemars::JsonSchema; use serde::{ - Deserialize, Deserializer, Serialize, Serializer, + Deserialize, Deserializer, Serialize, de::{self, Visitor}, - ser::SerializeMap, }; use super::prebuilt::SourceField; @@ -83,20 +80,9 @@ fn deserialize_fields<'de, D: Deserializer<'de>>( Ok(fields.map(|fields| fields.into_iter().map(|(k, v)| (k, v.0)).collect())) } -/// A single manifest path together with the map key it was declared under. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct NamedPath { - /// The map key this path was declared under, or `None` for a plain-list - /// entry. Non-unique: several paths share a key when the key maps to a list. - pub key: Option, - /// The path itself, relative to the canister directory. - pub path: String, -} - -/// A set of manifest paths declared either as a plain list or as a map of -/// name → path(s). Used for a plugin step's `dirs` and `files`. +/// The paths declared for a plugin step's `dirs` or `files`: either a plain list +/// of paths, or a map of name → path(s) whose keys are surfaced to the plugin. /// -/// In `canister.yaml` this accepts three shapes: /// ```yaml /// # a plain list — entries carry no key /// files: @@ -105,130 +91,90 @@ pub struct NamedPath { /// # a map whose keys each name a single path... /// files: /// main: config.txt -/// # ...or a list of paths, which all share that key +/// # ...or a list of paths, which then all share that key /// files: /// seeds: /// - a.json /// - b.json /// ``` /// -/// Order is preserved: list entries in written order; map entries in written -/// key order, each key's paths in written order. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct NamedPaths(Vec); - -/// A list of paths, or a map of name → path (or list of paths). The map form -/// tags each path with its key for the plugin; a key may map to several paths. -/// -/// This type exists only to describe [`NamedPaths`] in the generated JSON schema -/// (see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`] -/// owns the actual (de)serialization. -#[derive(JsonSchema)] +/// Order is preserved in both forms: list entries in written order; map entries +/// in written key order, each key's paths in written order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] -#[allow(dead_code)] -enum NamedPathsSchema { +pub enum NamedPaths { + /// A plain list of paths, carrying no keys. List(Vec), - Map(HashMap), + /// A map of name → path(s), tagging each path with the key it sits under. + Map(IndexMap), } -/// One map value in [`NamedPathsSchema`]: a single path, or a list of paths that -/// share the key. -#[derive(JsonSchema)] +/// One value of a [`NamedPaths::Map`]: a single path, or a list of paths that +/// all share the key. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] -#[allow(dead_code)] -enum PathOrListSchema { +pub enum PathOrList { + /// A single path under the key. One(String), + /// Several paths, all sharing the key. Many(Vec), } -impl NamedPaths { - /// Build from an ordered list of key-tagged paths. - pub fn from_entries(entries: Vec) -> Self { - NamedPaths(entries) - } - - /// The declared paths, in order, each tagged with its map key (if any). - pub fn entries(&self) -> &[NamedPath] { - &self.0 - } - - /// Consume into the ordered list of key-tagged paths. - pub fn into_entries(self) -> Vec { - self.0 - } -} - -impl FromIterator for NamedPaths { - fn from_iter>(iter: I) -> Self { - NamedPaths(iter.into_iter().collect()) - } +/// A declared path together with the map key it sits under, as yielded by +/// [`NamedPaths::entries`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NamedPath<'a> { + /// The map key this path sits under, or `None` for a plain-list entry. + /// Non-unique: the paths of a key that maps to a list all share it. + pub key: Option<&'a str>, + /// The path itself, relative to the canister directory. + pub path: &'a str, } -impl<'de> Deserialize<'de> for NamedPaths { - fn deserialize>(d: D) -> Result { - /// A map value: a single path or a list of paths sharing the key. - #[derive(Deserialize)] - #[serde(untagged)] - enum PathOrList { - One(String), - Many(Vec), - } - - #[derive(Deserialize)] - #[serde(untagged)] - enum Repr { - List(Vec), - Map(IndexMap), +impl NamedPaths { + /// The declared paths in written order, each tagged with its key (if any). + pub fn entries(&self) -> impl Iterator> { + match self { + Self::List(paths) => Either::Left(paths.iter().map(|path| NamedPath { + key: None, + path: path.as_str(), + })), + Self::Map(map) => Either::Right(map.iter().flat_map(|(key, value)| { + value.paths().iter().map(move |path| NamedPath { + key: Some(key.as_str()), + path: path.as_str(), + }) + })), } + } - let entries = match Repr::deserialize(d)? { - Repr::List(paths) => paths - .into_iter() - .map(|path| NamedPath { key: None, path }) - .collect(), - Repr::Map(map) => map - .into_iter() - .flat_map(|(key, value)| { - let paths = match value { - PathOrList::One(path) => vec![path], - PathOrList::Many(paths) => paths, - }; - paths.into_iter().map(move |path| NamedPath { - key: Some(key.clone()), - path, + /// Rewrite every path, leaving the keys and the written shape intact. + pub fn map_paths(&self, mut f: impl FnMut(&str) -> String) -> Self { + match self { + Self::List(paths) => Self::List(paths.iter().map(|path| f(path)).collect()), + Self::Map(map) => Self::Map( + map.iter() + .map(|(key, value)| { + let value = match value { + PathOrList::One(path) => PathOrList::One(f(path)), + PathOrList::Many(paths) => { + PathOrList::Many(paths.iter().map(|path| f(path)).collect()) + } + }; + (key.clone(), value) }) - }) - .collect(), - }; - Ok(NamedPaths(entries)) + .collect(), + ), + } } } -impl Serialize for NamedPaths { - fn serialize(&self, s: S) -> Result { - // Deserialization yields either all-unkeyed (list form) or all-keyed - // (map form) entries; serialize back to whichever it was. - if self.0.iter().all(|e| e.key.is_none()) { - let paths: Vec<&str> = self.0.iter().map(|e| e.path.as_str()).collect(); - paths.serialize(s) - } else { - // Group paths by key, preserving order. A key with one path - // serializes as a scalar; multiple as a list. - let mut groups: IndexMap<&str, Vec<&str>> = IndexMap::new(); - for e in &self.0 { - groups - .entry(e.key.as_deref().unwrap_or_default()) - .or_default() - .push(&e.path); - } - let mut map = s.serialize_map(Some(groups.len()))?; - for (key, paths) in groups { - match paths.as_slice() { - [one] => map.serialize_entry(key, one)?, - many => map.serialize_entry(key, many)?, - } - } - map.end() +impl PathOrList { + /// The paths sitting under this key. + fn paths(&self) -> &[String] { + match self { + Self::One(path) => std::slice::from_ref(path), + Self::Many(paths) => paths, } } } @@ -287,14 +233,12 @@ pub struct Adapter { /// can traverse it using standard filesystem APIs. Written as a plain list /// of paths, or as a map of name → path (or list of paths); the name is /// surfaced to the plugin as each entry's `key`. - #[schemars(with = "Option")] pub dirs: Option, /// Files (relative to canister directory) the host reads and passes to /// the plugin as part of `sync-exec-input.files`. Written as a plain list /// of paths, or as a map of name → path (or list of paths); the name is /// surfaced to the plugin as each entry's `key`. - #[schemars(with = "Option")] pub files: Option, /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. @@ -349,27 +293,25 @@ mod tests { use super::*; - /// [`NamedPaths`] with no keys, as a plain-list manifest entry produces. - fn unkeyed(paths: [&str; N]) -> NamedPaths { - NamedPaths::from_entries( - paths - .into_iter() - .map(|path| NamedPath { - key: None, - path: path.to_string(), - }) - .collect(), - ) + use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; + + /// The plain-list form of `dirs:`/`files:`. + fn list(paths: [&str; N]) -> NamedPaths { + NamedPaths::List(paths.into_iter().map(str::to_string).collect()) } - /// A single key-tagged [`NamedPath`]. - fn keyed(key: &str, path: &str) -> NamedPath { + /// A key-tagged entry, as [`NamedPaths::entries`] yields for the map form. + fn keyed<'a>(key: &'a str, path: &'a str) -> NamedPath<'a> { NamedPath { - key: Some(key.to_string()), - path: path.to_string(), + key: Some(key), + path, } } - use crate::manifest::adapter::prebuilt::{LocalSource, RemoteSource}; + + /// The flattened entries of an optional `dirs:`/`files:` setting. + fn entries(paths: &Option) -> Option>> { + paths.as_ref().map(|paths| paths.entries().collect()) + } #[test] fn local_path() { @@ -413,8 +355,8 @@ mod tests { path: "plugins/my-sync.wasm".into(), }), sha256: Some("abc123".to_string()), - dirs: Some(unkeyed(["assets/seed-data", "config"])), - files: Some(unkeyed(["config.txt"])), + dirs: Some(list(["assets/seed-data", "config"])), + files: Some(list(["config.txt"])), fields: None, canisters: None, }, @@ -445,8 +387,8 @@ mod tests { "#, ) .expect("failed to deserialize Adapter with list dirs/files"); - assert_eq!(adapter.dirs, Some(unkeyed(["assets"]))); - assert_eq!(adapter.files, Some(unkeyed(["a.txt", "b.txt"]))); + assert_eq!(adapter.dirs, Some(list(["assets"]))); + assert_eq!(adapter.files, Some(list(["a.txt", "b.txt"]))); } /// The map form tags each entry with its key. A key mapping to a list yields @@ -467,7 +409,7 @@ mod tests { ) .expect("failed to deserialize Adapter with map dirs/files"); assert_eq!( - adapter.dirs.map(NamedPaths::into_entries), + entries(&adapter.dirs), Some(vec![ keyed("seed", "assets/seed-data"), keyed("extra", "one"), @@ -475,11 +417,23 @@ mod tests { ]), ); assert_eq!( - adapter.files.map(NamedPaths::into_entries), + entries(&adapter.files), Some(vec![keyed("main", "config.txt")]), ); } + /// Rewriting paths (as bundling does) leaves keys and the written shape alone. + #[test] + fn map_paths_preserves_keys_and_shape() { + let paths: NamedPaths = serde_yaml::from_str("single: one.txt\nmany:\n- x.txt\n- y.txt\n") + .expect("failed to parse NamedPaths"); + let mapped = paths.map_paths(|path| format!("bundled/{path}")); + assert_eq!( + serde_yaml::to_string(&mapped).expect("failed to serialize"), + "single: bundled/one.txt\nmany:\n- bundled/x.txt\n- bundled/y.txt\n", + ); + } + /// The list and map forms round-trip through serialization back to their /// natural YAML shape. #[test] diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 23c9ca853..8efab30de 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -318,7 +318,8 @@ pub enum SyncStep { /// Represents a sync step executed by a WebAssembly plugin running inside /// a wasmtime WASI sandbox. The plugin can call canister methods on exactly /// the canister being synced and read files from the declared `dirs`. - Plugin(adapter::plugin::Adapter), + // Boxed: a plugin step carries far more configuration than a script one. + Plugin(Box), } impl<'de> Deserialize<'de> for SyncStep { @@ -332,7 +333,7 @@ impl<'de> Deserialize<'de> for SyncStep { #[serde(tag = "type", rename_all = "lowercase")] enum Helper { Script(adapter::script::Adapter), - Plugin(adapter::plugin::Adapter), + Plugin(Box), Assets(serde::de::IgnoredAny), } @@ -383,6 +384,7 @@ mod tests { use crate::{ manifest::{ adapter::{ + plugin, prebuilt::{self, RemoteSource, SourceField}, script, }, @@ -785,25 +787,18 @@ mod tests { })] }, sync: Some(SyncSteps { - steps: vec![SyncStep::Plugin( - crate::manifest::adapter::plugin::Adapter { - source: prebuilt::SourceField::Local(prebuilt::LocalSource { - path: "./plugins/my-sync.wasm".into(), - }), - sha256: None, - dirs: Some( - crate::manifest::adapter::plugin::NamedPaths::from_entries( - vec![crate::manifest::adapter::plugin::NamedPath { - key: None, - path: "assets/seed-data/".to_string(), - }], - ) - ), - files: None, - fields: None, - canisters: None, - } - )] + steps: vec![SyncStep::Plugin(Box::new(plugin::Adapter { + source: prebuilt::SourceField::Local(prebuilt::LocalSource { + path: "./plugins/my-sync.wasm".into(), + }), + sha256: None, + dirs: Some(plugin::NamedPaths::List(vec![ + "assets/seed-data/".to_string() + ])), + files: None, + fields: None, + canisters: None, + }))] }), }, }, @@ -836,7 +831,7 @@ mod tests { })] }, sync: Some(SyncSteps { - steps: vec![SyncStep::Plugin(crate::manifest::adapter::plugin::Adapter { + steps: vec![SyncStep::Plugin(Box::new(plugin::Adapter { source: prebuilt::SourceField::Remote(prebuilt::RemoteSource { url: "https://example.com/plugins/migrate-v2.wasm".to_string(), }), @@ -848,7 +843,7 @@ mod tests { files: None, fields: None, canisters: None, - })] + }))] }), }, }, diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index a5c7e262d..c8aaeae3f 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -82,7 +82,7 @@ "dirs": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -107,7 +107,7 @@ "files": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -341,9 +341,10 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, - "NamedPathsSchema": { + "NamedPaths": { "anyOf": [ { + "description": "A plain list of paths, carrying no keys.", "items": { "type": "string" }, @@ -351,26 +352,29 @@ }, { "additionalProperties": { - "$ref": "#/$defs/PathOrListSchema" + "$ref": "#/$defs/PathOrList" }, + "description": "A map of name → path(s), tagging each path with the key it sits under.", "type": "object" } ], - "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + "description": "The paths declared for a plugin step's `dirs` or `files`: either a plain list\nof paths, or a map of name → path(s) whose keys are surfaced to the plugin.\n\n```yaml\n# a plain list — entries carry no key\nfiles:\n - config.txt\n - data.json\n# a map whose keys each name a single path...\nfiles:\n main: config.txt\n# ...or a list of paths, which then all share that key\nfiles:\n seeds:\n - a.json\n - b.json\n```\n\nOrder is preserved in both forms: list entries in written order; map entries\nin written key order, each key's paths in written order." }, - "PathOrListSchema": { + "PathOrList": { "anyOf": [ { + "description": "A single path under the key.", "type": "string" }, { + "description": "Several paths, all sharing the key.", "items": { "type": "string" }, "type": "array" } ], - "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + "description": "One value of a [`NamedPaths::Map`]: a single path, or a list of paths that\nall share the key." }, "Recipe": { "properties": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 4121c9bb4..b0ca7f2d8 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -82,7 +82,7 @@ "dirs": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -107,7 +107,7 @@ "files": { "anyOf": [ { - "$ref": "#/$defs/NamedPathsSchema" + "$ref": "#/$defs/NamedPaths" }, { "type": "null" @@ -797,9 +797,10 @@ ], "description": "An amount of memory in bytes.\n\nDeserializes from a number or a string with suffixes (kb, kib, mb, mib, gb, gib),\noptional decimals, and optional underscore separators." }, - "NamedPathsSchema": { + "NamedPaths": { "anyOf": [ { + "description": "A plain list of paths, carrying no keys.", "items": { "type": "string" }, @@ -807,12 +808,13 @@ }, { "additionalProperties": { - "$ref": "#/$defs/PathOrListSchema" + "$ref": "#/$defs/PathOrList" }, + "description": "A map of name → path(s), tagging each path with the key it sits under.", "type": "object" } ], - "description": "A list of paths, or a map of name → path (or list of paths). The map form\ntags each path with its key for the plugin; a key may map to several paths.\n\nThis type exists only to describe [`NamedPaths`] in the generated JSON schema\n(see the `#[schemars(with = ...)]` on the adapter fields); [`NamedPaths`]\nowns the actual (de)serialization." + "description": "The paths declared for a plugin step's `dirs` or `files`: either a plain list\nof paths, or a map of name → path(s) whose keys are surfaced to the plugin.\n\n```yaml\n# a plain list — entries carry no key\nfiles:\n - config.txt\n - data.json\n# a map whose keys each name a single path...\nfiles:\n main: config.txt\n# ...or a list of paths, which then all share that key\nfiles:\n seeds:\n - a.json\n - b.json\n```\n\nOrder is preserved in both forms: list entries in written order; map entries\nin written key order, each key's paths in written order." }, "NetworkManifest": { "description": "A network definition for the project", @@ -854,19 +856,21 @@ ], "type": "object" }, - "PathOrListSchema": { + "PathOrList": { "anyOf": [ { + "description": "A single path under the key.", "type": "string" }, { + "description": "Several paths, all sharing the key.", "items": { "type": "string" }, "type": "array" } ], - "description": "One map value in [`NamedPathsSchema`]: a single path, or a list of paths that\nshare the key." + "description": "One value of a [`NamedPaths::Map`]: a single path, or a list of paths that\nall share the key." }, "Recipe": { "properties": { From b491205ea1b87ccb3337ab3d2640f4b27ec34a96 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Fri, 21 Aug 2026 14:12:10 -0700 Subject: [PATCH 43/51] Handle duplicate paths properly --- Cargo.lock | 1 + crates/icp-cli/Cargo.toml | 1 + crates/icp-cli/src/operations/bundle.rs | 75 ++++++++----- crates/icp-cli/tests/bundle_tests.rs | 102 ++++++++++++++++++ crates/icp-sync-plugin/src/lib.rs | 1 + crates/icp-sync-plugin/src/path.rs | 101 +++++++++++++++++ crates/icp-sync-plugin/src/runtime.rs | 60 ++++++++++- crates/icp-sync-plugin/sync-plugin.wit | 11 +- .../tests/fixtures/test-plugin/src/lib.rs | 19 ++++ crates/icp/src/manifest/adapter/plugin.rs | 10 +- docs/concepts/sync-plugins.md | 3 +- docs/reference/configuration.md | 2 +- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- 14 files changed, 346 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ca77dd4d..b68d3f5b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3728,6 +3728,7 @@ dependencies = [ "ic-utils", "icp", "icp-canister-interfaces", + "icp-sync-plugin", "icrc-ledger-types", "indicatif", "indoc", diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index 3b85f287b..7e326084c 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -44,6 +44,7 @@ ic-management-canister-types.workspace = true ic-utils.workspace = true icp-canister-interfaces.workspace = true icp = { workspace = true, features = ["clap"] } +icp-sync-plugin.workspace = true icrc-ledger-types.workspace = true indicatif.workspace = true indoc.workspace = true diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs index 5bd881d9e..918bb599f 100644 --- a/crates/icp-cli/src/operations/bundle.rs +++ b/crates/icp-cli/src/operations/bundle.rs @@ -25,6 +25,7 @@ use icp::{ project::{WorkspaceInstance, WorkspaceInstancesError, workspace_instances}, store_artifact, }; +use icp_sync_plugin::{covering_dirs, distinct_paths}; use snafu::{OptionExt, ResultExt, Snafu}; use tar::Builder; @@ -768,36 +769,56 @@ async fn prepare_plugin_step( // `files` cannot collide with the `files/` area used for plugin input files. // The declared paths are rewritten to their archive locations; each entry's // map key is carried through unchanged. - let bundle_dirs = adapter.dirs.as_ref().map(|dirs| { - dirs.map_paths(|dir| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/dirs/{}", - normalize_archive_dir(dir) - ); - out.plugin_dirs.push(DirEntry { - src_path: canister_path.join(dir), - archive_prefix: archive_join(prefix, &manifest_path), - }); - manifest_path - }) - }); - + let dirs_prefix = format!("plugins/{path_name}/{idx}/dirs"); + let files_prefix = format!("plugins/{path_name}/{idx}/files"); + let bundle_dirs = adapter + .dirs + .as_ref() + .map(|dirs| dirs.map_paths(|dir| format!("{dirs_prefix}/{}", normalize_archive_dir(dir)))); let bundle_files = adapter.files.as_ref().map(|files| { - files.map_paths(|file| { - let manifest_path = format!( - "plugins/{path_name}/{idx}/files/{}", - normalize_archive_dir(file) - ); - out.plugin_files.push(PluginFile { - src_path: canister_path.join(file), - archive_path: archive_join(prefix, &manifest_path), - canister_name: canister.name.clone(), - orig_file: file.to_string(), - }); - manifest_path - }) + files.map_paths(|file| format!("{files_prefix}/{}", normalize_archive_dir(file))) }); + // The rewritten manifest above keeps every declared entry; the archive holds + // the trees and files behind them, of which there are fewer. A directory + // named under two keys is one tree to copy, and a declared subdirectory of + // another is already inside its copy — writing either twice would collide in + // the archive. The reduction runs over the paths as declared, so two that + // only *look* alike once rewritten (`../shared` and `shared` both normalize + // to `shared`) stay separate and are still caught as a collision. + for dir in covering_dirs( + adapter + .dirs + .iter() + .flat_map(plugin::NamedPaths::entries) + .map(|entry| entry.path), + ) { + out.plugin_dirs.push(DirEntry { + src_path: canister_path.join(dir), + archive_prefix: archive_join( + prefix, + &format!("{dirs_prefix}/{}", normalize_archive_dir(dir)), + ), + }); + } + for file in distinct_paths( + adapter + .files + .iter() + .flat_map(plugin::NamedPaths::entries) + .map(|entry| entry.path), + ) { + out.plugin_files.push(PluginFile { + src_path: canister_path.join(file), + archive_path: archive_join( + prefix, + &format!("{files_prefix}/{}", normalize_archive_dir(file)), + ), + canister_name: canister.name.clone(), + orig_file: file.to_string(), + }); + } + Ok(SyncStep::Plugin(Box::new(plugin::Adapter { source: SourceField::Local(LocalSource { path: plugin_wasm_path.as_str().into(), diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs index 66feb2c84..49b6c2a99 100644 --- a/crates/icp-cli/tests/bundle_tests.rs +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -1383,6 +1383,108 @@ fn bundle_preserves_plugin_path_keys() { ); } +/// `dirs:`/`files:` are configuration as well as sandbox grants, so the same path may be +/// named under several keys, and one key's directory may sit inside another's. The bundled +/// manifest keeps every entry as declared; the archive holds one copy of each tree, since +/// two copies of one directory (or a copy of a directory already inside another) cannot be +/// written to the archive at all. +#[test] +fn bundle_archives_aliased_plugin_paths_once() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let plugin_bytes: &[u8] = b"\x00asm\x01\x00\x00\x00plugin"; + write(&project_dir.join("plugin.wasm"), plugin_bytes).expect("failed to write plugin"); + + let inner = project_dir.join("data/inner"); + create_dir_all(&inner).expect("failed to create dir"); + write_string(&project_dir.join("data/top.txt"), "top").expect("failed to write file"); + write_string(&inner.join("deep.txt"), "deep").expect("failed to write file"); + write_string(&project_dir.join("config.toml"), "key=value").expect("failed to write config"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: plugin.wasm + dirs: + seed: data + backup: data + sub: data/inner + files: + main: config.toml + fallback: ./config.toml + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut archived: Vec = Vec::new(); + let mut manifest_yaml = String::new(); + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + if path == "icp.yaml" { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + archived.push(path); + } + + // One copy of the tree, holding what the nested entry points at. + for expected in [ + "plugins/my-canister/0/dirs/data/top.txt", + "plugins/my-canister/0/dirs/data/inner/deep.txt", + "plugins/my-canister/0/files/config.toml", + ] { + assert_eq!( + archived.iter().filter(|path| *path == expected).count(), + 1, + "{expected} should appear exactly once; archive holds {archived:?}" + ); + } + + // Every declared entry survives, keys and all. + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let step = &parsed["canisters"][0]["sync"]["steps"][0]; + for (key, expected) in [ + ("seed", "plugins/my-canister/0/dirs/data"), + ("backup", "plugins/my-canister/0/dirs/data"), + ("sub", "plugins/my-canister/0/dirs/data/inner"), + ] { + assert_eq!(step["dirs"][key].as_str(), Some(expected)); + } + for key in ["main", "fallback"] { + assert_eq!( + step["files"][key].as_str(), + Some("plugins/my-canister/0/files/config.toml") + ); + } +} + /// An `icp_appmanifest.yaml` next to the project manifest must be included in the bundle, with its /// top-level `images` paths relocated under a top-level `images/` folder and the /// referenced image files copied alongside. Unrelated metadata is preserved. diff --git a/crates/icp-sync-plugin/src/lib.rs b/crates/icp-sync-plugin/src/lib.rs index d2fa023b4..a6f212fca 100644 --- a/crates/icp-sync-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/src/lib.rs @@ -1,6 +1,7 @@ mod path; mod runtime; +pub use path::{covering_dirs, distinct_paths}; pub use runtime::{ CallableCanisters, DEFAULT_PLUGIN_COMPUTE_LIMIT_SECS, KeyedPath, PLUGIN_COMPUTE_LIMIT_ENV, PluginInvocation, RunPluginError, run_plugin, diff --git a/crates/icp-sync-plugin/src/path.rs b/crates/icp-sync-plugin/src/path.rs index 625f7ac68..3f8615c70 100644 --- a/crates/icp-sync-plugin/src/path.rs +++ b/crates/icp-sync-plugin/src/path.rs @@ -1,6 +1,8 @@ //! Path-safety helpers used by the host runtime to validate declared `dirs`/`files` //! entries before preopening directories or reading files under the canister base dir. +use std::collections::HashSet; + use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; /// Returns `true` if `rel` cannot be safely joined onto a base directory @@ -59,6 +61,105 @@ pub(crate) fn first_symlink_component(base: &Utf8Path, rel: &str) -> Option Vec<&str> { + path.split('/') + .filter(|part| !part.is_empty() && *part != ".") + .collect() +} + +/// Reduce declared directories to the ones that actually have to be opened. +/// +/// `dirs` is configuration as much as it is a sandbox grant: a plugin may +/// legitimately be handed the same tree under several keys, or a tree and a +/// subtree of it, and it is told about every entry that was declared. The grant +/// behind those entries has no such multiplicity — opening a directory twice, or +/// opening one already reachable through an ancestor, conveys no further access. +/// Callers keep the declared list as configuration and open only what this +/// returns; a nested declared directory is reached through the ancestor covering +/// it. +/// +/// Retained paths keep their written spelling and first-occurrence order. +/// Comparison is component-wise, so `data` covers `./data/inner` but not +/// `database`. Paths are expected to be relative and free of `..` (see +/// [`escapes_base`]); a `..` compares as an ordinary name, which can only leave +/// the result less reduced. +pub fn covering_dirs<'a>(dirs: impl IntoIterator) -> Vec<&'a str> { + let dirs: Vec<&str> = dirs.into_iter().collect(); + let parts: Vec> = dirs.iter().map(|dir| components(dir)).collect(); + dirs.iter() + .enumerate() + .filter(|(i, _)| { + !parts.iter().enumerate().any(|(j, other)| { + j != *i + && parts[*i].starts_with(other) + // A strict ancestor always covers; between equals, the first written wins. + && (other.len() < parts[*i].len() || j < *i) + }) + }) + .map(|(_, dir)| *dir) + .collect() +} + +/// Reduce declared paths to the distinct ones, keeping the written spelling and +/// first-occurrence order. +/// +/// [`covering_dirs`] without the containment rule, for entries that name files: +/// `./a.json` and `a.json` are one file, but a file never subsumes another the +/// way a directory subsumes its contents. +pub fn distinct_paths<'a>(paths: impl IntoIterator) -> Vec<&'a str> { + let mut seen: HashSet> = HashSet::new(); + paths + .into_iter() + .filter(|path| seen.insert(components(path))) + .collect() +} + +#[cfg(test)] +mod covering_tests { + use super::*; + + #[test] + fn unrelated_dirs_are_all_kept() { + assert_eq!( + covering_dirs(["assets", "config", "data/seed"]), + ["assets", "config", "data/seed"], + ); + } + + #[test] + fn duplicates_collapse_to_the_first_spelling() { + assert_eq!(covering_dirs(["./data", "data", "data/"]), ["./data"]); + } + + #[test] + fn nested_dirs_collapse_to_their_ancestor_whichever_is_written_first() { + assert_eq!(covering_dirs(["data", "data/inner"]), ["data"]); + assert_eq!(covering_dirs(["data/inner", "data"]), ["data"]); + // Transitive: `data` covers `data/a` covers `data/a/b`. + assert_eq!(covering_dirs(["data/a/b", "data/a", "data"]), ["data"]); + } + + #[test] + fn a_name_prefix_is_not_an_ancestor() { + assert_eq!(covering_dirs(["data", "database"]), ["data", "database"]); + } + + #[test] + fn distinct_paths_dedupes_without_containment() { + assert_eq!( + distinct_paths(["./a.json", "a.json", "b.json", "dir/a.json"]), + ["./a.json", "b.json", "dir/a.json"], + ); + } +} + #[cfg(test)] mod escapes_base_tests { use super::*; diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index ae3d76f91..c7690e7b4 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -289,6 +289,9 @@ pub enum RunPluginError { ))] SymlinkDir { dir: String, link: Utf8PathBuf }, + #[snafu(display("plugin dir '{dir}' is not an existing directory"))] + MissingDir { dir: String }, + #[snafu(display("failed to preopen directory '{dir}' for the plugin"))] PreopenDir { source: wasmtime::Error, @@ -498,9 +501,9 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin path: wasm_path.clone(), })?; - // Preopen each declared directory read-only. The guest sees it at the - // same relative path it used in the manifest. - let mut wasi_builder = wasmtime_wasi::WasiCtxBuilder::new(); + // Check every declared directory: each one is handed to the plugin as + // configuration, so it is rejected for being unsafe or unusable whether or + // not it ends up needing a preopen of its own. for KeyedPath { path: dir, .. } in &dirs { ensure!(!crate::path::escapes_base(dir), UnsafeDirSnafu { dir }); // Reject symlinks in the declared path: neither the final entry nor any @@ -510,6 +513,17 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin if let Some(link) = crate::path::first_symlink_component(&base_dir, dir) { return SymlinkDirSnafu { dir, link }.fail(); } + let host_path = base_dir.join(dir); + let is_dir = std::fs::metadata(host_path.as_std_path()).is_ok_and(|meta| meta.is_dir()); + ensure!(is_dir, MissingDirSnafu { dir }); + } + + // Preopen read-only, one per distinct tree — a directory declared twice, or + // one already reachable through a declared ancestor, needs no preopen of its + // own. The guest sees each preopen at the same relative path it used in the + // manifest, and reaches a nested declared directory through its ancestor. + let mut wasi_builder = wasmtime_wasi::WasiCtxBuilder::new(); + for dir in crate::path::covering_dirs(dirs.iter().map(|d| d.path.as_str())) { let host_path = base_dir.join(dir); wasi_builder .preopened_dir( @@ -938,7 +952,7 @@ mod tests { // ------------------------------------------------------------------------- #[test] - fn preopen_dir_error_on_missing_dir() { + fn missing_dir_is_rejected() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { return; }; @@ -946,10 +960,46 @@ mod tests { inv.dirs = unkeyed(&["nonexistent_dir"]); assert!(matches!( run_plugin(inv), - Err(RunPluginError::PreopenDir { .. }) + Err(RunPluginError::MissingDir { .. }) )); } + /// A directory declared under several keys, or nested inside another + /// declared one, reaches the plugin as every entry it was written as. Only + /// the preopens behind those entries collapse — `data/inner` has none of its + /// own here, and is read through the `data` preopen that covers it. + #[test] + fn aliased_and_nested_dirs_are_all_readable() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let base = tmp.path(); + std::fs::create_dir_all(base.join("data/inner")).expect("create dir"); + std::fs::write(base.join("data/top.txt"), b"top").expect("write file"); + std::fs::write(base.join("data/inner/deep.txt"), b"deep").expect("write file"); + + let mut inv = invocation(wasm_path, "read-dirs"); + inv.base_dir = base.to_path_buf(); + inv.dirs = [("seed", "data"), ("backup", "data"), ("sub", "data/inner")] + .into_iter() + .map(|(key, path)| KeyedPath { + key: Some(key.to_owned()), + path: path.to_owned(), + }) + .collect(); + + let lines = run_plugin(inv).expect("plugin should succeed"); + assert_eq!( + lines, + [ + "seed=inner,top.txt".to_string(), + "backup=inner,top.txt".to_string(), + "sub=deep.txt".to_string(), + ], + ); + } + #[cfg(unix)] #[test] fn symlinked_dir_is_rejected() { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index d5a4481cc..ef6fa7587 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -5,14 +5,17 @@ interface types { /// Whether a canister call is an update or a query. enum call-type { update, query } - /// A directory the host preopened on behalf of the plugin. + /// A directory the host made readable for the plugin. record dir-input { /// The map key this directory was declared under in the manifest, or /// `none` when `dirs` was written as a plain list. Several entries share /// one key when a key maps to a list of directories. key: option, /// Path of the directory as declared in the manifest (relative to the - /// canister directory). The host preopens it at this same path. + /// canister directory). It is readable at this same path. Entries may + /// repeat a path or name a directory inside another entry's; the host + /// preopens each distinct tree once, so such an entry is read through + /// the preopen that covers it. path: string, } @@ -82,8 +85,8 @@ interface types { /// Name of the environment being synced (e.g. "production", "local"). environment: string, /// Directories declared in the manifest step's `dirs` setting. - /// The host preopens each entry via WASI; the plugin can traverse - /// them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). + /// The host makes each entry readable via WASI preopens; the plugin + /// traverses them with standard `wasi:filesystem` (e.g. Rust's `std::fs`). /// Each entry carries the map key it was declared under, if any (see /// `dir-input`). dirs: list, diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 2c7db0cb7..c3c849f83 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -44,6 +44,25 @@ impl Guest for TestPlugin { } Ok(()) } + // List each declared dir as `key=entry,entry`, so the host can + // assert a dir reached only through a preopened ancestor is still + // readable. + "read-dirs" => { + for dir in &input.dirs { + let mut names = std::fs::read_dir(&dir.path) + .and_then(|entries| { + entries + .map(|entry| { + entry.map(|e| e.file_name().to_string_lossy().into_owned()) + }) + .collect::, _>>() + }) + .map_err(|err| format!("reading '{}': {err}", dir.path))?; + names.sort(); + eprintln!("{}={}", dir.key.as_deref().unwrap_or("-"), names.join(",")); + } + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index d3814f7d3..9c7069ecf 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -229,10 +229,12 @@ pub struct Adapter { pub sha256: Option, /// Directories (relative to canister directory) the plugin may read from. - /// Each entry must be a directory; it is preopened via WASI so the plugin - /// can traverse it using standard filesystem APIs. Written as a plain list - /// of paths, or as a map of name → path (or list of paths); the name is - /// surfaced to the plugin as each entry's `key`. + /// Each entry must be a directory; it is made readable via WASI so the + /// plugin can traverse it using standard filesystem APIs. Written as a plain + /// list of paths, or as a map of name → path (or list of paths); the name is + /// surfaced to the plugin as each entry's `key`. Entries may repeat a + /// directory or name one inside another's — the plugin is told about each + /// entry as written, and reads them all. pub dirs: Option, /// Files (relative to canister directory) the host reads and passes to diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 853cab13e..2c53a347f 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -108,7 +108,8 @@ The plugin runs with a deliberately narrow capability surface. ### Filesystem -- Each directory in `dirs:` is preopened **read-only**. The plugin sees it at the same relative path it used in the manifest (e.g. `dirs: ["assets"]` is visible as `assets/` inside the guest) and traverses it with standard filesystem APIs (`std::fs` in Rust). +- Each directory in `dirs:` is readable **read-only**. The plugin sees it at the same relative path it used in the manifest (e.g. `dirs: ["assets"]` is visible as `assets/` inside the guest) and traverses it with standard filesystem APIs (`std::fs` in Rust). +- Entries may name the same directory under several keys, or name a directory inside another entry's, and the plugin is told about each entry as written. The preopens behind them are one per distinct tree: an entry nested inside another is read through the preopen covering it, which grants nothing extra. - Files in `files:` are read by the host up front and passed inline in `sync-exec-input.files`. The plugin reads their content from the input struct, not from disk. - Any path outside a preopen is invisible. Writes, creates, deletes, renames, and symlinks that escape a preopen are rejected by the sandbox at runtime. - Paths in `dirs:`/`files:` must be relative and may not contain `..`. They also may not be — or traverse — a symlink: each declared entry is rejected if it or any of its parent components is a symlink, so a declared path cannot resolve to a target outside the canister directory. (This restriction may be relaxed later if a safe use case emerges.) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d91e58c77..b1a46582e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -169,7 +169,7 @@ sync: | `path` | string | One of `path` or `url` | Local path to the wasm, relative to the canister directory | | `url` | string | One of `path` or `url` | URL to download the wasm from | | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | -| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is preopened read-only via WASI | +| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is made readable read-only via WASI | | `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | | `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | | `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index c8aaeae3f..9c2c60912 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -88,7 +88,7 @@ "type": "null" } ], - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is made readable via WASI so the\nplugin can traverse it using standard filesystem APIs. Written as a plain\nlist of paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`. Entries may repeat a\ndirectory or name one inside another's — the plugin is told about each\nentry as written, and reads them all." }, "fields": { "additionalProperties": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index b0ca7f2d8..ef3096767 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -88,7 +88,7 @@ "type": "null" } ], - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is preopened via WASI so the plugin\ncan traverse it using standard filesystem APIs. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." + "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is made readable via WASI so the\nplugin can traverse it using standard filesystem APIs. Written as a plain\nlist of paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`. Entries may repeat a\ndirectory or name one inside another's — the plugin is told about each\nentry as written, and reads them all." }, "fields": { "additionalProperties": { From 55d8aa6bdbda217cbd61cb974a34cdb9d82a24bc Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Wed, 26 Aug 2026 10:37:28 -0700 Subject: [PATCH 44/51] Allow files inside project dir, not just inside canister dir --- CHANGELOG.md | 1 + crates/icp-cli/src/commands/deploy.rs | 1 + crates/icp-cli/src/commands/sync.rs | 1 + crates/icp-cli/src/operations/sync.rs | 7 +- crates/icp-cli/tests/sync_tests.rs | 126 +++++++ crates/icp-sync-plugin/DESIGN.md | 86 +++-- crates/icp-sync-plugin/src/path.rs | 424 +++++++++++++++++----- crates/icp-sync-plugin/src/runtime.rs | 256 ++++++++++++- crates/icp/src/canister/sync/mod.rs | 5 + crates/icp/src/canister/sync/plugin.rs | 9 +- crates/icp/src/canister/sync/script.rs | 1 + crates/icp/src/manifest/adapter/plugin.rs | 28 +- docs/concepts/sync-plugins.md | 3 +- docs/guides/writing-sync-plugins.md | 2 +- docs/reference/configuration.md | 6 +- docs/schemas/canister-yaml-schema.json | 6 +- docs/schemas/icp-yaml-schema.json | 6 +- 17 files changed, 799 insertions(+), 169 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54afa1527..532761e87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ air-gapped signing * feat: `script` build steps now receive `ICP_CLI_ENVIRONMENT`, the name of the environment the canisters are being built for, so a build can vary by environment the way a sync step already could. * feat: `icp completions ` prints a shell completion script for `bash`, `zsh`, `fish`, `powershell`, or `elvish` to stdout. See the [installation guide](docs/guides/installation.md#shell-completions) for where to put it. +* feat(sync-plugin): `dirs:` and `files:` on a `plugin` sync step may now name anything inside the project, not just paths below the canister's own directory. Entries are still written relative to the canister directory, but may rise out of it — `dirs: ["../shared/assets"]` — so several canisters can be handed the same tree without duplicating it. The project directory is the boundary: an entry that resolves above it is rejected before the plugin runs, as is an absolute one, and an entry that is (or traverses) a symlink is still rejected outright. The plugin sees each directory at the path the manifest wrote, `..` and all. * fix: `icp canister logs` output formats are corrected. `--json` now emits machine-readable JSON and the default emits the human-readable lines (the two were swapped), and `--follow --json` emits newline-delimited JSON, one record per line, streamed as each record arrives. This is breaking for scripts: parsing the default output as JSON now requires `--json`, and consumers of `--follow --json` must read one JSON object per line. ## Experimental diff --git a/crates/icp-cli/src/commands/deploy.rs b/crates/icp-cli/src/commands/deploy.rs index b2a1fe4ca..50acdad2d 100644 --- a/crates/icp-cli/src/commands/deploy.rs +++ b/crates/icp-cli/src/commands/deploy.rs @@ -494,6 +494,7 @@ pub(crate) async fn exec(ctx: &Context, args: &DeployArgs) -> Result<(), anyhow: ctx.syncer.clone(), agent.clone(), sync_canisters, + ctx.project.load().await?.dir, environment_selection.name().to_owned(), env.network.name.clone(), canister_ids, diff --git a/crates/icp-cli/src/commands/sync.rs b/crates/icp-cli/src/commands/sync.rs index ac2d1e099..c3f293c0e 100644 --- a/crates/icp-cli/src/commands/sync.rs +++ b/crates/icp-cli/src/commands/sync.rs @@ -129,6 +129,7 @@ pub(crate) async fn exec(ctx: &Context, args: &SyncArgs) -> Result<(), anyhow::E ctx.syncer.clone(), agent, sync_canisters, + ctx.project.load().await?.dir, environment_selection.name().to_owned(), env.network.name.clone(), canister_ids, diff --git a/crates/icp-cli/src/operations/sync.rs b/crates/icp-cli/src/operations/sync.rs index 77ea04174..b24775f96 100644 --- a/crates/icp-cli/src/operations/sync.rs +++ b/crates/icp-cli/src/operations/sync.rs @@ -5,7 +5,7 @@ use icp::{ Canister, canister::sync::{Params, Synchronize, SynchronizeError}, package::PackageCache, - prelude::PathBuf, + prelude::{Path, PathBuf}, }; use snafu::prelude::*; use std::collections::BTreeMap; @@ -33,6 +33,7 @@ async fn sync_canister( syncer: &Arc, agent: &Agent, canister_path: PathBuf, + project_dir: &Path, canister_id: Principal, canister_info: &Canister, environment: &str, @@ -58,6 +59,7 @@ async fn sync_canister( step, &Params { path: canister_path.clone(), + project_dir: project_dir.to_path_buf(), cid: canister_id, name: canister_info.name.clone(), environment: environment.to_owned(), @@ -85,6 +87,7 @@ pub(crate) async fn sync_many( syncer: Arc, agent: Agent, canisters: Vec<(Principal, PathBuf, Canister)>, + project_dir: PathBuf, environment: String, network: String, canister_ids: BTreeMap, @@ -104,6 +107,7 @@ pub(crate) async fn sync_many( let environment = environment.clone(); let network = network.clone(); let canister_ids = canister_ids.clone(); + let project_dir = project_dir.clone(); async move { // Define the sync logic @@ -111,6 +115,7 @@ pub(crate) async fn sync_many( &syncer, &agent, canister_path, + &project_dir, cid, &canister_info, &environment, diff --git a/crates/icp-cli/tests/sync_tests.rs b/crates/icp-cli/tests/sync_tests.rs index 888dabdb6..c93f21c8e 100644 --- a/crates/icp-cli/tests/sync_tests.rs +++ b/crates/icp-cli/tests/sync_tests.rs @@ -534,6 +534,132 @@ async fn sync_plugin_accepts_map_form_dirs() { .stdout(contains("apple").and(contains("carrot"))); } +/// A `dirs:` entry may rise out of the canister directory and name a directory +/// elsewhere in the project — here a `shared-seed` tree next to the canister's +/// own directory — and the plugin reads it end-to-end. +#[tokio::test] +async fn sync_plugin_reads_a_dir_elsewhere_in_the_project() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + + let (canister_wasm, plugin_wasm) = build_sync_plugin_example(); + + // The seed data is a sibling of the canister's directory, not below it. + let seed_data = project_dir.join("shared-seed"); + create_dir_all(&seed_data).expect("failed to create shared-seed"); + write_string(&seed_data.join("fruit-01.txt"), "apple").expect("failed to write fruit-01.txt"); + write_string(&seed_data.join("fruit-02.txt"), "banana").expect("failed to write fruit-02.txt"); + + let canister_dir = project_dir.join("canisters/my-canister"); + create_dir_all(&canister_dir).expect("failed to create canister dir"); + let cm = formatdoc! {r#" + name: my-canister + build: + steps: + - type: script + command: cp '{canister_wasm}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: {plugin_wasm} + dirs: + - ../../shared-seed + "#}; + write_string(&canister_dir.join("canister.yaml"), &cm) + .expect("failed to write canister manifest"); + + let pm = formatdoc! {r#" + canisters: + - canisters/my-canister + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let _g = ctx.start_network_in(&project_dir, "random-network").await; + ctx.ping_until_healthy(&project_dir, "random-network"); + + clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) + .mint_cycles(10 * TRILLION); + + ctx.icp() + .current_dir(&project_dir) + .args(["deploy", "--environment", "random-environment"]) + .assert() + .success(); + + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "call", + "my-canister", + "show", + "()", + "--query", + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout(contains("apple").and(contains("banana"))); +} + +/// The project directory is the boundary: a `dirs:` entry that resolves above it +/// is rejected before the plugin runs, however many `..` it takes to get there. +#[tokio::test] +async fn sync_plugin_rejects_dir_outside_the_project() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + + let (canister_wasm, plugin_wasm) = build_sync_plugin_example(); + + // A real directory next to the project, named by a relative path that walks + // out of it — no symlink involved, so only the project bound rejects it. + let outside = ctx.home_path().join("outside-seed-data"); + create_dir_all(&outside).expect("failed to create outside dir"); + write_string(&outside.join("fruit-01.txt"), "apple").expect("failed to write fruit-01.txt"); + let escape = "../outside-seed-data"; + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{canister_wasm}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: {plugin_wasm} + dirs: + - {escape} + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let _g = ctx.start_network_in(&project_dir, "random-network").await; + ctx.ping_until_healthy(&project_dir, "random-network"); + + clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) + .mint_cycles(10 * TRILLION); + + ctx.icp() + .current_dir(&project_dir) + .env("NO_COLOR", "1") + .args(["deploy", "--environment", "random-environment"]) + .assert() + .failure() + .stderr( + contains("resolves outside") + .and(contains("outside-seed-data")) + .and(contains("inside the project directory")), + ); +} + /// A malformed `ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS` must abort the sync with an /// actionable error rather than being silently ignored. This also exercises the /// end-to-end wiring: it proves the override is actually read on the real plugin diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 27617fc14..9f1d8b95a 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -62,7 +62,7 @@ crates/icp-sync-plugin/ src/ lib.rs — public API: run_plugin(), RunPluginError runtime.rs — wasmtime component setup, HostState, bindgen!, exec() call - path.rs — declared-path safety checks (escapes_base, symlinks) + path.rs — declared-path resolution and safety checks (project bound, symlinks) sync-plugin.wit — current WIT interface, v0.2.0 sync-plugin-v1.wit — frozen WIT interface, v0.1.0 Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, candid, camino, snafu, tokio, semver @@ -74,41 +74,67 @@ Public function: pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPluginError> ``` -`PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `dirs`, `files`, -`fields`, `host_canister_id` (the canister being synced), `agent`, `proxy`, -`identity_principal`, `environment`, `compute_limit_secs`, the exposed -`canister_ids` table, the `callable: CallableCanisters` enforcement set, and -`stdio`. The CLI resolves the manifest's declared `canisters:` into +`PluginInvocation` bundles the inputs: `wasm_path`, `base_dir`, `project_dir`, +`dirs`, `files`, `fields`, `host_canister_id` (the canister being synced), +`agent`, `proxy`, `identity_principal`, `environment`, `compute_limit_secs`, the +exposed `canister_ids` table, the `callable: CallableCanisters` enforcement set, +and `stdio`. The CLI resolves the manifest's declared `canisters:` into `CallableCanisters` before calling; this crate stays free of any manifest knowledge. `dirs` and `files` are the manifest-relative paths (as `KeyedPath`s carrying the map key each was declared under, if any), straight from the adapter. The runtime -owns *all* filesystem access anchored at `base_dir`: it preopens each `dir` from -`base_dir.join(dir.path)` and reads each `file` from `base_dir.join(file.path)`, -passing the contents — and the keys — inline in `SyncExecInput`. Keeping -both inside the runtime means the path-safety logic (below) lives in one place -and stays private to this crate — the CLI just forwards strings. The returned +owns *all* filesystem access: it resolves each entry against `base_dir`, +preopens each `dir` and reads each `file` from the location that resolves to, +passing the contents — and the keys — inline in `SyncExecInput`. Keeping both +inside the runtime means the path-safety logic (below) lives in one place and +stays private to this crate — the CLI just forwards strings. The returned `Vec` is the plugin's persistent stderr lines (see stdio capture below); `stdio`, when set, receives the rolling progress lines live. -### Declared-path safety (no symlinks) - -Declared `dirs`/`files` entries are resolved on the host *before* the WASI -sandbox boundary, so a lexical "relative, no `..`" check is not enough on two -counts. First, a Windows drive-relative path such as `C:foo` carries a `Prefix` -component yet is not "absolute", so joining it would discard `base_dir`; -`escapes_base` (in `path.rs`) rejects `..`, root, and drive-prefix components, -mirroring the bundler's checks. Second, a declared entry that *is* a symlink — -or that traverses a symlinked parent component — would let a preopen or a read -resolve outside the canister directory; `first_symlink_component` walks each -component of the declared path under `base_dir` and rejects the entry if any -prefix is a symlink (returning the offending sub-path relative to `base_dir`, so -errors don't leak absolute on-disk paths). Both helpers are crate-private and -applied uniformly to `dirs` and `files`. Symlinks are forbidden outright for -now; the restriction can be relaxed later if a safe use case emerges. (Symlinks -*inside* a preopen that escape it are a separate concern, already rejected by -the WASI sandbox — cap-std — at runtime.) +### Declared-path safety (project-bounded, no symlinks) + +An entry is written relative to `base_dir` (the canister directory) but bounded +by `project_dir`: it may rise out of the canister directory with `..` and reach +anything else in the project, and nothing above the project. `path.rs` resolves +one against the other: + +- `base_within_root` places `base_dir` inside `project_dir` as a clean component + list. When `base_dir` does not lie within it — a dependency project reached by + an out-of-tree `path:`, which `icp project bundle` rejects but `icp sync` + allows — there is no project-relative position to anchor at, so `base_dir` + becomes its own root: exactly the rule that predated the widening, and no + narrower than what such a project could already reach. (This is a fallback for + an unanchorable base, not a tighter grant for dependencies. A dependency + vendored inside the workspace is bounded by the workspace root like any other + canister, and its manifest can in any case run arbitrary commands through a + `script` step.) +- `resolve` walks the declared entry from there, resolving `.`/`..` lexically. A + `..` with nothing left to pop is `Escape::AboveRoot`; a root or drive-prefix + component is `Escape::NotRelative` — a Windows drive-relative path such as + `C:foo` carries a `Prefix` component yet is not "absolute", so joining it + would discard the base. This mirrors the bundler's checks. +- The host path is the *resolved* location joined onto the root, never the + declared path joined onto `base_dir`: the latter would leave a `..` for the OS + to resolve through whatever `base_dir`'s own components happen to be. +- `Resolved::first_symlink_component` then walks the resolved path under the + root and rejects the entry if any component is a symlink (returning the + offending sub-path relative to the root, so errors don't leak absolute on-disk + paths). An entry that stays below `base_dir` is checked only from there down — + the ancestry reaching the canister directory is exempt on the same grounds as + the root itself, since how the project reaches its own canister is not + something a manifest declared. An entry that rises *out* of `base_dir` is + checked from the root down instead: it re-anchors on an ancestor and descends + where the canister directory's own path never went, so a symlink in that + ancestry would put its target outside the project. An entry that *is* a + symlink, or that traverses one, + would otherwise let a preopen or a read resolve outside the project. Symlinks + are forbidden outright for now; the restriction can be relaxed later if a safe + use case emerges. (Symlinks *inside* a preopen that escape it are a separate + concern, already rejected by the WASI sandbox — cap-std — at runtime.) + +The guest still sees each preopen under the path the manifest wrote, `..` and +all, so a plugin opens `dir.path` verbatim regardless of where it points. ### `HostState` and bindgen @@ -229,7 +255,9 @@ verifies sha256, builds the exposed canister ID table and the `CallableCanisters enforcement set (resolving `canisters:` against the project's IDs), then calls `icp_sync_plugin::run_plugin(...)` with a `PluginInvocation`. The runtime — not the CLI — opens the declared paths and enforces the path-safety checks, so the -CLI no longer touches the plugin's input files itself. `exposed_canister_ids` +CLI no longer touches the plugin's input files itself; it supplies the canister +directory and the project directory (`sync::Params::path` and `project_dir`) +that bound them. `exposed_canister_ids` adds a bare-local-name duplicate for every canister in the same subproject as the one being synced; `resolve_callable` fails the step if a name in `canisters:` does not resolve. diff --git a/crates/icp-sync-plugin/src/path.rs b/crates/icp-sync-plugin/src/path.rs index 3f8615c70..705aa634d 100644 --- a/crates/icp-sync-plugin/src/path.rs +++ b/crates/icp-sync-plugin/src/path.rs @@ -1,55 +1,134 @@ //! Path-safety helpers used by the host runtime to validate declared `dirs`/`files` -//! entries before preopening directories or reading files under the canister base dir. +//! entries before preopening directories or reading files inside the project. use std::collections::HashSet; use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; -/// Returns `true` if `rel` cannot be safely joined onto a base directory -/// because it contains a component that would escape it: `..`, a filesystem -/// root, or a (Windows) drive prefix such as `C:` — the latter makes a path -/// drive-relative even without a leading separator, so `is_absolute()` returns -/// `false` yet joining it discards the base. Mirrors the escape checks in the -/// bundler (`crates/icp-cli/src/operations/bundle.rs`). -/// -/// Callers reject such paths before resolving them; `first_symlink_component` -/// only inspects `Normal` components and so would not otherwise catch these. -pub(crate) fn escapes_base(rel: &str) -> bool { - Utf8Path::new(rel).components().any(|c| { - matches!( - c, - Utf8Component::ParentDir | Utf8Component::RootDir | Utf8Component::Prefix(_) - ) - }) +/// Why a declared `dirs`/`files` entry cannot be anchored inside the sandbox +/// root. Reported by [`resolve`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Escape { + /// The entry is absolute, or (on Windows) drive-relative such as `C:foo` — + /// which makes it drive-relative even without a leading separator, so + /// `is_absolute()` returns `false` yet joining it discards the base. + NotRelative, + /// The entry rose above the sandbox root with `..`. + AboveRoot, } -/// Walks `rel` one component at a time under `base` and returns the first -/// sub-path of `rel` (relative to `base`) that is a symlink, if any. -/// -/// Declared `dirs`/`files` entries are resolved on the host *before* the WASI -/// sandbox boundary, so a symlinked entry — or an entry that traverses a -/// symlinked directory — would let a preopen or a host read escape `base` to an -/// arbitrary location on disk (the lexical [`escapes_base`] check does not catch -/// this). Rejecting any symlink in the declared portion keeps every preopen and -/// read anchored within `base`. Symlinks *inside* a preopen that escape it are -/// separately rejected by the WASI sandbox (cap-std) at runtime. +/// A declared `dirs`/`files` entry resolved against the sandbox root. /// -/// The returned path is relative to `base` (e.g. `link` or `link/inner`), -/// matching what the user wrote in the manifest, so it can be surfaced in an -/// error without leaking the absolute on-disk location. +/// Produced by [`resolve`]; the components are free of `.` and `..`, so the +/// on-disk location is [`Self::path`] joined onto the root — never the declared +/// path joined onto the base directory, which for an entry containing `..` +/// would be resolved by the OS through whatever the base directory's own +/// components happen to be. +#[derive(Debug)] +pub(crate) struct Resolved<'a> { + /// The entry's location relative to the sandbox root. + components: Vec<&'a str>, + /// Index of the first component the symlink check covers. For an entry + /// that stays below the base directory this is the whole base-directory + /// ancestry, exempt for the same reason the root itself is: how the project + /// reaches its own canister directory is not something a manifest declared. + /// An entry that rises out of the base directory is checked from the root + /// (index 0) — it re-anchors at an ancestor and descends somewhere the base + /// directory's own path never went, so none of that ancestry can be assumed + /// to lead where it lexically says it does. + traversed_from: usize, +} + +/// The base directory's position relative to the sandbox root, as the +/// components of a `.`/`..`-free relative path. /// -/// `base` itself may be reached through symlinks (e.g. the project lives under -/// a symlinked path); only the declared relative portion is checked. +/// `None` when `base` lies outside `root` — a dependency project reached by an +/// out-of-tree `path:` — which callers treat as "the base directory is its own +/// root", granting nothing above it. +pub(crate) fn base_within_root<'a>(root: &Utf8Path, base: &'a Utf8Path) -> Option> { + let rel = base.strip_prefix(root).ok()?; + let mut components = Vec::new(); + for component in rel.components() { + match component { + Utf8Component::Normal(name) => components.push(name), + Utf8Component::CurDir => {} + Utf8Component::ParentDir => { + components.pop()?; + } + Utf8Component::RootDir | Utf8Component::Prefix(_) => return None, + } + } + Some(components) +} + +/// Resolve a declared entry, written relative to the base directory, against +/// the sandbox root. /// -/// `rel` is expected to be relative and free of `..` (callers validate that via -/// [`escapes_base`] first); `.` components are ignored. Components that do not -/// exist are not symlinks, so a missing path returns `None` and the subsequent -/// read or preopen surfaces the not-found error. -pub(crate) fn first_symlink_component(base: &Utf8Path, rel: &str) -> Option { - let mut host = base.to_path_buf(); - let mut relative = Utf8PathBuf::new(); +/// `base` is the base directory's own position relative to the root (see +/// [`base_within_root`]). The entry may rise out of the base directory into the +/// rest of the project; it may not rise above the root, be absolute, or carry a +/// drive prefix. Mirrors the escape checks in the bundler +/// (`crates/icp-cli/src/operations/bundle.rs`). +pub(crate) fn resolve<'a>(base: &[&'a str], rel: &'a str) -> Result, Escape> { + let mut components = base.to_vec(); + let mut traversed_from = base.len(); for component in Utf8Path::new(rel).components() { - if let Utf8Component::Normal(name) = component { + match component { + Utf8Component::Normal(name) => components.push(name), + Utf8Component::CurDir => {} + Utf8Component::ParentDir => { + components.pop().ok_or(Escape::AboveRoot)?; + // Rising out of the base directory re-anchors the entry on an + // ancestor of it, so that ancestry stops being ambient: a + // symlink anywhere in it would put the entry's target outside + // the root even though the base directory itself is inside. + if components.len() < base.len() { + traversed_from = 0; + } + } + Utf8Component::RootDir | Utf8Component::Prefix(_) => return Err(Escape::NotRelative), + } + } + Ok(Resolved { + components, + traversed_from, + }) +} + +impl Resolved<'_> { + /// The entry's location relative to the sandbox root. + pub(crate) fn path(&self) -> Utf8PathBuf { + self.components.iter().copied().collect() + } + + /// Walks the entry one component at a time under `root` and returns the + /// first sub-path that is a symlink, if any. + /// + /// Declared `dirs`/`files` entries are resolved on the host *before* the + /// WASI sandbox boundary, so a symlinked entry — or an entry that traverses + /// a symlinked directory — would let a preopen or a host read escape the + /// project to an arbitrary location on disk (the lexical [`resolve`] check + /// does not catch this). Rejecting any symlink in the traversed portion + /// keeps every preopen and read anchored within the project. Symlinks + /// *inside* a preopen that escape it are separately rejected by the WASI + /// sandbox (cap-std) at runtime. + /// + /// The returned path is relative to `root`, so it can be surfaced in an + /// error without leaking the absolute on-disk location. + /// + /// `root` itself may be reached through symlinks (e.g. the project lives + /// under a symlinked path), as may the base directory — but only for an + /// entry that stays below it (see [`Resolved::traversed_from`]). + /// Components that do not exist are not symlinks, so a + /// missing path returns `None` and the subsequent read or preopen surfaces + /// the not-found error. + pub(crate) fn first_symlink_component(&self, root: &Utf8Path) -> Option { + let mut relative: Utf8PathBuf = self.components[..self.traversed_from] + .iter() + .copied() + .collect(); + let mut host = root.join(&relative); + for name in &self.components[self.traversed_from..] { host.push(name); relative.push(name); match std::fs::symlink_metadata(host.as_std_path()) { @@ -57,8 +136,8 @@ pub(crate) fn first_symlink_component(base: &Utf8Path, rel: &str) -> Option {} } } + None } - None } /// The meaningful components of a declared relative path: the `/`-separated @@ -86,10 +165,17 @@ fn components(path: &str) -> Vec<&str> { /// it. /// /// Retained paths keep their written spelling and first-occurrence order. -/// Comparison is component-wise, so `data` covers `./data/inner` but not -/// `database`. Paths are expected to be relative and free of `..` (see -/// [`escapes_base`]); a `..` compares as an ordinary name, which can only leave -/// the result less reduced. +/// Comparison is over the written spelling rather than the resolved location, +/// because the guest opens each entry at the spelling the manifest gave it, and +/// is component-wise, so `data` covers `./data/inner` but not `database`. +/// +/// A spelling prefix alone is not containment once entries may contain `..`: +/// `..` is a prefix of `../../shared`, yet one is the canister directory's +/// parent and the other a child of its grandparent — neither holds the other. +/// So an entry only covers one whose remaining components descend, `..`-free. +/// Two spellings that coincide only once resolved (`../data` and `data` from a +/// canister in `data`'s parent) still stay separate, which merely leaves the +/// result less reduced. pub fn covering_dirs<'a>(dirs: impl IntoIterator) -> Vec<&'a str> { let dirs: Vec<&str> = dirs.into_iter().collect(); let parts: Vec> = dirs.iter().map(|dir| components(dir)).collect(); @@ -99,6 +185,7 @@ pub fn covering_dirs<'a>(dirs: impl IntoIterator) -> Vec<&'a str !parts.iter().enumerate().any(|(j, other)| { j != *i && parts[*i].starts_with(other) + && !parts[*i][other.len()..].contains(&"..") // A strict ancestor always covers; between equals, the first written wins. && (other.len() < parts[*i].len() || j < *i) }) @@ -151,6 +238,30 @@ mod covering_tests { assert_eq!(covering_dirs(["data", "database"]), ["data", "database"]); } + /// An entry reaching further out than another is not inside it, however + /// much of its spelling they share: `..` is the canister directory's + /// parent, `../../shared` a child of its grandparent. Collapsing them would + /// leave the second with no preopen of its own and none that contains it. + #[test] + fn an_entry_that_rises_further_is_not_covered() { + assert_eq!( + covering_dirs(["..", "../../shared"]), + ["..", "../../shared"], + ); + assert_eq!( + covering_dirs(["../shared", "../shared/../assets"]), + ["../shared", "../shared/../assets"], + ); + } + + /// Entries that reach out of the canister directory still cover what + /// descends from them, and still collapse with a repeat of themselves. + #[test] + fn entries_outside_the_canister_dir_cover_their_own_contents() { + assert_eq!(covering_dirs(["../data", "../data/inner"]), ["../data"]); + assert_eq!(covering_dirs(["../data", "./../data"]), ["../data"]); + } + #[test] fn distinct_paths_dedupes_without_containment() { assert_eq!( @@ -161,34 +272,67 @@ mod covering_tests { } #[cfg(test)] -mod escapes_base_tests { +mod resolve_tests { use super::*; + /// The base directory a canister in `backend/` sits at, relative to the + /// project root. + const BASE: &[&str] = &["backend"]; + + /// The resolved location of `rel`, as a `/`-joined project-relative path. + fn at(base: &[&str], rel: &str) -> Result { + Ok(resolve(base, rel)?.path().to_string()) + } + + /// The base directory's position within the root, `/`-joined. + fn base_rel(root: &str, base: &str) -> Option { + base_within_root(Utf8Path::new(root), Utf8Path::new(base)).map(|parts| parts.join("/")) + } + + #[test] + fn plain_relative_paths_resolve_under_the_base_dir() { + assert_eq!(at(BASE, "a/b").unwrap(), "backend/a/b"); + assert_eq!(at(BASE, "./a").unwrap(), "backend/a"); + assert_eq!(at(BASE, "a/b/file.txt").unwrap(), "backend/a/b/file.txt"); + // A base at the project root leaves the declared path as written. + assert_eq!(at(&[], "a/b").unwrap(), "a/b"); + } + + #[test] + fn parent_components_reach_the_rest_of_the_project() { + assert_eq!(at(BASE, "../shared").unwrap(), "shared"); + assert_eq!(at(BASE, "a/../b").unwrap(), "backend/b"); + assert_eq!( + at(&["services", "crm"], "../../shared/seed").unwrap(), + "shared/seed" + ); + // Rising exactly to the root is fine; the root itself is in bounds. + assert_eq!(at(BASE, "..").unwrap(), ""); + } + #[test] - fn plain_relative_paths_are_safe() { - assert!(!escapes_base("a/b")); - assert!(!escapes_base("./a")); - assert!(!escapes_base("a/b/file.txt")); + fn rising_above_the_root_is_rejected() { + assert_eq!(at(BASE, "../.."), Err(Escape::AboveRoot)); + assert_eq!(at(&[], "../a"), Err(Escape::AboveRoot)); + assert_eq!(at(BASE, "../../../elsewhere"), Err(Escape::AboveRoot)); } #[test] - fn parent_and_root_components_escape() { - assert!(escapes_base("../a")); - assert!(escapes_base("a/../b")); + fn absolute_paths_are_rejected() { // An absolute path carries a `RootDir` component on every platform. - assert!(escapes_base("/abs")); + assert_eq!(at(BASE, "/abs"), Err(Escape::NotRelative)); } // On Windows a drive-relative path like `C:foo` has a `Prefix` component // yet is NOT absolute, so an `is_absolute()` check alone would admit it and - // joining it onto a base would discard the base. `escapes_base` must reject - // it. (On Unix the same string is just an ordinary filename — see below.) + // joining it onto a base would discard the base. `resolve` must reject it. + // (On Unix the same string is just an ordinary filename — see below.) #[cfg(windows)] #[test] - fn windows_drive_and_unc_prefixes_escape() { - assert!(escapes_base("C:foo")); // drive-relative (prefix, no root) - assert!(escapes_base(r"C:\foo")); // absolute (prefix + root) - assert!(escapes_base(r"\\server\share\x")); // UNC prefix + fn windows_drive_and_unc_prefixes_are_rejected() { + assert_eq!(at(BASE, "C:foo"), Err(Escape::NotRelative)); // drive-relative + assert_eq!(at(BASE, r"C:\foo"), Err(Escape::NotRelative)); // absolute + assert_eq!(at(BASE, r"\\server\share\x"), Err(Escape::NotRelative)); // UNC } #[cfg(unix)] @@ -196,7 +340,29 @@ mod escapes_base_tests { fn unix_treats_drive_prefix_as_a_plain_name() { // There is no `Prefix` parsing on Unix, so `C:foo` is just a (weird) // filename with no escaping component. - assert!(!escapes_base("C:foo")); + assert_eq!(at(BASE, "C:foo").unwrap(), "backend/C:foo"); + } + + #[test] + fn base_within_root_is_the_path_from_the_root_down() { + assert_eq!( + base_rel("/work", "/work/backend").as_deref(), + Some("backend") + ); + assert_eq!(base_rel("/work", "/work").as_deref(), Some("")); + assert_eq!( + base_rel(".", "./services/crm").as_deref(), + Some("services/crm") + ); + } + + #[test] + fn base_outside_the_root_has_no_position_within_it() { + // A dependency reached by an out-of-tree `path:` keeps the root as a + // prefix lexically, but resolving the `..` leaves the root behind. + assert_eq!(base_rel("/work", "/work/../outside/backend"), None); + // An unrelated directory is not under the root at all. + assert_eq!(base_rel("/work", "/elsewhere/backend"), None); } } @@ -207,72 +373,142 @@ mod symlink_tests { use camino_tempfile::tempdir; + /// The first symlinked component of `rel`, declared from a canister at + /// `base` and resolved against `root`. + fn first_symlink_from(root: &Utf8Path, base: &[&str], rel: &str) -> Option { + resolve(base, rel).unwrap().first_symlink_component(root) + } + + /// [`first_symlink_from`] for the common case of a canister in `backend/`. + fn first_symlink(root: &Utf8Path, rel: &str) -> Option { + first_symlink_from(root, &["backend"], rel) + } + #[test] fn plain_relative_path_has_no_symlink() { let tmp = tempdir().unwrap(); - let base = tmp.path(); - std::fs::create_dir_all(base.join("a/b")).unwrap(); - std::fs::write(base.join("a/b/file.txt"), b"hi").unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("backend/a/b")).unwrap(); + std::fs::write(root.join("backend/a/b/file.txt"), b"hi").unwrap(); - assert_eq!(first_symlink_component(base, "a/b"), None); - assert_eq!(first_symlink_component(base, "a/b/file.txt"), None); + assert_eq!(first_symlink(root, "a/b"), None); + assert_eq!(first_symlink(root, "a/b/file.txt"), None); } #[test] fn final_entry_is_symlink() { let tmp = tempdir().unwrap(); - let base = tmp.path(); - std::fs::create_dir_all(base.join("real")).unwrap(); - symlink(base.join("real"), base.join("link")).unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("backend/real")).unwrap(); + symlink(root.join("backend/real"), root.join("backend/link")).unwrap(); + // The reported path is relative to the sandbox root, so it names the + // offending component without leaking the absolute on-disk location. assert_eq!( - first_symlink_component(base, "link"), - Some(Utf8PathBuf::from("link")) + first_symlink(root, "link"), + Some(Utf8PathBuf::from("backend/link")) ); } #[test] fn intermediate_component_is_symlink() { let tmp = tempdir().unwrap(); - let base = tmp.path(); - // base/real/inner exists; base/link -> base/real, so "link/inner" - // traverses a symlink even though "inner" itself is a real dir. - std::fs::create_dir_all(base.join("real/inner")).unwrap(); - symlink(base.join("real"), base.join("link")).unwrap(); - - // The reported path is the offending sub-path relative to `base`, - // i.e. the symlinked component, not the trailing real directory. + let root = tmp.path(); + // backend/real/inner exists; backend/link -> backend/real, so + // "link/inner" traverses a symlink even though "inner" is a real dir. + std::fs::create_dir_all(root.join("backend/real/inner")).unwrap(); + symlink(root.join("backend/real"), root.join("backend/link")).unwrap(); + + // The reported path stops at the symlinked component rather than + // continuing to the trailing real directory. assert_eq!( - first_symlink_component(base, "link/inner"), - Some(Utf8PathBuf::from("link")) + first_symlink(root, "link/inner"), + Some(Utf8PathBuf::from("backend/link")) + ); + } + + /// An entry that rises out of the canister directory is checked the whole + /// way down from the root, so a symlink anywhere in the part it traverses + /// is caught. + #[test] + fn symlink_outside_the_base_dir_is_caught() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("backend")).unwrap(); + std::fs::create_dir_all(root.join("real")).unwrap(); + symlink(root.join("real"), root.join("shared")).unwrap(); + + assert_eq!( + first_symlink(root, "../shared"), + Some(Utf8PathBuf::from("shared")) ); } #[test] fn missing_path_is_not_a_symlink() { let tmp = tempdir().unwrap(); - let base = tmp.path(); - assert_eq!(first_symlink_component(base, "does/not/exist"), None); + assert_eq!(first_symlink(tmp.path(), "does/not/exist"), None); } #[test] fn dot_components_are_ignored() { let tmp = tempdir().unwrap(); - let base = tmp.path(); - std::fs::create_dir_all(base.join("a")).unwrap(); - assert_eq!(first_symlink_component(base, "./a"), None); + let root = tmp.path(); + std::fs::create_dir_all(root.join("backend/a")).unwrap(); + assert_eq!(first_symlink(root, "./a"), None); + } + + #[test] + fn symlinked_root_is_allowed() { + // A symlink *above* the sandbox root is fine; only what the declared + // entry traverses below the root is checked. + let tmp = tempdir().unwrap(); + let real_root = tmp.path().join("real-root"); + std::fs::create_dir_all(real_root.join("backend/data")).unwrap(); + let linked_root = tmp.path().join("linked-root"); + symlink(&real_root, &linked_root).unwrap(); + + assert_eq!(first_symlink(&linked_root, "data"), None); } + /// The base directory's own ancestry is exempt as long as the entry does + /// not rise out of it: a symlinked canister directory is how the project + /// reaches its own canister, not something the manifest declared. #[test] - fn symlinked_base_is_allowed() { - // A symlink *above* the declared portion (i.e. reaching `base`) is fine; - // only components of `rel` are checked. + fn symlinked_base_dir_is_allowed_for_an_entry_below_it() { let tmp = tempdir().unwrap(); - let real_base = tmp.path().join("real-base"); - std::fs::create_dir_all(real_base.join("data")).unwrap(); - let linked_base = tmp.path().join("linked-base"); - symlink(&real_base, &linked_base).unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("real-backend/data")).unwrap(); + symlink(root.join("real-backend"), root.join("backend")).unwrap(); - assert_eq!(first_symlink_component(&linked_base, "data"), None); + assert_eq!(first_symlink(root, "data"), None); + // Rising out of it puts that ancestry back in scope, and it is rejected. + assert_eq!( + first_symlink(root, "../backend/data"), + Some(Utf8PathBuf::from("backend")) + ); + } + + /// An entry that rises out of the canister directory is checked from the + /// root down, not just from where it re-anchored. Without that, a symlink + /// above the canister's own directory would land the preopen outside the + /// project — here `canisters` leads out of the project, so `../secrets` + /// would otherwise open a directory the project does not contain. + #[test] + fn symlinked_ancestor_of_a_deep_base_dir_is_caught() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + std::fs::create_dir_all(root.join("elsewhere/backend")).unwrap(); + std::fs::create_dir_all(root.join("elsewhere/secrets")).unwrap(); + symlink(root.join("elsewhere"), root.join("canisters")).unwrap(); + + let base = &["canisters", "backend"]; + assert_eq!( + first_symlink_from(root, base, "../secrets"), + Some(Utf8PathBuf::from("canisters")) + ); + // An entry that stays below the canister directory is unaffected: that + // ancestry is how the project reaches the canister either way. + assert_eq!(first_symlink_from(root, base, "data"), None); } } diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index c7690e7b4..f87fef0b1 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -279,10 +279,17 @@ pub enum RunPluginError { path: Utf8PathBuf, }, + #[snafu(display("plugin dir '{dir}' is not a relative path (absolute paths are not allowed)"))] + UnsafeDir { dir: String }, + #[snafu(display( - "plugin dir '{dir}' is not a safe relative path (no absolute paths or '..' allowed)" + "plugin dir '{dir}' resolves outside '{project_dir}'; \ + a plugin may only read paths inside the project directory" ))] - UnsafeDir { dir: String }, + DirOutsideProject { + dir: String, + project_dir: Utf8PathBuf, + }, #[snafu(display( "plugin dir '{dir}' resolves through a symlink ('{link}'); symlinks are not allowed in plugin dirs" @@ -299,10 +306,19 @@ pub enum RunPluginError { }, #[snafu(display( - "plugin file '{name}' is not a safe relative path (no absolute paths or '..' allowed)" + "plugin file '{name}' is not a relative path (absolute paths are not allowed)" ))] UnsafeFile { name: String }, + #[snafu(display( + "plugin file '{name}' resolves outside '{project_dir}'; \ + a plugin may only read paths inside the project directory" + ))] + FileOutsideProject { + name: String, + project_dir: Utf8PathBuf, + }, + #[snafu(display( "plugin file '{name}' resolves through a symlink ('{link}'); symlinks are not allowed in plugin files" ))] @@ -411,6 +427,14 @@ pub struct PluginInvocation { pub wasm_path: Utf8PathBuf, /// Directory the declared `dirs`/`files` are anchored at (the canister dir). pub base_dir: Utf8PathBuf, + /// The project directory: the sandbox boundary. A declared path may rise + /// out of `base_dir` with `..` and reach anything inside the project, but + /// nothing above it. + /// + /// A `base_dir` that does not lie within this directory — a dependency + /// project reached by an out-of-tree `path:` — is its own boundary instead, + /// which grants nothing above the canister directory. + pub project_dir: Utf8PathBuf, /// Manifest-relative directories to preopen read-only, each tagged with the /// map key it was declared under (if any). pub dirs: Vec, @@ -448,6 +472,7 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin let PluginInvocation { wasm_path, base_dir, + project_dir, dirs, files, fields, @@ -501,21 +526,44 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin path: wasm_path.clone(), })?; + // Declared paths are written relative to the canister directory but are + // resolved against — and confined to — the project directory, so an entry + // may reach a sibling canister's tree with `..` while nothing outside the + // project is reachable. A canister directory that lies outside the project + // is its own root, which grants nothing above it (see `project_dir`). + let (root, base_rel) = match crate::path::base_within_root(&project_dir, &base_dir) { + Some(base_rel) => (&project_dir, base_rel), + None => (&base_dir, Vec::new()), + }; + // Check every declared directory: each one is handed to the plugin as // configuration, so it is rejected for being unsafe or unusable whether or - // not it ends up needing a preopen of its own. + // not it ends up needing a preopen of its own. The resolved host paths are + // kept for the preopens below, keyed by the spelling they were declared as. + let mut host_paths: BTreeMap<&str, Utf8PathBuf> = BTreeMap::new(); for KeyedPath { path: dir, .. } in &dirs { - ensure!(!crate::path::escapes_base(dir), UnsafeDirSnafu { dir }); - // Reject symlinks in the declared path: neither the final entry nor any + let resolved = match crate::path::resolve(&base_rel, dir) { + Ok(resolved) => resolved, + Err(crate::path::Escape::NotRelative) => return UnsafeDirSnafu { dir }.fail(), + Err(crate::path::Escape::AboveRoot) => { + return DirOutsideProjectSnafu { + dir, + project_dir: root, + } + .fail(); + } + }; + // Reject symlinks in the resolved path: neither the final entry nor any // intermediate component may be a symlink, so the preopen cannot escape - // `base_dir` to a target elsewhere on disk. (Symlinks *inside* a preopen - // that escape it are separately rejected by the WASI sandbox.) - if let Some(link) = crate::path::first_symlink_component(&base_dir, dir) { + // the project to a target elsewhere on disk. (Symlinks *inside* a + // preopen that escape it are separately rejected by the WASI sandbox.) + if let Some(link) = resolved.first_symlink_component(root) { return SymlinkDirSnafu { dir, link }.fail(); } - let host_path = base_dir.join(dir); + let host_path = root.join(resolved.path()); let is_dir = std::fs::metadata(host_path.as_std_path()).is_ok_and(|meta| meta.is_dir()); ensure!(is_dir, MissingDirSnafu { dir }); + host_paths.insert(dir, host_path); } // Preopen read-only, one per distinct tree — a directory declared twice, or @@ -524,7 +572,11 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin // manifest, and reaches a nested declared directory through its ancestor. let mut wasi_builder = wasmtime_wasi::WasiCtxBuilder::new(); for dir in crate::path::covering_dirs(dirs.iter().map(|d| d.path.as_str())) { - let host_path = base_dir.join(dir); + // `covering_dirs` returns a subset of the declared spellings, every one + // of which the loop above resolved. + let host_path = host_paths + .get(dir) + .expect("covering dir was not among the declared dirs"); wasi_builder .preopened_dir( host_path.as_std_path(), @@ -532,19 +584,31 @@ pub fn run_plugin(invocation: PluginInvocation) -> Result, RunPlugin DirPerms::READ, FilePerms::READ, ) - .context(PreopenDirSnafu { dir: host_path })?; + .context(PreopenDirSnafu { + dir: host_path.clone(), + })?; } // Read each declared file on the host and pass its content inline. The same - // path-safety checks as `dirs` apply: reject escaping or symlinked paths so - // a read cannot leave `base_dir`. + // path-safety checks as `dirs` apply: reject unsafe, escaping, or symlinked + // paths so a read cannot leave the project. let mut file_contents: Vec = Vec::with_capacity(files.len()); for KeyedPath { key, path: name } in &files { - ensure!(!crate::path::escapes_base(name), UnsafeFileSnafu { name }); - if let Some(link) = crate::path::first_symlink_component(&base_dir, name) { + let resolved = match crate::path::resolve(&base_rel, name) { + Ok(resolved) => resolved, + Err(crate::path::Escape::NotRelative) => return UnsafeFileSnafu { name }.fail(), + Err(crate::path::Escape::AboveRoot) => { + return FileOutsideProjectSnafu { + name, + project_dir: root, + } + .fail(); + } + }; + if let Some(link) = resolved.first_symlink_component(root) { return SymlinkFileSnafu { name, link }.fail(); } - let path = base_dir.join(name); + let path = root.join(resolved.path()); let content = std::fs::read_to_string(path.as_std_path()).context(ReadFileSnafu { path })?; file_contents.push(FileContent { @@ -870,12 +934,13 @@ mod tests { /// A [`PluginInvocation`] with test-friendly defaults: anonymous canister /// and identity, no proxy, no declared callable canisters, the default - /// compute limit, and the current directory as the base. Tests override - /// the few fields they care about. + /// compute limit, and the current directory as both the base and the + /// project. Tests override the few fields they care about. fn invocation(wasm_path: &str, environment: &str) -> PluginInvocation { PluginInvocation { wasm_path: wasm_path.into(), base_dir: ".".into(), + project_dir: ".".into(), dirs: vec![], files: vec![], fields: BTreeMap::new(), @@ -1000,6 +1065,159 @@ mod tests { ); } + /// A `dirs:` entry may rise out of the canister directory into the rest of + /// the project. The guest sees it at the path it was declared as, so it + /// reads `../shared` verbatim, and a directory below the entry is reached + /// through the same preopen. + #[test] + fn dirs_above_the_canister_dir_are_readable() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let root = tmp.path(); + std::fs::create_dir_all(root.join("canisters/backend")).expect("create dir"); + std::fs::create_dir_all(root.join("shared/inner")).expect("create dir"); + std::fs::write(root.join("shared/top.txt"), b"top").expect("write file"); + std::fs::write(root.join("shared/inner/deep.txt"), b"deep").expect("write file"); + + let mut inv = invocation(wasm_path, "read-dirs"); + inv.base_dir = root.join("canisters/backend"); + inv.project_dir = root.to_path_buf(); + inv.dirs = [("shared", "../../shared"), ("inner", "../../shared/inner")] + .into_iter() + .map(|(key, path)| KeyedPath { + key: Some(key.to_owned()), + path: path.to_owned(), + }) + .collect(); + + let lines = run_plugin(inv).expect("plugin should succeed"); + assert_eq!( + lines, + [ + "shared=inner,top.txt".to_string(), + "inner=deep.txt".to_string(), + ], + ); + } + + /// Two entries where one reaches further out than the other each need their + /// own preopen: `..` is the canister's parent and `../../shared` a child of + /// its grandparent, so neither is readable through the other's. + #[test] + fn dirs_reaching_out_by_different_amounts_are_both_readable() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let root = tmp.path(); + std::fs::create_dir_all(root.join("canisters/backend")).expect("create dir"); + std::fs::create_dir_all(root.join("shared")).expect("create dir"); + std::fs::write(root.join("shared/top.txt"), b"top").expect("write file"); + + let mut inv = invocation(wasm_path, "read-dirs"); + inv.base_dir = root.join("canisters/backend"); + inv.project_dir = root.to_path_buf(); + inv.dirs = [("siblings", ".."), ("shared", "../../shared")] + .into_iter() + .map(|(key, path)| KeyedPath { + key: Some(key.to_owned()), + path: path.to_owned(), + }) + .collect(); + + let lines = run_plugin(inv).expect("plugin should succeed"); + assert_eq!( + lines, + ["siblings=backend".to_string(), "shared=top.txt".to_string(),], + ); + } + + /// The project directory is the boundary: an entry that rises above it is + /// rejected before the plugin runs. + #[test] + fn dir_above_the_project_is_rejected() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let root = tmp.path().join("project"); + std::fs::create_dir_all(root.join("backend")).expect("create dir"); + std::fs::create_dir_all(tmp.path().join("outside")).expect("create dir"); + + let mut inv = invocation(wasm_path, "read-dirs"); + inv.base_dir = root.join("backend"); + inv.project_dir = root.clone(); + inv.dirs = unkeyed(&["../../outside"]); + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::DirOutsideProject { .. }) + )); + } + + /// A canister directory outside the project — a dependency reached by an + /// out-of-tree path — is its own boundary, so nothing above it is reachable. + #[test] + fn dir_above_an_out_of_project_canister_dir_is_rejected() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + std::fs::create_dir_all(tmp.path().join("outside/backend")).expect("create dir"); + std::fs::create_dir_all(tmp.path().join("outside/shared")).expect("create dir"); + + let mut inv = invocation(wasm_path, "read-dirs"); + inv.base_dir = tmp.path().join("outside/backend"); + inv.project_dir = tmp.path().join("project"); + inv.dirs = unkeyed(&["../shared"]); + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::DirOutsideProject { .. }) + )); + } + + /// A `files:` entry may reach the rest of the project too; its content is + /// read by the host and passed inline. + #[test] + fn files_above_the_canister_dir_are_read() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let root = tmp.path(); + std::fs::create_dir_all(root.join("backend")).expect("create dir"); + std::fs::write(root.join("cfg.txt"), b"data").expect("write file"); + + let mut inv = invocation(wasm_path, "keys"); + inv.base_dir = root.join("backend"); + inv.project_dir = root.to_path_buf(); + inv.files = unkeyed(&["../cfg.txt"]); + + let lines = run_plugin(inv).expect("plugin should succeed"); + assert_eq!(lines, ["file -=../cfg.txt".to_string()]); + } + + #[test] + fn file_above_the_project_is_rejected() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let tmp = camino_tempfile::tempdir().expect("create tempdir"); + let root = tmp.path().join("project"); + std::fs::create_dir_all(root.join("backend")).expect("create dir"); + std::fs::write(tmp.path().join("secret.txt"), b"secret").expect("write file"); + + let mut inv = invocation(wasm_path, "keys"); + inv.base_dir = root.join("backend"); + inv.project_dir = root.clone(); + inv.files = unkeyed(&["../../secret.txt"]); + assert!(matches!( + run_plugin(inv), + Err(RunPluginError::FileOutsideProject { .. }) + )); + } + #[cfg(unix)] #[test] fn symlinked_dir_is_rejected() { diff --git a/crates/icp/src/canister/sync/mod.rs b/crates/icp/src/canister/sync/mod.rs index a90ff93be..996ec99f2 100644 --- a/crates/icp/src/canister/sync/mod.rs +++ b/crates/icp/src/canister/sync/mod.rs @@ -18,6 +18,10 @@ use script::{HostScripts, ScriptInvocation, ScriptRunError, ScriptRunner}; pub struct Params { pub path: PathBuf, + /// The project (workspace root) directory. It bounds what a sync plugin may + /// read: a declared `dirs`/`files` entry may rise out of the canister + /// directory into the rest of the project, but not out of the project. + pub project_dir: PathBuf, pub cid: Principal, /// Fully-qualified store key of the canister being synced (e.g. `backend`, /// or `services/open-crm:backend` for a canister in a subproject). Its namespace @@ -168,6 +172,7 @@ mod tests { let cid = Principal::from_slice(&[7; 4]); let params = Params { path: "/work/backend".into(), + project_dir: "/work".into(), cid, name: "backend".to_owned(), environment: "production".to_owned(), diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index fbde4dde4..43f1bf980 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -110,10 +110,11 @@ pub(super) async fn sync( .await?; // 2. Collect inputs as manifest strings. `run_plugin` preopens the `dirs` - // and reads the `files` itself — both anchored at `base_dir`, and both - // subject to the runtime's path-safety checks (no escaping or symlinked - // paths). + // and reads the `files` itself — both anchored at `base_dir`, confined to + // `project_dir`, and subject to the runtime's path-safety checks (no + // escaping or symlinked paths). let base_dir = Utf8PathBuf::from(params.path.as_str()); + let project_dir = Utf8PathBuf::from(params.project_dir.as_str()); let dirs = keyed_paths(adapter.dirs.as_ref()); let files = keyed_paths(adapter.files.as_ref()); let fields: BTreeMap = adapter.fields.clone().unwrap_or_default(); @@ -136,6 +137,7 @@ pub(super) async fn sync( run_plugin(PluginInvocation { wasm_path, base_dir, + project_dir, dirs, files, fields, @@ -233,6 +235,7 @@ mod tests { fn params_named(name: &str, ids: &[(&str, Principal)]) -> Params { Params { path: "/work".into(), + project_dir: "/work".into(), cid: principal(0), name: name.to_owned(), environment: "demo".to_owned(), diff --git a/crates/icp/src/canister/sync/script.rs b/crates/icp/src/canister/sync/script.rs index e26d73171..982396495 100644 --- a/crates/icp/src/canister/sync/script.rs +++ b/crates/icp/src/canister/sync/script.rs @@ -144,6 +144,7 @@ mod tests { fn params(canister_ids: &[(&str, Principal)]) -> Params { Params { path: "/work/backend".into(), + project_dir: "/work".into(), cid: principal(1), name: "backend".to_owned(), environment: "production".to_owned(), diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 9c7069ecf..d37189388 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -185,7 +185,8 @@ impl PathOrList { /// specific canister. It runs inside a WASI sandbox whose filesystem access /// is limited to the directories listed in `dirs` (preopened read-only) plus /// the contents of any files listed in `files` (read by the host and passed -/// inline to the plugin). +/// inline to the plugin). Both are written relative to the canister directory +/// and may name anything inside the project, but nothing outside it. /// /// Example (local path): /// ```yaml @@ -228,19 +229,22 @@ pub struct Adapter { /// Optional for `path`; required for `url`. pub sha256: Option, - /// Directories (relative to canister directory) the plugin may read from. - /// Each entry must be a directory; it is made readable via WASI so the - /// plugin can traverse it using standard filesystem APIs. Written as a plain - /// list of paths, or as a map of name → path (or list of paths); the name is - /// surfaced to the plugin as each entry's `key`. Entries may repeat a - /// directory or name one inside another's — the plugin is told about each - /// entry as written, and reads them all. + /// Directories the plugin may read from, written relative to the canister + /// directory and confined to the project (an entry may reach elsewhere in + /// the project with `..`, but not out of it). Each entry must be a + /// directory; it is made readable via WASI so the plugin can traverse it + /// using standard filesystem APIs. Written as a plain list of paths, or as + /// a map of name → path (or list of paths); the name is surfaced to the + /// plugin as each entry's `key`. Entries may repeat a directory or name one + /// inside another's — the plugin is told about each entry as written, and + /// reads them all. pub dirs: Option, - /// Files (relative to canister directory) the host reads and passes to - /// the plugin as part of `sync-exec-input.files`. Written as a plain list - /// of paths, or as a map of name → path (or list of paths); the name is - /// surfaced to the plugin as each entry's `key`. + /// Files the host reads and passes to the plugin as part of + /// `sync-exec-input.files`, written relative to the canister directory and + /// confined to the project on the same terms as [`Self::dirs`]. Written as + /// a plain list of paths, or as a map of name → path (or list of paths); + /// the name is surfaced to the plugin as each entry's `key`. pub files: Option, /// Key-value fields passed to the plugin as part of `sync-exec-input.fields`. diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 2c53a347f..e45409d64 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -112,7 +112,8 @@ The plugin runs with a deliberately narrow capability surface. - Entries may name the same directory under several keys, or name a directory inside another entry's, and the plugin is told about each entry as written. The preopens behind them are one per distinct tree: an entry nested inside another is read through the preopen covering it, which grants nothing extra. - Files in `files:` are read by the host up front and passed inline in `sync-exec-input.files`. The plugin reads their content from the input struct, not from disk. - Any path outside a preopen is invisible. Writes, creates, deletes, renames, and symlinks that escape a preopen are rejected by the sandbox at runtime. -- Paths in `dirs:`/`files:` must be relative and may not contain `..`. They also may not be — or traverse — a symlink: each declared entry is rejected if it or any of its parent components is a symlink, so a declared path cannot resolve to a target outside the canister directory. (This restriction may be relaxed later if a safe use case emerges.) +- Paths in `dirs:`/`files:` are relative to the canister directory and may rise out of it with `..` to reach the rest of the project (`dirs: ["../shared/assets"]`). The project directory is the boundary: an entry that resolves above it — or that is absolute — is rejected before the plugin runs. +- A declared entry may not be, or traverse, a symlink: it is rejected if it or any component it traverses below the project root is a symlink, so a declared path cannot resolve to a target outside the project. (This restriction may be relaxed later if a safe use case emerges.) ### Capabilities diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 8e466b585..557c89d8a 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -114,7 +114,7 @@ for file in &input.files { Declaring `dirs:`/`files:` as a map instead of a list tags each entry with a `key`, so a plugin can group or label paths (for example, tell `seed:` directories from `migrations:`) without hardcoding paths. A key that maps to a list of paths yields several entries sharing that key. -Writes, paths outside a preopen, and `..` traversal are all rejected by the sandbox. See [The Sandbox](../concepts/sync-plugins.md#the-sandbox) for the full capability list and resource limits. +Open each entry at the `path` it arrives with, whatever it looks like: a manifest may declare a directory elsewhere in the project (`../shared/assets`), and the preopen carries that same spelling. Writes, and paths that escape a preopen, are rejected by the sandbox at runtime. See [The Sandbox](../concepts/sync-plugins.md#the-sandbox) for the full capability list and resource limits. ## Read Declared Fields diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b1a46582e..ecba7009d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -169,8 +169,8 @@ sync: | `path` | string | One of `path` or `url` | Local path to the wasm, relative to the canister directory | | `url` | string | One of `path` or `url` | URL to download the wasm from | | `sha256` | string | Required for `url`, optional for `path` | SHA-256 hex digest of the wasm file, verified before execution | -| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory) the plugin may read; each is made readable read-only via WASI | -| `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory) read by the host and passed inline to the plugin | +| `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory, anywhere inside the project) the plugin may read; each is made readable read-only via WASI | +| `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory, anywhere inside the project) read by the host and passed inline to the plugin | | `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | | `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | @@ -188,7 +188,7 @@ sync: - config.txt # a plain list is still fine ``` -Entries in `dirs:`/`files:` must be relative, may not contain `..`, and may not be — or traverse — a symlink, so a declared path cannot resolve to a target outside the canister directory. +Entries in `dirs:`/`files:` must be relative to the canister directory. They may reach the rest of the project with `..` — `dirs: ["../shared/assets"]` is fine — but may not resolve outside the project directory. They may not be, or traverse, a symlink, so a declared path cannot resolve to a target outside the project. A plugin receives every `fields:` value as a string. Numbers and booleans need no quoting — `port: 8080` arrives as `"8080"` — but a value may not be a list, a mapping, or empty. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 9c2c60912..5934033cc 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -88,7 +88,7 @@ "type": "null" } ], - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is made readable via WASI so the\nplugin can traverse it using standard filesystem APIs. Written as a plain\nlist of paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`. Entries may repeat a\ndirectory or name one inside another's — the plugin is told about each\nentry as written, and reads them all." + "description": "Directories the plugin may read from, written relative to the canister\ndirectory and confined to the project (an entry may reach elsewhere in\nthe project with `..`, but not out of it). Each entry must be a\ndirectory; it is made readable via WASI so the plugin can traverse it\nusing standard filesystem APIs. Written as a plain list of paths, or as\na map of name → path (or list of paths); the name is surfaced to the\nplugin as each entry's `key`. Entries may repeat a directory or name one\ninside another's — the plugin is told about each entry as written, and\nreads them all." }, "fields": { "additionalProperties": { @@ -113,7 +113,7 @@ "type": "null" } ], - "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." + "description": "Files the host reads and passes to the plugin as part of\n`sync-exec-input.files`, written relative to the canister directory and\nconfined to the project on the same terms as [`Self::dirs`]. Written as\na plain list of paths, or as a map of name → path (or list of paths);\nthe name is surfaced to the plugin as each entry's `key`." }, "sha256": { "description": "Optional sha256 checksum of the wasm file.\nOptional for `path`; required for `url`.", diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index ef3096767..18f217425 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -67,7 +67,7 @@ "description": "Remote url to fetch a WASM file from" } ], - "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin).\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", + "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", @@ -88,7 +88,7 @@ "type": "null" } ], - "description": "Directories (relative to canister directory) the plugin may read from.\nEach entry must be a directory; it is made readable via WASI so the\nplugin can traverse it using standard filesystem APIs. Written as a plain\nlist of paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`. Entries may repeat a\ndirectory or name one inside another's — the plugin is told about each\nentry as written, and reads them all." + "description": "Directories the plugin may read from, written relative to the canister\ndirectory and confined to the project (an entry may reach elsewhere in\nthe project with `..`, but not out of it). Each entry must be a\ndirectory; it is made readable via WASI so the plugin can traverse it\nusing standard filesystem APIs. Written as a plain list of paths, or as\na map of name → path (or list of paths); the name is surfaced to the\nplugin as each entry's `key`. Entries may repeat a directory or name one\ninside another's — the plugin is told about each entry as written, and\nreads them all." }, "fields": { "additionalProperties": { @@ -113,7 +113,7 @@ "type": "null" } ], - "description": "Files (relative to canister directory) the host reads and passes to\nthe plugin as part of `sync-exec-input.files`. Written as a plain list\nof paths, or as a map of name → path (or list of paths); the name is\nsurfaced to the plugin as each entry's `key`." + "description": "Files the host reads and passes to the plugin as part of\n`sync-exec-input.files`, written relative to the canister directory and\nconfined to the project on the same terms as [`Self::dirs`]. Written as\na plain list of paths, or as a map of name → path (or list of paths);\nthe name is surfaced to the plugin as each entry's `key`." }, "sha256": { "description": "Optional sha256 checksum of the wasm file.\nOptional for `path`; required for `url`.", From 42744ed362bfda5c014cda3fa348847741237846 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Sat, 22 Aug 2026 21:34:47 -0700 Subject: [PATCH 45/51] Add get-metadata-section function --- Cargo.lock | 1 + crates/icp-cli/tests/sync_tests.rs | 18 ++- crates/icp-sync-plugin/Cargo.toml | 1 + crates/icp-sync-plugin/DESIGN.md | 46 ++++++- crates/icp-sync-plugin/src/runtime.rs | 128 ++++++++++++++++-- crates/icp-sync-plugin/sync-plugin.wit | 36 ++++- .../tests/fixtures/test-plugin/src/lib.rs | 13 ++ crates/icp/src/manifest/adapter/plugin.rs | 11 +- docs/concepts/sync-plugins.md | 30 +++- docs/guides/writing-sync-plugins.md | 22 ++- docs/reference/configuration.md | 6 +- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- examples/icp-sync-plugin/README.md | 23 +++- examples/icp-sync-plugin/plugin/src/lib.rs | 17 ++- 15 files changed, 319 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b68d3f5b2..bd140f275 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3785,6 +3785,7 @@ dependencies = [ "console 0.16.3", "hex", "ic-agent", + "ic-management-canister-types 0.8.0", "icp-canister-interfaces", "semver", "snafu", diff --git a/crates/icp-cli/tests/sync_tests.rs b/crates/icp-cli/tests/sync_tests.rs index c93f21c8e..14afac03d 100644 --- a/crates/icp-cli/tests/sync_tests.rs +++ b/crates/icp-cli/tests/sync_tests.rs @@ -437,11 +437,18 @@ async fn sync_plugin_registers_seed_data() { clients::icp(&ctx, &project_dir, Some("random-environment".to_string())) .mint_cycles(10 * TRILLION); + // The plugin also reads the canister's candid:service metadata section and + // reports it. No proxy is configured here, so the read is a direct + // read_state; this manifest builds the wasm with a plain `cp`, skipping the + // example's ic-wasm step, so the section genuinely isn't there — proving the + // host performed the round-trip and mapped a proven-absent section to `none` + // rather than to an error. ctx.icp() .current_dir(&project_dir) .args(["deploy", "--environment", "random-environment"]) .assert() - .success(); + .success() + .stderr(contains("candid:service: absent")); // Query the canister to verify all three fruits were registered ctx.icp() @@ -940,6 +947,12 @@ async fn sync_plugin_routes_through_proxy() { // Deploy through proxy so the proxy canister becomes a controller of my-canister. // deploy also runs the sync step: the plugin routes set_uploader through the proxy // (direct: false, proxy is controller), then calls register directly with the user identity. + // + // Its metadata read is proxied too, so it reaches the canister as the + // management canister's `canister_metadata` rather than as a read_state. + // This manifest skips the example's ic-wasm step, so the section really is + // missing — and the host must report the resulting rejection as an absent + // section, the same answer a direct read proves from the certificate. ctx.icp() .current_dir(&project_dir) .args([ @@ -950,7 +963,8 @@ async fn sync_plugin_routes_through_proxy() { "random-environment", ]) .assert() - .success(); + .success() + .stderr(contains("candid:service: absent")); // Query the canister to verify all three fruits were registered ctx.icp() diff --git a/crates/icp-sync-plugin/Cargo.toml b/crates/icp-sync-plugin/Cargo.toml index 74feb682e..a43722968 100644 --- a/crates/icp-sync-plugin/Cargo.toml +++ b/crates/icp-sync-plugin/Cargo.toml @@ -14,6 +14,7 @@ candid.workspace = true console.workspace = true hex.workspace = true ic-agent.workspace = true +ic-management-canister-types.workspace = true icp-canister-interfaces.workspace = true semver.workspace = true snafu.workspace = true diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 9f1d8b95a..e3649cf95 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -36,6 +36,13 @@ docs; the *reasons* behind those choices are recorded here. one deployment. (In the earlier `@0.1.0` interface `canister-call` had no target and always reached the canister being synced; see *Interface versioning* below.) +- **`get-metadata-section` mirrors `canister-call`'s targeting and routing** — it + takes the same `call-target` (enforced against `canisters:` the same way) and + the same `direct` flag, so one mental model covers both imports. Its return is + `result>, string>`: a missing section is an ordinary answer for + a plugin probing for an optional section, not a failure it must recognize by + parsing error text. The host pays for that guarantee on the proxied path — see + *Metadata reads* below. - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still @@ -65,7 +72,8 @@ crates/icp-sync-plugin/ path.rs — declared-path resolution and safety checks (project bound, symlinks) sync-plugin.wit — current WIT interface, v0.2.0 sync-plugin-v1.wit — frozen WIT interface, v0.1.0 - Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, candid, camino, snafu, tokio, semver + Cargo.toml — wasmtime, wasmtime-wasi, ic-agent, ic-management-canister-types, + candid, camino, snafu, tokio, semver ``` Public function: @@ -160,8 +168,8 @@ struct HostState { ``` `HostState` implements `WasiView` so wasmtime_wasi can access the WASI context. -`canister_call` uses `tokio::runtime::Handle::current().block_on(...)` because -the caller already wraps the synchronous `run_plugin` in +Both imports use `tokio::runtime::Handle::current().block_on(...)` because the +caller already wraps the synchronous `run_plugin` in `tokio::task::block_in_place`. For a v0.2.0 plugin the target is resolved from the request's `call-target` by `resolve_call_target`, which enforces the `callable` set; for a v0.1.0 plugin the target is always `host_canister_id`. @@ -169,6 +177,29 @@ When a proxy is configured and the call is a non-`direct` update, it is encoded as `ProxyArgs` and routed through the proxy's `proxy` method; otherwise it goes straight to the resolved target via `ic-agent`. +### Metadata reads (two routes, one answer) + +`get-metadata-section` cannot reuse the call path: `read_state` is not a canister +method, so a proxy canister has nothing to forward. The two routes are therefore +different protocols reaching the same data, chosen by the request's `direct` flag +exactly as `canister-call` chooses one: + +- **Direct** — `Agent::read_state_canister_metadata`, signed by the sync + identity. Absence is *proven* by the certificate, surfacing as + `AgentError::LookupPathAbsent`, which the host maps to `Ok(None)`. +- **Proxied** — `ProxyArgs` aimed at the management canister's + `canister_metadata`, so the controller check runs against the proxy. This is + the same shape the CLI's own management calls take through + `update_or_proxy_raw`; the runtime inlines it rather than depending on the CLI. + +The routes disagree on how absence arrives: the management canister *rejects* +("The canister `` has no metadata section with the name ``.") and the +proxy hands the plugin a reason string with no reject code attached, so the host +matches `NO_SUCH_SECTION_REJECT` against it to produce the same `Ok(None)` a +direct read proves. Matching replica text is the price of one uniform contract; +it fails in the safe direction — a reword upstream turns absence back into an +error rather than into a wrong answer. + ### Interface versioning (parallel v0.1.0 / v0.2.0 support) A component built with wit-bindgen imports the interface it `use`s as a @@ -189,10 +220,11 @@ simply dropped for a v1 plugin; a v1 plugin cannot observe them, so declaring The compute-time limit is enforced with wasmtime's epoch interruption: a background thread calls `Engine::increment_epoch` once per second, and the store -deadline (`set_epoch_deadline`) bounds pure wasm execution. Because canister -calls block the guest while the host awaits the network, `canister_call` records -the elapsed time and the `epoch_deadline_callback` grants it back via -`epoch_extension` — so network latency is *not* charged against the limit. The +deadline (`set_epoch_deadline`) bounds pure wasm execution. Because a host +call blocks the guest while the host awaits the network, both imports record the +elapsed time (`refund_host_call_time`) and the `epoch_deadline_callback` grants +it back via `epoch_extension` — so network latency is *not* charged against the +limit. The ticker thread stops when its RAII guard drops at the end of `run_plugin`. The deadline in seconds is the `compute_limit_secs` parameter. The CLI resolves diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index f87fef0b1..35df8f9a9 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -24,7 +24,9 @@ pub const PLUGIN_COMPUTE_LIMIT_ENV: &str = "ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS"; use bytes::Bytes; use camino::Utf8PathBuf; use candid::{Encode, Principal}; -use ic_agent::Agent; +use ic_agent::{Agent, AgentError}; +use ic_management_canister_types::{CanisterMetadataArgs, CanisterMetadataResult}; +use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; use semver::{Version, VersionReq}; use snafu::prelude::*; use tokio::io::{self, AsyncWrite}; @@ -93,6 +95,16 @@ pub struct CallableCanisters { pub by_name: BTreeMap, } +/// The distinguishing phrase in the management canister's rejection of a +/// metadata read for a section the target does not have ("The canister has +/// no metadata section with the name ."). A proxied read reaches the +/// plugin as reject text, not as a code, so recognizing absence — which +/// [`HostState::do_get_metadata_section`] reports as `Ok(None)`, matching what +/// a direct read proves from the certificate — means matching that text. A +/// reword upstream turns absence back into an error rather than into a wrong +/// answer. +const NO_SUCH_SECTION_REJECT: &str = "no metadata section"; + /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing /// that the plugin listed it in `canisters`. The canister being synced (`host`) /// is always permitted. @@ -119,7 +131,8 @@ struct HostState { /// Canisters the plugin declared in `canisters` and may also call. callable: CallableCanisters, agent: Arc, - /// Proxy canister to route update calls through, if configured. + /// Proxy canister to route update calls and metadata reads through, if + /// configured. proxy: Option, // WASI context. Preopened directories in this context are the only // filesystem locations the plugin can access. @@ -154,8 +167,6 @@ impl HostState { direct: bool, cycles: u64, ) -> Result, String> { - use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; - let agent = Arc::clone(&self.agent); let proxy = if direct { None } else { self.proxy }; @@ -201,12 +212,85 @@ impl HostState { .map_err(|e| format!("canister call failed: {e}")), } }); - // Return the time spent in the host call to the compute budget so - // canister network latency doesn't count against the plugin's limit. + self.refund_host_call_time(start); + result + } + + /// Read a metadata section from an already-resolved target principal. + /// `Ok(None)` is the target reporting it has no such section, kept distinct + /// from a failed read so a plugin can probe for an optional section without + /// inspecting error text (see [`NO_SUCH_SECTION_REJECT`]). + /// + /// A direct read is a certified `read_state` signed by the sync identity — + /// `read_state` is not a canister method, so it cannot be forwarded. A + /// proxied read therefore goes the other way around: the proxy calls the + /// management canister's `canister_metadata` on the plugin's behalf, which + /// checks the *proxy* against the target's controllers and so reaches + /// sections private to it. + fn do_get_metadata_section( + &mut self, + target: Principal, + name: String, + direct: bool, + ) -> Result>, String> { + let agent = Arc::clone(&self.agent); + let proxy = if direct { None } else { self.proxy }; + + let start = Instant::now(); + let result = tokio::runtime::Handle::current().block_on(async move { + let Some(proxy_cid) = proxy else { + return match agent.read_state_canister_metadata(target, &name).await { + Ok(bytes) => Ok(Some(bytes)), + Err(AgentError::LookupPathAbsent(_)) => Ok(None), + Err(err) => Err(format!("metadata read failed: {err}")), + }; + }; + + let metadata_args = Encode!(&CanisterMetadataArgs { + canister_id: target, + name, + }) + .map_err(|e| format!("metadata encode failed: {e}"))?; + let proxy_args = ProxyArgs { + canister_id: Principal::management_canister(), + method: "canister_metadata".to_string(), + args: metadata_args, + cycles: candid::Nat::from(0u8), + }; + let encoded = Encode!(&proxy_args).map_err(|e| format!("proxy encode failed: {e}"))?; + let raw = agent + .update(&proxy_cid, "proxy") + .with_arg(encoded) + .await + .map_err(|e| format!("proxy call failed: {e}"))?; + let (result,): (ProxyResult,) = + candid::decode_args(&raw).map_err(|e| format!("proxy decode failed: {e}"))?; + match result { + ProxyResult::Ok(ok) => { + let (metadata,): (CanisterMetadataResult,) = candid::decode_args(&ok.result) + .map_err(|e| format!("metadata decode failed: {e}"))?; + Ok(Some(metadata.value)) + } + ProxyResult::Err(err) => { + let message = err.format_error(); + if message.contains(NO_SUCH_SECTION_REJECT) { + Ok(None) + } else { + Err(message) + } + } + } + }); + self.refund_host_call_time(start); + result + } + + /// Return the wall-clock time a host call spent off-wasm to the compute + /// budget, so network latency doesn't count against the plugin's limit. + fn refund_host_call_time(&self, start: Instant) { let elapsed_ticks = start.elapsed().as_secs() + 1; self.epoch_extension .fetch_add(elapsed_ticks, Ordering::Relaxed); - result } } @@ -230,6 +314,14 @@ impl v2::SyncPluginImports for HostState { req.cycles, ) } + + fn get_metadata_section( + &mut self, + req: v2::icp::sync_plugin::types::MetadataSectionRequest, + ) -> Result>, String> { + let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; + self.do_get_metadata_section(target, req.name, req.direct) + } } // -- v0.1.0 interface: calls always go to the canister being synced. ----------- @@ -448,7 +540,8 @@ pub struct PluginInvocation { pub host_canister_id: Principal, /// Agent used for canister calls. pub agent: Agent, - /// Proxy canister to route update calls through, if configured. + /// Proxy canister to route update calls and metadata reads through, if + /// configured. pub proxy: Option, /// Signing identity principal, surfaced to the plugin. pub identity_principal: Principal, @@ -1292,6 +1385,25 @@ mod tests { )); } + /// A metadata read names its target the same way a call does, and the host + /// enforces the `canisters` list before going to the network — so an + /// undeclared target is refused without a live canister to read from. + #[test] + fn metadata_read_of_undeclared_canister_is_rejected() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let lines = run_plugin(invocation(wasm_path, "metadata-undeclared")) + .expect("plugin should succeed"); + let [refusal] = &lines[..] else { + panic!("expected one refusal line, got: {lines:?}"); + }; + assert!( + refusal.contains("not permitted") && refusal.contains("undeclared"), + "got: {refusal}" + ); + } + #[test] fn plugin_exceeding_compute_limit_is_trapped() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index ef6fa7587..3ef62decc 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -136,11 +136,32 @@ interface types { /// for query calls. cycles: u64, } + + /// A request to read a canister's metadata section. + record metadata-section-request { + /// Which canister to read from. The same rule as + /// `canister-call-request.target` applies: `host` is always permitted, + /// a `name` must appear in the sync step's `canisters` list. + target: call-target, + /// Name of the metadata section, as spelled in the wasm module's custom + /// section minus the `icp:public `/`icp:private ` prefix — e.g. + /// `candid:service`. + name: string, + /// When true, the section is read straight from the target canister + /// with a certified `read_state` request signed by the sync identity, + /// which reaches a private section only if that identity controls the + /// target. When false (the default), the read is routed through the + /// proxy canister configured via `--proxy` — as a call to the + /// management canister's `canister_metadata` method, so a private + /// section gated on the proxy's control is readable. With no proxy + /// configured the read goes directly either way. + direct: bool, + } } /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, call-target, canister-id-entry, dir-input, file-input, field-input}; + use types.{sync-exec-input, canister-call-request, metadata-section-request, call-target, canister-id-entry, dir-input, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin @@ -154,6 +175,19 @@ world sync-plugin { /// message on failure. The plugin is responsible for decoding. import canister-call: func(req: canister-call-request) -> result, string>; + /// Read a metadata section from a canister. + /// The `req.target` selects the canister under the same rule as + /// `canister-call`: the canister being synced (`host`), or one listed in + /// the sync step's `canisters` list, by name. + /// A direct read is a certified `read_state` request signed by the sync + /// identity; a proxied read (`direct` false, with `--proxy` configured) is + /// a call to the management canister's `canister_metadata` method made by + /// the proxy, which reaches sections private to the proxy's control. + /// Returns the section's raw bytes on success, `none` when the target + /// reports it has no section by that name, or an error message on failure. + /// The plugin is responsible for interpreting the bytes. + import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + // The plugin's stdout is captured and shown as transient progress in // the rolling step view of icp-cli; it is discarded when the step ends. // diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index c3c849f83..0655b5348 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -63,6 +63,19 @@ impl Guest for TestPlugin { } Ok(()) } + // Ask for a metadata section from a canister the step did not + // declare. The host must reject the target before it touches the + // network, so this needs no live canister; echo the refusal. + "metadata-undeclared" => { + let err = get_metadata_section(&MetadataSectionRequest { + target: CallTarget::Name("undeclared".to_string()), + name: "candid:service".to_string(), + direct: true, + }) + .expect_err("host must reject an undeclared target"); + eprintln!("{err}"); + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index d37189388..5a56a9271 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -253,11 +253,12 @@ pub struct Adapter { #[schemars(with = "Option>")] pub fields: Option>, - /// Canisters this plugin may call in addition to the canister being synced. - /// Each entry is a canister name resolved against the project's canister ID - /// table for the environment being synced (e.g. `backend`, or a namespaced - /// subproject canister such as `services/open-crm:backend`). The plugin - /// picks a target per call via the `call-target` in its `canister-call` + /// Canisters this plugin may call, or read metadata from, in addition to + /// the canister being synced. Each entry is a canister name resolved against + /// the project's canister ID table for the environment being synced (e.g. + /// `backend`, or a namespaced subproject canister such as + /// `services/open-crm:backend`). The plugin picks a target per request via + /// the `call-target` in its `canister-call` or `get-metadata-section` /// request; a target not listed here is rejected by the host. pub canisters: Option>, } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index e45409d64..cbc4da760 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls, read canister metadata, and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -39,19 +39,23 @@ icp sync │ dirs/files/fields = what you declared in the manifest │ └─ plugin makes canister-call({ target, ... }) (× N) + and get-metadata-section({ target, name }) target = host (the canister being synced), or a canister from `canisters:` by name ``` ## The Plugin Interface -The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides one import (`canister-call`); the plugin provides one export (`exec`): +The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `get-metadata-section`); the plugin provides one export (`exec`): ```wit world sync-plugin { // Host import: call the canister being synced or one listed in `canisters:`. import canister-call: func(req: canister-call-request) -> result, string>; + // Host import: read a metadata section from one of those same canisters. + import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + // Plugin export: run the sync step. export exec: func(input: sync-exec-input) -> result<_, string>; } @@ -93,6 +97,25 @@ The plugin calls methods through the `canister-call` import. It picks a `target` The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. A name is the only way to address another canister — the host owns the name→principal mapping, which differs per environment. +### Reading canister metadata — `get-metadata-section` + +The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canister-metadata) — `candid:service`, for instance — through the `get-metadata-section` import: + +| Request field | Meaning | +|---------------|---------| +| `target` | Which canister to read from: `host`, or a canister declared in `canisters:` addressed by `name` — the same targets, and the same enforcement, as `canister-call` | +| `name` | The section name, without the `icp:public `/`icp:private ` prefix the wasm custom section carries (e.g. `candid:service`) | +| `direct` | When `false` (default), the read is routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, it always goes straight to the target | + +A successful read returns the section's raw bytes, or **absent** when the target reports it has no section by that name — so a plugin can probe for an optional section without matching on error text. Anything else (an unreachable canister, a section the caller may not read) is an error. + +The two routes differ in who the target sees asking, which decides what a **private** section will yield: + +- **Direct** — a certified `read_state` request signed by the sync identity. A private section requires that identity to control the target. +- **Proxied** — a call to the management canister's `canister_metadata` method made by the proxy, because `read_state` is not a canister method and cannot be forwarded. A private section requires the *proxy* to control the target — the same arrangement proxied update calls rely on. + +With no proxy configured, both settings read directly. + ### Logging — stdout and stderr The plugin's stdout and stderr are captured by the host (no logging import is needed — use ordinary `println!` / `eprintln!`): @@ -123,6 +146,7 @@ The plugin runs with a deliberately narrow capability surface. | Clocks, RNG, `wasi:io` | yes | Rust's `HashMap`, `chrono`, etc. work normally | | `process::exit` / panics | yes | abort the guest cleanly; the host surfaces the error | | Canister calls | yes | to the canister being synced, and to canisters declared in `canisters:` | +| Canister metadata reads | yes | the same set of canisters as calls | | Environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | | Network sockets / DNS | blocked | treat the network as unavailable | | Filesystem writes | blocked | no writable preopens | @@ -137,7 +161,7 @@ The plugin runs with a deliberately narrow capability surface. | Linear memory | wasm32 address space (≤ 4 GiB) | | stdout / stderr per stream | 1 MiB | -The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a `canister-call` to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. +The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `get-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. ## Next Steps diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 557c89d8a..84a5f0bcf 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -37,7 +37,7 @@ wit-bindgen = { version = "0.56", features = ["realloc"] } ## Generate Bindings and Implement `exec` -`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the `canister_call` host function. The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. +`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `get_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. ```rust // src/lib.rs @@ -88,6 +88,26 @@ A few things to note: - **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` — the name matches the entries in `input.canister_ids`. Names are the only way to reach another canister; the host resolves them per environment. The host rejects a target you did not declare. - **`direct` and `cycles` control proxy routing.** With `direct: false`, update calls go through the [proxy canister](proxy-canister.md) when one is configured, and `cycles` can fund the forwarded call. With `direct: true`, the call always goes straight to the target. See [The Plugin Interface](../concepts/sync-plugins.md#the-plugin-interface) for the full semantics. +## Read Canister Metadata + +`get_metadata_section` reads a [metadata section](../reference/cli.md#icp-canister-metadata) off a canister — useful for inspecting what is actually deployed before acting on it, e.g. its `candid:service` interface: + +```rust +let interface = get_metadata_section(&MetadataSectionRequest { + target: CallTarget::Host, // same targets, same rules, as canister_call + name: "candid:service".to_string(), + direct: false, // route through the proxy if one is configured +})?; + +match interface { + Some(bytes) => println!("interface: {}", String::from_utf8_lossy(&bytes)), + // `None` means the canister has no such section — not a failure. + None => println!("canister exposes no Candid interface"), +} +``` + +`direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--get-metadata-section) for the full semantics. + ## Read Declared Files and Directories A plugin can't see the filesystem freely — only what you grant it in the manifest's `dirs:` and `files:`. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index ecba7009d..1342516d9 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -154,7 +154,7 @@ sync: fields: # key-value fields passed inline api_url: https://example.com retries: 3 - canisters: # extra canisters the plugin may call + canisters: # extra canisters the plugin may reach - ledger # by name (resolved for the environment) - services/open-crm:backend @@ -172,7 +172,7 @@ sync: | `dirs` | list of paths, or map of name → path(s) | No | Directories (relative to the canister directory, anywhere inside the project) the plugin may read; each is made readable read-only via WASI | | `files` | list of paths, or map of name → path(s) | No | Files (relative to the canister directory, anywhere inside the project) read by the host and passed inline to the plugin | | `fields` | map of string to string | No | Key-value fields passed inline to the plugin; the plugin decides how to interpret them | -| `canisters` | array of string | No | Canisters the plugin may call in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | +| `canisters` | array of string | No | Canisters the plugin may call, or read metadata from, in addition to the one being synced. Each entry is a canister name, resolved against the project's canister IDs for the environment | `dirs:` and `files:` each accept either a plain list of paths or a map. As a map, each key names a single path or a list of paths, and the key is surfaced to the plugin as that entry's `key` (a key mapping to a list produces several entries sharing it). A plain-list entry has no key. For example: @@ -194,7 +194,7 @@ A plugin receives every `fields:` value as a string. Numbers and booleans need n A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a canister defined in a subproject. A name that does not resolve to a known canister for the environment fails the sync step. -The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`) and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. +The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`), read those canisters' metadata sections, and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. ## Recipes diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 5934033cc..739e7d532 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 18f217425..996b9e905 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call in addition to the canister being synced.\nEach entry is a canister name resolved against the project's canister ID\ntable for the environment being synced (e.g. `backend`, or a namespaced\nsubproject canister such as `services/open-crm:backend`). The plugin\npicks a target per call via the `call-target` in its `canister-call`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/examples/icp-sync-plugin/README.md b/examples/icp-sync-plugin/README.md index 57518b0ab..058ec7218 100644 --- a/examples/icp-sync-plugin/README.md +++ b/examples/icp-sync-plugin/README.md @@ -27,13 +27,27 @@ A simple Rust canister with three methods: A Rust Wasm component that implements the `sync-plugin` world defined in `crates/icp-sync-plugin/sync-plugin.wit`. The host runtime calls its `exec` -export and provides a `canister-call` import the plugin uses to reach the -canister. +export and provides the `canister-call` and `get-metadata-section` imports the +plugin uses to reach the canister. ## How the plugin system is exercised This example is designed to demonstrate both routing modes of the -`canister-call` import — the `direct` flag — in a single sync run. +`canister-call` import — the `direct` flag — in a single sync run, plus a +metadata read that follows the same routing. + +### Read — `candid:service` via proxy (`direct: false`) + +Before calling anything, the plugin asks for the canister's `candid:service` +metadata section and reports its size. The build embeds that section with +`ic-wasm`, so it is there; had it not been, the read would return "absent" +rather than fail — a missing section is an answer, not an error. + +Routed through the proxy (`direct: false`), the read reaches the canister as the +management canister's `canister_metadata` method called by the proxy, so it is +the proxy's control over the canister that a private section would be checked +against. A direct read (`direct: true`) is a `read_state` signed by the user +identity instead. ### Call 1 — `set_uploader` via proxy (`direct: false`) @@ -65,6 +79,9 @@ icp sync │ identity-principal = │ proxy-canister-id = │ + ├─ get-metadata-section candid:service direct=false → proxy → mgmt canister + │ reports the section's size, or "absent" + │ ├─ canister-call set_uploader() direct=false → proxy → canister │ canister stores uploader = │ diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index c3fdee356..2e19ce4d0 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -17,7 +17,20 @@ impl Guest for Plugin { input.canister_id, input.environment ); - // 1. Set the uploader to the current identity principal. + // 1. Report the canister's Candid interface, read from its metadata. + // Reported rather than required: the section is only there if the + // build embedded it (this project's build does, via ic-wasm). + let interface = get_metadata_section(&MetadataSectionRequest { + target: CallTarget::Host, + name: "candid:service".to_string(), + direct: false, + })?; + match &interface { + Some(bytes) => eprintln!("candid:service: {} bytes", bytes.len()), + None => eprintln!("candid:service: absent"), + } + + // 2. Set the uploader to the current identity principal. // Routed through the proxy (direct: false) so the controller-gated // call is signed by the proxy canister, which is a controller. let uploader = Principal::from_text(&input.identity_principal) @@ -33,7 +46,7 @@ impl Guest for Plugin { })?; println!("set_uploader ({}): ok", input.identity_principal); - // 2. Register every file found by traversing the preopened dirs. + // 3. Register every file found by traversing the preopened dirs. // Direct calls (direct: true) because register is gated on the // uploader principal, which is the current identity — not the proxy. let mut registered = 0u32; From 1955070240a63c7395357bbb3245b939748fcc0c Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 24 Aug 2026 08:55:25 -0700 Subject: [PATCH 46/51] rename --- crates/icp-sync-plugin/DESIGN.md | 4 ++-- crates/icp-sync-plugin/src/runtime.rs | 8 ++++---- crates/icp-sync-plugin/sync-plugin.wit | 2 +- .../tests/fixtures/test-plugin/src/lib.rs | 2 +- crates/icp/src/manifest/adapter/plugin.rs | 2 +- docs/concepts/sync-plugins.md | 12 ++++++------ docs/guides/writing-sync-plugins.md | 8 ++++---- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- examples/icp-sync-plugin/README.md | 4 ++-- examples/icp-sync-plugin/plugin/src/lib.rs | 2 +- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index e3649cf95..59979f344 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -36,7 +36,7 @@ docs; the *reasons* behind those choices are recorded here. one deployment. (In the earlier `@0.1.0` interface `canister-call` had no target and always reached the canister being synced; see *Interface versioning* below.) -- **`get-metadata-section` mirrors `canister-call`'s targeting and routing** — it +- **`canister-metadata-section` mirrors `canister-call`'s targeting and routing** — it takes the same `call-target` (enforced against `canisters:` the same way) and the same `direct` flag, so one mental model covers both imports. Its return is `result>, string>`: a missing section is an ordinary answer for @@ -179,7 +179,7 @@ straight to the resolved target via `ic-agent`. ### Metadata reads (two routes, one answer) -`get-metadata-section` cannot reuse the call path: `read_state` is not a canister +`canister-metadata-section` cannot reuse the call path: `read_state` is not a canister method, so a proxy canister has nothing to forward. The two routes are therefore different protocols reaching the same data, chosen by the request's `direct` flag exactly as `canister-call` chooses one: diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 35df8f9a9..de4a12fa6 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -99,7 +99,7 @@ pub struct CallableCanisters { /// metadata read for a section the target does not have ("The canister has /// no metadata section with the name ."). A proxied read reaches the /// plugin as reject text, not as a code, so recognizing absence — which -/// [`HostState::do_get_metadata_section`] reports as `Ok(None)`, matching what +/// [`HostState::do_canister_metadata_section`] reports as `Ok(None)`, matching what /// a direct read proves from the certificate — means matching that text. A /// reword upstream turns absence back into an error rather than into a wrong /// answer. @@ -227,7 +227,7 @@ impl HostState { /// management canister's `canister_metadata` on the plugin's behalf, which /// checks the *proxy* against the target's controllers and so reaches /// sections private to it. - fn do_get_metadata_section( + fn do_canister_metadata_section( &mut self, target: Principal, name: String, @@ -315,12 +315,12 @@ impl v2::SyncPluginImports for HostState { ) } - fn get_metadata_section( + fn canister_metadata_section( &mut self, req: v2::icp::sync_plugin::types::MetadataSectionRequest, ) -> Result>, String> { let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; - self.do_get_metadata_section(target, req.name, req.direct) + self.do_canister_metadata_section(target, req.name, req.direct) } } diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 3ef62decc..27066621b 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -186,7 +186,7 @@ world sync-plugin { /// Returns the section's raw bytes on success, `none` when the target /// reports it has no section by that name, or an error message on failure. /// The plugin is responsible for interpreting the bytes. - import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; // The plugin's stdout is captured and shown as transient progress in // the rolling step view of icp-cli; it is discarded when the step ends. diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 0655b5348..116eb0bc8 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -67,7 +67,7 @@ impl Guest for TestPlugin { // declare. The host must reject the target before it touches the // network, so this needs no live canister; echo the refusal. "metadata-undeclared" => { - let err = get_metadata_section(&MetadataSectionRequest { + let err = canister_metadata_section(&MetadataSectionRequest { target: CallTarget::Name("undeclared".to_string()), name: "candid:service".to_string(), direct: true, diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 5a56a9271..a51f99d55 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -258,7 +258,7 @@ pub struct Adapter { /// the project's canister ID table for the environment being synced (e.g. /// `backend`, or a namespaced subproject canister such as /// `services/open-crm:backend`). The plugin picks a target per request via - /// the `call-target` in its `canister-call` or `get-metadata-section` + /// the `call-target` in its `canister-call` or `canister-metadata-section` /// request; a target not listed here is rejected by the host. pub canisters: Option>, } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index cbc4da760..aa15f9cbb 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -39,14 +39,14 @@ icp sync │ dirs/files/fields = what you declared in the manifest │ └─ plugin makes canister-call({ target, ... }) (× N) - and get-metadata-section({ target, name }) + and canister-metadata-section({ target, name }) target = host (the canister being synced), or a canister from `canisters:` by name ``` ## The Plugin Interface -The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `get-metadata-section`); the plugin provides one export (`exec`): +The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `canister-metadata-section`); the plugin provides one export (`exec`): ```wit world sync-plugin { @@ -54,7 +54,7 @@ world sync-plugin { import canister-call: func(req: canister-call-request) -> result, string>; // Host import: read a metadata section from one of those same canisters. - import get-metadata-section: func(req: metadata-section-request) -> result>, string>; + import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; // Plugin export: run the sync step. export exec: func(input: sync-exec-input) -> result<_, string>; @@ -97,9 +97,9 @@ The plugin calls methods through the `canister-call` import. It picks a `target` The `host` target always resolves to `sync-exec-input.canister-id` and is always permitted. A `name` target is permitted only if that canister appears in the sync step's [`canisters:`](../reference/configuration.md#plugin-sync) list; the host rejects any other target without making a call. A name is the only way to address another canister — the host owns the name→principal mapping, which differs per environment. -### Reading canister metadata — `get-metadata-section` +### Reading canister metadata — `canister-metadata-section` -The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canister-metadata) — `candid:service`, for instance — through the `get-metadata-section` import: +The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canister-metadata) — `candid:service`, for instance — through the `canister-metadata-section` import: | Request field | Meaning | |---------------|---------| @@ -161,7 +161,7 @@ The plugin runs with a deliberately narrow capability surface. | Linear memory | wasm32 address space (≤ 4 GiB) | | stdout / stderr per stream | 1 MiB | -The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `get-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. +The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `canister-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. ## Next Steps diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 84a5f0bcf..59d86588c 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -37,7 +37,7 @@ wit-bindgen = { version = "0.56", features = ["realloc"] } ## Generate Bindings and Implement `exec` -`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `get_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. +`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `canister_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. ```rust // src/lib.rs @@ -90,10 +90,10 @@ A few things to note: ## Read Canister Metadata -`get_metadata_section` reads a [metadata section](../reference/cli.md#icp-canister-metadata) off a canister — useful for inspecting what is actually deployed before acting on it, e.g. its `candid:service` interface: +`canister_metadata_section` reads a [metadata section](../reference/cli.md#icp-canister-metadata) off a canister — useful for inspecting what is actually deployed before acting on it, e.g. its `candid:service` interface: ```rust -let interface = get_metadata_section(&MetadataSectionRequest { +let interface = canister_metadata_section(&MetadataSectionRequest { target: CallTarget::Host, // same targets, same rules, as canister_call name: "candid:service".to_string(), direct: false, // route through the proxy if one is configured @@ -106,7 +106,7 @@ match interface { } ``` -`direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--get-metadata-section) for the full semantics. +`direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--canister-metadata-section) for the full semantics. ## Read Declared Files and Directories diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 739e7d532..44a1ab210 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `canister-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 996b9e905..4b072af5a 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `get-metadata-section`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `canister-metadata-section`\nrequest; a target not listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/examples/icp-sync-plugin/README.md b/examples/icp-sync-plugin/README.md index 058ec7218..8f2c8ee2e 100644 --- a/examples/icp-sync-plugin/README.md +++ b/examples/icp-sync-plugin/README.md @@ -27,7 +27,7 @@ A simple Rust canister with three methods: A Rust Wasm component that implements the `sync-plugin` world defined in `crates/icp-sync-plugin/sync-plugin.wit`. The host runtime calls its `exec` -export and provides the `canister-call` and `get-metadata-section` imports the +export and provides the `canister-call` and `canister-metadata-section` imports the plugin uses to reach the canister. ## How the plugin system is exercised @@ -79,7 +79,7 @@ icp sync │ identity-principal = │ proxy-canister-id = │ - ├─ get-metadata-section candid:service direct=false → proxy → mgmt canister + ├─ canister-metadata-section candid:service direct=false → proxy → mgmt canister │ reports the section's size, or "absent" │ ├─ canister-call set_uploader() direct=false → proxy → canister diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index 2e19ce4d0..09595a47d 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -20,7 +20,7 @@ impl Guest for Plugin { // 1. Report the canister's Candid interface, read from its metadata. // Reported rather than required: the section is only there if the // build embedded it (this project's build does, via ic-wasm). - let interface = get_metadata_section(&MetadataSectionRequest { + let interface = canister_metadata_section(&MetadataSectionRequest { target: CallTarget::Host, name: "candid:service".to_string(), direct: false, From eb7b15d33a29d31a4b25e16cd0bf85393e37f2e7 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 24 Aug 2026 09:42:53 -0700 Subject: [PATCH 47/51] fixes --- crates/icp-sync-plugin/DESIGN.md | 7 ++-- crates/icp-sync-plugin/src/runtime.rs | 50 ++++++++++++++++++++++++--- examples/icp-sync-plugin/icp.yaml | 2 +- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index 59979f344..e295617b6 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -184,9 +184,10 @@ method, so a proxy canister has nothing to forward. The two routes are therefore different protocols reaching the same data, chosen by the request's `direct` flag exactly as `canister-call` chooses one: -- **Direct** — `Agent::read_state_canister_metadata`, signed by the sync - identity. Absence is *proven* by the certificate, surfacing as - `AgentError::LookupPathAbsent`, which the host maps to `Ok(None)`. +- **Direct** — a `read_state` signed by the sync identity, so absence is + *proven* by the certificate rather than asserted. It requests `controllers` + alongside the metadata path, since only that distinguishes a canister with no + such section from one that was never created. - **Proxied** — `ProxyArgs` aimed at the management canister's `canister_metadata`, so the controller check runs against the proxy. This is the same shape the CLI's own management calls take through diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index de4a12fa6..e48b6f401 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -24,7 +24,8 @@ pub const PLUGIN_COMPUTE_LIMIT_ENV: &str = "ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS"; use bytes::Bytes; use camino::Utf8PathBuf; use candid::{Encode, Principal}; -use ic_agent::{Agent, AgentError}; +use ic_agent::Agent; +use ic_agent::hash_tree::{Label, LookupResult}; use ic_management_canister_types::{CanisterMetadataArgs, CanisterMetadataResult}; use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; use semver::{Version, VersionReq}; @@ -239,10 +240,49 @@ impl HostState { let start = Instant::now(); let result = tokio::runtime::Handle::current().block_on(async move { let Some(proxy_cid) = proxy else { - return match agent.read_state_canister_metadata(target, &name).await { - Ok(bytes) => Ok(Some(bytes)), - Err(AgentError::LookupPathAbsent(_)) => Ok(None), - Err(err) => Err(format!("metadata read failed: {err}")), + // A metadata path proven absent is equally what a canister that + // was never created looks like, and the certificate error names + // only the path asked for, so it cannot tell the two apart. + // Read `controllers` in the same request to disambiguate. + let metadata_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "metadata".into(), + name.as_str().into(), + ]; + let controllers_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "controllers".into(), + ]; + let cert = agent + .read_state_raw( + vec![metadata_path.clone(), controllers_path.clone()], + target, + ) + .await + .map_err(|err| format!("metadata read failed: {err}"))?; + + return match cert.tree.lookup_path(&metadata_path) { + LookupResult::Found(bytes) => Ok(Some(bytes.to_vec())), + // Creation writes `controllers`, so unlike `module_hash` it + // is present for a canister with no module installed — which + // has no sections at all, and so is a genuine `Ok(None)`. + LookupResult::Absent => match cert.tree.lookup_path(&controllers_path) { + LookupResult::Found(_) => Ok(None), + LookupResult::Absent => Err(format!("canister {target} does not exist")), + _ => Err(format!( + "metadata read failed: certificate proves nothing about \ + canister {target}" + )), + }, + // Not proof of absence, just a certificate that says nothing + // about the path — reporting the section missing off this + // would be a guess. + _ => Err(format!( + "metadata read failed: certificate proves nothing about section \ + `{name}` of canister {target}" + )), }; }; diff --git a/examples/icp-sync-plugin/icp.yaml b/examples/icp-sync-plugin/icp.yaml index 134cfd827..73a5e6dba 100644 --- a/examples/icp-sync-plugin/icp.yaml +++ b/examples/icp-sync-plugin/icp.yaml @@ -10,7 +10,7 @@ canisters: - type: script commands: - command -v ic-wasm >/dev/null 2>&1 || { echo >&2 "ic-wasm not found. To install ic-wasm, see https://github.com/dfinity/ic-wasm\n"; exit 1; } - - ic-wasm "$ICP_WASM_OUTPUT_PATH" -o "$ICP_WASM_OUTPUT_PATH" metadata candid:service -f demo.did --keep-name-section + - ic-wasm "$ICP_WASM_OUTPUT_PATH" -o "$ICP_WASM_OUTPUT_PATH" metadata candid:service -f demo.did -v public --keep-name-section sync: steps: From e620a30c70f6b0fceff035e02b76aebe72f3eee3 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 24 Aug 2026 12:51:48 -0700 Subject: [PATCH 48/51] improvements to proxy consistency --- crates/icp-sync-plugin/DESIGN.md | 14 +- crates/icp-sync-plugin/src/runtime.rs | 207 +++++++++++++++++-------- crates/icp-sync-plugin/sync-plugin.wit | 8 +- docs/concepts/sync-plugins.md | 4 +- docs/guides/writing-sync-plugins.md | 3 +- 5 files changed, 162 insertions(+), 74 deletions(-) diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index e295617b6..f80f04c26 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -193,13 +193,13 @@ exactly as `canister-call` chooses one: the same shape the CLI's own management calls take through `update_or_proxy_raw`; the runtime inlines it rather than depending on the CLI. -The routes disagree on how absence arrives: the management canister *rejects* -("The canister `` has no metadata section with the name ``.") and the -proxy hands the plugin a reason string with no reject code attached, so the host -matches `NO_SUCH_SECTION_REJECT` against it to produce the same `Ok(None)` a -direct read proves. Matching replica text is the price of one uniform contract; -it fails in the safe direction — a reword upstream turns absence back into an -error rather than into a wrong answer. +Only a certificate can make a read `none`. The management canister answers a +section that isn't there and one private to someone else with the same +rejection, so the proxied route treats that rejection as a claim to check rather +than an answer, and confirms it with a certified read before reporting absence. +A plugin then sees one answer either way: no section by that name and no module +installed at all are `none`; a private section it may not have, a canister that +does not exist, and any other failure are errors. ### Interface versioning (parallel v0.1.0 / v0.2.0 support) diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index e48b6f401..6e5cba9a4 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -96,15 +96,86 @@ pub struct CallableCanisters { pub by_name: BTreeMap, } -/// The distinguishing phrase in the management canister's rejection of a -/// metadata read for a section the target does not have ("The canister has -/// no metadata section with the name ."). A proxied read reaches the -/// plugin as reject text, not as a code, so recognizing absence — which -/// [`HostState::do_canister_metadata_section`] reports as `Ok(None)`, matching what -/// a direct read proves from the certificate — means matching that text. A -/// reword upstream turns absence back into an error rather than into a wrong -/// answer. -const NO_SUCH_SECTION_REJECT: &str = "no metadata section"; +/// What a certificate says about a metadata section. A section the reader may +/// not have is neither of these: the state tree will not certify it, so it +/// reaches the caller as an error like any other failed read. +enum CertifiedSection { + Present(Vec), + Absent, +} + +/// Ask the target's subnet to certify a metadata section, reporting only what +/// the certificate proves. +/// +/// The section path is requested together with `controllers`, because a +/// metadata path proven absent is equally what a canister that was never created +/// looks like — `controllers` is written at creation, so its presence is what +/// separates the two. A canister with no module installed has no sections at +/// all, which the certificate reports as an absent path under a canister that +/// exists, and so as [`CertifiedSection::Absent`]. +async fn certified_metadata_section( + agent: &Agent, + target: Principal, + name: &str, +) -> Result { + let metadata_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "metadata".into(), + name.into(), + ]; + let controllers_path: Vec>> = vec![ + "canister".into(), + Label::from_bytes(target.as_slice()), + "controllers".into(), + ]; + let cert = agent + .read_state_raw( + vec![metadata_path.clone(), controllers_path.clone()], + target, + ) + .await + .map_err(|err| format!("metadata read failed: {err}"))?; + + match cert.tree.lookup_path(&metadata_path) { + LookupResult::Found(bytes) => Ok(CertifiedSection::Present(bytes.to_vec())), + LookupResult::Absent => match cert.tree.lookup_path(&controllers_path) { + LookupResult::Found(_) => Ok(CertifiedSection::Absent), + LookupResult::Absent => Err(format!("canister {target} does not exist")), + _ => Err(format!( + "metadata read failed: certificate proves nothing about canister {target}" + )), + }, + // Not proof of absence, just a certificate that says nothing about the + // path — reporting the section missing off this would be a guess. + _ => Err(format!( + "metadata read failed: certificate proves nothing about section `{name}` \ + of canister {target}" + )), + } +} + +/// Whether the management canister rejected a metadata read by claiming the +/// target has no such section, rather than because the read itself failed. +/// +/// The claim is not proof: the same rejection covers a section private to +/// someone other than the proxy, so the caller confirms it against a +/// certificate. A proxied read reaches the plugin as reject text with no code +/// attached, so recognizing the claim at all means matching the replica's +/// wording. Both sentences name the canister and one names the section, so the +/// match is anchored on the values this call supplied rather than on a loose +/// phrase that text relayed from elsewhere might happen to contain. A reword +/// upstream turns the claim into an error rather than into a wrong answer. +fn rejected_as_no_such_section(message: &str, target: Principal, name: &str) -> bool { + // A canister with no module installed has no sections at all, so it reports + // absence in its own words. The certificate says the same thing about it: + // the metadata path is absent while the canister itself is there. + message.contains(&format!( + "The canister {target} has no Wasm module and hence no metadata is available." + )) || message.contains(&format!( + "The canister {target} has no metadata section with the name {name}." + )) +} /// Resolve a plugin-supplied [`CallTarget`] to a concrete principal, enforcing /// that the plugin listed it in `canisters`. The canister being synced (`host`) @@ -218,16 +289,19 @@ impl HostState { } /// Read a metadata section from an already-resolved target principal. - /// `Ok(None)` is the target reporting it has no such section, kept distinct - /// from a failed read so a plugin can probe for an optional section without - /// inspecting error text (see [`NO_SUCH_SECTION_REJECT`]). + /// `Ok(None)` means a certificate proved the target has no such section, + /// kept distinct from a failed read so a plugin can probe for an optional + /// section without inspecting error text. A section the reader may not have + /// is a failed read, not an absent one, whichever route asked. /// /// A direct read is a certified `read_state` signed by the sync identity — /// `read_state` is not a canister method, so it cannot be forwarded. A /// proxied read therefore goes the other way around: the proxy calls the /// management canister's `canister_metadata` on the plugin's behalf, which /// checks the *proxy* against the target's controllers and so reaches - /// sections private to it. + /// sections private to it. The management canister does not distinguish + /// absence from privacy, so a proxied read that comes back claiming absence + /// is confirmed against a certificate before it is reported as one. fn do_canister_metadata_section( &mut self, target: Principal, @@ -240,55 +314,17 @@ impl HostState { let start = Instant::now(); let result = tokio::runtime::Handle::current().block_on(async move { let Some(proxy_cid) = proxy else { - // A metadata path proven absent is equally what a canister that - // was never created looks like, and the certificate error names - // only the path asked for, so it cannot tell the two apart. - // Read `controllers` in the same request to disambiguate. - let metadata_path: Vec>> = vec![ - "canister".into(), - Label::from_bytes(target.as_slice()), - "metadata".into(), - name.as_str().into(), - ]; - let controllers_path: Vec>> = vec![ - "canister".into(), - Label::from_bytes(target.as_slice()), - "controllers".into(), - ]; - let cert = agent - .read_state_raw( - vec![metadata_path.clone(), controllers_path.clone()], - target, - ) + return certified_metadata_section(&agent, target, &name) .await - .map_err(|err| format!("metadata read failed: {err}"))?; - - return match cert.tree.lookup_path(&metadata_path) { - LookupResult::Found(bytes) => Ok(Some(bytes.to_vec())), - // Creation writes `controllers`, so unlike `module_hash` it - // is present for a canister with no module installed — which - // has no sections at all, and so is a genuine `Ok(None)`. - LookupResult::Absent => match cert.tree.lookup_path(&controllers_path) { - LookupResult::Found(_) => Ok(None), - LookupResult::Absent => Err(format!("canister {target} does not exist")), - _ => Err(format!( - "metadata read failed: certificate proves nothing about \ - canister {target}" - )), - }, - // Not proof of absence, just a certificate that says nothing - // about the path — reporting the section missing off this - // would be a guess. - _ => Err(format!( - "metadata read failed: certificate proves nothing about section \ - `{name}` of canister {target}" - )), - }; + .map(|section| match section { + CertifiedSection::Present(bytes) => Some(bytes), + CertifiedSection::Absent => None, + }); }; let metadata_args = Encode!(&CanisterMetadataArgs { canister_id: target, - name, + name: name.clone(), }) .map_err(|e| format!("metadata encode failed: {e}"))?; let proxy_args = ProxyArgs { @@ -313,10 +349,19 @@ impl HostState { } ProxyResult::Err(err) => { let message = err.format_error(); - if message.contains(NO_SUCH_SECTION_REJECT) { - Ok(None) - } else { - Err(message) + if !rejected_as_no_such_section(&message, target, &name) { + return Err(format!("metadata read failed: {message}")); + } + // The management canister says the same thing about a + // section that isn't there and one that is private to + // someone else, so its word alone cannot be reported as + // absence. Only a certificate proves the section absent. + match certified_metadata_section(&agent, target, &name).await? { + CertifiedSection::Absent => Ok(None), + CertifiedSection::Present(_) => Err(format!( + "metadata read failed: canister {target} does not let the proxy \ + read section `{name}`" + )), } } } @@ -1444,6 +1489,46 @@ mod tests { ); } + /// The replica's own wording for the two ways a target reports it has no + /// section, copied from `CanisterManagerError` in the IC repo. Both are + /// absence, not failure, so both must reach the plugin as `none`. + #[test] + fn management_canister_absence_rejects_are_recognized() { + let target = Principal::from_text("aaaaa-aa").unwrap(); + let other = Principal::from_text("2vxsx-fae").unwrap(); + + let no_module = format!( + "Proxy call failed: The canister {target} has no Wasm module and hence no metadata is available." + ); + let no_section = format!( + "Proxy call failed: The canister {target} has no metadata section with the name candid:service." + ); + assert!(rejected_as_no_such_section( + &no_module, + target, + "candid:service" + )); + assert!(rejected_as_no_such_section( + &no_section, + target, + "candid:service" + )); + + // A section by another name, a canister other than the one asked about, + // and an unrelated failure are all reads that failed. + assert!(!rejected_as_no_such_section(&no_section, target, "dfx")); + assert!(!rejected_as_no_such_section( + &no_module, + other, + "candid:service" + )); + assert!(!rejected_as_no_such_section( + &format!("Proxy call failed: Canister {target} not found."), + target, + "candid:service" + )); + } + #[test] fn plugin_exceeding_compute_limit_is_trapped() { let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 27066621b..cd44a694f 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -183,9 +183,11 @@ world sync-plugin { /// identity; a proxied read (`direct` false, with `--proxy` configured) is /// a call to the management canister's `canister_metadata` method made by /// the proxy, which reaches sections private to the proxy's control. - /// Returns the section's raw bytes on success, `none` when the target - /// reports it has no section by that name, or an error message on failure. - /// The plugin is responsible for interpreting the bytes. + /// Returns the section's raw bytes on success, or `none` when the target + /// provably has no section by that name — including when it has no module + /// installed at all, and so no sections. A section the reader may not have + /// is an error, as is a canister that does not exist or any other failed + /// read. The plugin is responsible for interpreting the bytes. import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; // The plugin's stdout is captured and shown as transient progress in diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index aa15f9cbb..1c9638417 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -107,9 +107,9 @@ The plugin reads a canister's [metadata sections](../reference/cli.md#icp-canist | `name` | The section name, without the `icp:public `/`icp:private ` prefix the wasm custom section carries (e.g. `candid:service`) | | `direct` | When `false` (default), the read is routed through the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, it always goes straight to the target | -A successful read returns the section's raw bytes, or **absent** when the target reports it has no section by that name — so a plugin can probe for an optional section without matching on error text. Anything else (an unreachable canister, a section the caller may not read) is an error. +A successful read returns the section's raw bytes, or **absent** when the target provably has no section by that name — including when it has no module installed at all — so a plugin can probe for an optional section without matching on error text. Everything else is an error: a section the reader may not have, a canister that does not exist, a read that fails. -The two routes differ in who the target sees asking, which decides what a **private** section will yield: +The two routes differ in who the target sees asking, which decides whether a **private** section reads as its bytes or as an error: - **Direct** — a certified `read_state` request signed by the sync identity. A private section requires that identity to control the target. - **Proxied** — a call to the management canister's `canister_metadata` method made by the proxy, because `read_state` is not a canister method and cannot be forwarded. A private section requires the *proxy* to control the target — the same arrangement proxied update calls rely on. diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index 59d86588c..af117ed91 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -101,7 +101,8 @@ let interface = canister_metadata_section(&MetadataSectionRequest { match interface { Some(bytes) => println!("interface: {}", String::from_utf8_lossy(&bytes)), - // `None` means the canister has no such section — not a failure. + // `None` means the canister provably has no such section — not a failure. + // A section you may not read, or a canister that does not exist, is an error. None => println!("canister exposes no Candid interface"), } ``` From b50baddd5bd485329caa6e66d73fe574f1d5fb59 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Wed, 26 Aug 2026 14:49:55 -0700 Subject: [PATCH 49/51] copilot --- crates/icp-sync-plugin/sync-plugin.wit | 26 ++-- crates/icp/src/canister/sync/plugin.rs | 160 ++++++++++++++++++++-- crates/icp/src/manifest/adapter/plugin.rs | 13 +- docs/concepts/sync-plugins.md | 4 +- docs/guides/writing-sync-plugins.md | 2 +- docs/reference/configuration.md | 2 + docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- 8 files changed, 178 insertions(+), 33 deletions(-) diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index cd44a694f..4af24d363 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -51,11 +51,19 @@ interface types { /// defined directly in the app root has no subproject prefix and appears /// as its bare local name, e.g. "backend". /// - /// Every canister in the same subproject as the canister being synced is - /// additionally listed under its bare local name (a duplicate entry with - /// the same `id`), so a plugin can address a sibling by the same local - /// name the manifest uses. A bare name always means the sibling, so an - /// app-root canister sharing that local name is not listed. + /// Every canister the subproject being synced can name for itself is + /// additionally listed under that name (a duplicate entry with the same + /// `id`): its own canisters under their bare local names, and the + /// canisters of subprojects nested below it under the + /// `subproject:local` key those would have if it were the app root — + /// e.g. for "services/crm:backend", the canister + /// "services/crm/vendor/ledger:ledger" is also listed as + /// "vendor/ledger:ledger". These are the keys the subproject's own + /// manifest uses, so a plugin addresses the same canister by the same + /// name whether the subproject is deployed standalone or vendored into + /// a workspace. Such a name always means what the subproject means by + /// it, so a canister elsewhere in the workspace whose key is spelled + /// the same way is not listed under it. name: string, /// Textual principal the name resolves to for this environment. id: string, @@ -71,10 +79,10 @@ interface types { /// permitted, whether or not it also appears in `canisters`. host, /// A canister from the `canisters` list, identified by name, spelled - /// exactly as it appears in `sync-exec-input.canister-ids` — a bare - /// local name for a canister in the same subproject, or a - /// `subproject:local` key otherwise. The host resolves it against that - /// mapping table. + /// exactly as it appears in `sync-exec-input.canister-ids` — for a + /// canister the synced canister's own subproject names, the name that + /// subproject uses; otherwise the app-root-relative `subproject:local` + /// key. The host resolves it against that mapping table. name(string), } diff --git a/crates/icp/src/canister/sync/plugin.rs b/crates/icp/src/canister/sync/plugin.rs index 43f1bf980..54155db73 100644 --- a/crates/icp/src/canister/sync/plugin.rs +++ b/crates/icp/src/canister/sync/plugin.rs @@ -156,31 +156,60 @@ pub(super) async fn sync( } /// The canister ID table exposed to a sync plugin: every named canister in the -/// project, plus — for canisters in the same subproject as the one being synced -/// — a duplicate entry under the bare local name. A store key is -/// `:` for a canister in a subproject and a bare local name -/// for a canister defined directly in the app root (see the WIT -/// `canister-id-entry` docs), so the syncing canister's namespace is the prefix -/// of its own key. +/// project, plus — for every canister in the subproject the synced canister +/// belongs to, or in a subproject nested below it — a duplicate entry under the +/// name that subproject itself uses. A store key is `:` for a +/// canister in a subproject and a bare local name for a canister defined +/// directly in the app root (see the WIT `canister-id-entry` docs), so the +/// syncing canister's namespace is the prefix of its own key. /// -/// A local name never contains a colon but a subproject directory may, so keys -/// split on their *last* colon. The bare-name aliases take precedence over an -/// app-root canister of the same local name: a plugin resolving a bare name is -/// naming what the syncing canister's own manifest calls it. +/// The aliases exist so a subproject's manifest and plugins keep working when it +/// is vendored into a workspace: both the step's `canisters:` list and the name a +/// plugin passes back as a call target are written where the subproject's own +/// names apply, but store keys are relative to the app root, which moves. See +/// [`member_relative_alias`] for the names produced. +/// +/// The aliases take precedence over a canister elsewhere in the workspace whose +/// store key happens to be spelled the same way: a plugin resolving such a name +/// is naming what its own subproject calls it. fn exposed_canister_ids(params: &Params) -> BTreeMap { - let syncing_namespace = params.name.rsplit_once(':').map(|(namespace, _)| namespace); + // A canister in the app root is in the subproject the store keys are already + // relative to, so its names need no translation. + let Some((syncing_namespace, _)) = params.name.rsplit_once(':') else { + return params.canister_ids.clone(); + }; let mut table = params.canister_ids.clone(); for (key, id) in ¶ms.canister_ids { - if let Some((namespace, local)) = key.rsplit_once(':') - && Some(namespace) == syncing_namespace - { - table.insert(local.to_owned(), *id); + if let Some(alias) = member_relative_alias(syncing_namespace, key) { + table.insert(alias.to_owned(), *id); } } table } +/// The name a canister has *within* the subproject at `namespace`: its store key +/// with that subproject's prefix removed. A canister of the subproject itself +/// comes back under its bare local name; one belonging to a subproject nested +/// below it comes back under the `:` key it would have if that +/// subproject were the app root. `None` for a canister the subproject has no name +/// of its own for. +/// +/// A local name never contains a colon but a subproject directory may, so a key +/// splits on its *last* colon. +fn member_relative_alias<'a>(namespace: &str, key: &'a str) -> Option<&'a str> { + let (key_namespace, _) = key.rsplit_once(':')?; + let rest = key.strip_prefix(namespace)?; + match key_namespace == namespace { + // The colon separating the subproject from a local name of its own. + true => rest.strip_prefix(':'), + // Otherwise the key's subproject must sit *below* this one. Demanding + // the path separator is what keeps `services/crm-legacy:backend` out of + // `services/crm`, which it merely shares a spelling prefix with. + false => rest.strip_prefix('/'), + } +} + /// Resolve the step's `canisters` list into a [`CallableCanisters`] enforcement /// set. Each listed name is looked up in `canister_ids`; a name that does not /// resolve is a manifest error. @@ -291,6 +320,84 @@ mod tests { assert_eq!(table.get("backend"), Some(&backend)); } + /// A canister of a subproject nested below the syncing canister's own is + /// exposed under the key that subproject has relative to it — the very key + /// its manifest and plugins use when it is built standalone, so vendoring it + /// into a workspace leaves both spellings working. + #[test] + fn exposed_ids_add_member_relative_names_for_nested_subprojects() { + let ledger = principal(1); + let deep = principal(2); + let params = params_named( + "services/crm:backend", + &[ + ("services/crm:backend", principal(9)), + ("services/crm/vendor/ledger:ledger", ledger), + ("services/crm/vendor/ledger/vendor/util:util", deep), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("vendor/ledger:ledger"), Some(&ledger)); + // Nesting is not limited to one level: the whole subtree below the + // syncing canister's subproject is renamed relative to it. + assert_eq!(table.get("vendor/ledger/vendor/util:util"), Some(&deep)); + // The workspace-absolute keys remain. + assert_eq!( + table.get("services/crm/vendor/ledger:ledger"), + Some(&ledger) + ); + } + + /// A subproject whose path merely starts with the same characters is not + /// nested below the syncing canister's, so it contributes no alias. + #[test] + fn exposed_ids_ignore_a_subproject_sharing_a_spelling_prefix() { + let legacy = principal(1); + let params = params_named( + "services/crm:backend", + &[ + ("services/crm:backend", principal(9)), + ("services/crm-legacy:backend", legacy), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("services/crm-legacy:backend"), Some(&legacy)); + // Its local name belongs to the syncing canister, not to it. + assert_eq!(table.get("backend"), Some(&principal(9))); + assert_eq!(table.get("-legacy:backend"), None); + } + + /// A member-relative alias wins over a workspace canister whose store key is + /// spelled the same way, for the same reason a bare sibling name does: the + /// name is being read where the subproject's own names apply. + #[test] + fn exposed_ids_member_relative_alias_overrides_a_root_dependency_key() { + let root_ledger = principal(1); + let own_ledger = principal(2); + let params = params_named( + "services/crm:backend", + &[ + ("services/crm:backend", principal(9)), + ("services/crm/vendor/ledger:ledger", own_ledger), + ("vendor/ledger:ledger", root_ledger), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("vendor/ledger:ledger"), Some(&own_ledger)); + // The root's own dependency is still reachable, by its store key. + assert_eq!( + table.get("services/crm/vendor/ledger:ledger"), + Some(&own_ledger) + ); + assert!(!table.values().any(|id| *id == root_ledger)); + } + /// An app-root canister sharing a local name with a sibling of the syncing /// canister does not keep the bare name: the syncing subproject's own /// canister is what that name means to the plugin. @@ -321,11 +428,13 @@ mod tests { fn exposed_ids_split_subproject_prefix_at_the_last_colon() { let backend = principal(1); let frontend = principal(2); + let nested = principal(3); let params = params_named( "services/odd:name:backend", &[ ("services/odd:name:backend", backend), ("services/odd:name:frontend", frontend), + ("services/odd:name/vendor/ledger:ledger", nested), ], ); @@ -333,6 +442,27 @@ mod tests { assert_eq!(table.get("backend"), Some(&backend)); assert_eq!(table.get("frontend"), Some(&frontend)); + assert_eq!(table.get("vendor/ledger:ledger"), Some(&nested)); + } + + /// The mirror image: a directory whose *name* contains a colon is not a + /// subproject nested below the part before it. `services/odd:name` holds one + /// directory named `odd:name`, so from `services/odd` it is nothing at all. + #[test] + fn exposed_ids_do_not_read_a_colon_in_a_directory_name_as_nesting() { + let odd = principal(1); + let params = params_named( + "services/odd:backend", + &[ + ("services/odd:backend", principal(9)), + ("services/odd:name:frontend", odd), + ], + ); + + let table = exposed_canister_ids(¶ms); + + assert_eq!(table.get("services/odd:name:frontend"), Some(&odd)); + assert_eq!(table.get("name:frontend"), None); } /// A single-project layout keys canisters by bare local name already, so no diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index a51f99d55..579562503 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -255,11 +255,14 @@ pub struct Adapter { /// Canisters this plugin may call, or read metadata from, in addition to /// the canister being synced. Each entry is a canister name resolved against - /// the project's canister ID table for the environment being synced (e.g. - /// `backend`, or a namespaced subproject canister such as - /// `services/open-crm:backend`). The plugin picks a target per request via - /// the `call-target` in its `canister-call` or `canister-metadata-section` - /// request; a target not listed here is rejected by the host. + /// the project's canister ID table for the environment being synced, written + /// as this project spells it: a bare local name for one of its own canisters + /// (e.g. `backend`), or a `:` key for a canister of + /// something it depends on (e.g. `vendor/ledger:ledger`). The same spellings + /// hold when the project is a workspace member, so vendoring it does not + /// change them. The plugin picks a target per request via the `call-target` + /// in its `canister-call` or `canister-metadata-section` request; a target + /// not listed here is rejected by the host. pub canisters: Option>, } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 1c9638417..86646e197 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -78,7 +78,9 @@ The authoritative interface, including all record fields, lives in [`sync-plugin | `proxy-canister-id` | Textual principal of the proxy canister if one was configured via `--proxy`, otherwise absent | | `canister-ids` | The project's canister ID table for this environment — each entry a canister name and the principal it resolves to. Informational; being listed here does not grant permission to call a canister | -Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister defined in a subproject. Canisters in the same subproject as the one being synced are additionally listed under their bare local name, so a plugin can look up a sibling by the name that subproject's manifest uses. A bare name always means the sibling: if an app-root canister has the same local name, it is not listed for that sync. +Each `canister-ids` entry's name is the canister's fully-qualified project key: a bare local name for a canister defined in the app root, or a `subproject:canister` key for a canister defined in a subproject. + +Every canister the subproject being synced can name for itself is additionally listed under that name, so a plugin looks a canister up by the name that subproject's own manifest uses. For a canister in `services/crm`, that means its siblings under their bare local names, and the canisters of its own dependencies under keys relative to it — `services/crm/vendor/ledger:ledger` is also listed as `vendor/ledger:ledger`. Those are the names the subproject uses when it is deployed on its own, so a plugin written against them keeps working once the subproject is vendored into a workspace. Such a name always means what the subproject means by it: a canister elsewhere in the workspace whose key is spelled the same way is not listed under it for that sync. `dirs` and `files` each carry a `key`: the map key the entry was declared under in the manifest, or absent when `dirs:`/`files:` was written as a plain list. A key that maps to a list of paths produces several entries sharing that key, so the key is not unique. Use it to group or label declared paths — e.g. distinguish `seed:` directories from `migrations:` directories — without hardcoding paths in the plugin. diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index af117ed91..fde34ad84 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -85,7 +85,7 @@ export!(Plugin); A few things to note: - **You encode the arguments.** `arg` is raw Candid bytes. Encode with `candid::Encode!`; decode any response (`Vec`) with `candid::Decode!`. -- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` — the name matches the entries in `input.canister_ids`. Names are the only way to reach another canister; the host resolves them per environment. The host rejects a target you did not declare. +- **You choose the target.** `target: CallTarget::Host` reaches the canister being synced. To call another canister, declare it in the manifest's [`canisters:`](../reference/configuration.md#plugin-sync) list and address it with `CallTarget::Name("ledger".into())` — the name matches the entries in `input.canister_ids`. Names are the only way to reach another canister; the host resolves them per environment. The host rejects a target you did not declare. A name is always the one the plugin's own project uses, so hardcoding it stays correct when that project is vendored into a workspace as a subproject. - **`direct` and `cycles` control proxy routing.** With `direct: false`, update calls go through the [proxy canister](proxy-canister.md) when one is configured, and `cycles` can fund the forwarded call. With `direct: true`, the call always goes straight to the target. See [The Plugin Interface](../concepts/sync-plugins.md#the-plugin-interface) for the full semantics. ## Read Canister Metadata diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1342516d9..c8ccf0714 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -194,6 +194,8 @@ A plugin receives every `fields:` value as a string. Numbers and booleans need n A canister name in `canisters:` is the same name you use elsewhere in the project — a bare local name for a sibling canister, or a namespaced `subproject:canister` key for a canister defined in a subproject. A name that does not resolve to a known canister for the environment fails the sync step. +Names are always written from this project's point of view, so they keep working when the project is vendored into a workspace as a subproject: a plugin on a canister in `services/crm` reaches a canister of its own `vendor/ledger` dependency as `vendor/ledger:ledger` either way, even though the workspace keys that canister `services/crm/vendor/ledger:ledger`. Both spellings resolve; if a canister elsewhere in the workspace happens to be keyed `vendor/ledger:ledger`, the name means your own. + The plugin runs in a WASI sandbox: it can call update and query methods on the canister being synced (and any canister listed in `canisters:`), read those canisters' metadata sections, and read the declared `dirs`/`files`, but cannot open network sockets, spawn subprocesses, or write to disk. See [Sync Plugins](../concepts/sync-plugins.md) for the mechanism and [Writing a Sync Plugin](../guides/writing-sync-plugins.md) to author one. ## Recipes diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 44a1ab210..0c550ed58 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `canister-metadata-section`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call` or `canister-metadata-section` request; a target\nnot listed here is rejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 4b072af5a..32fece2fd 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -70,7 +70,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced (e.g.\n`backend`, or a namespaced subproject canister such as\n`services/open-crm:backend`). The plugin picks a target per request via\nthe `call-target` in its `canister-call` or `canister-metadata-section`\nrequest; a target not listed here is rejected by the host.", + "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call` or `canister-metadata-section` request; a target\nnot listed here is rejected by the host.", "items": { "type": "string" }, From 1454886a349b455caa3fc9cd464fed6e300fb175 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Tue, 25 Aug 2026 13:16:59 -0700 Subject: [PATCH 50/51] Allow specifying sync steps in conjunction with recipes --- CHANGELOG.md | 1 + crates/icp-cli/tests/recipe_tests.rs | 64 +++++++++++++ crates/icp/src/manifest/canister.rs | 109 ++++++++++++++++++---- crates/icp/src/project.rs | 120 ++++++++++++++++++++++++- docs/concepts/recipes.md | 7 ++ docs/guides/using-recipes.md | 21 +++++ docs/reference/configuration.md | 22 ++++- docs/schemas/canister-yaml-schema.json | 61 +++++++------ docs/schemas/icp-yaml-schema.json | 61 +++++++------ 9 files changed, 395 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 532761e87..9e5355a27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ air-gapped signing # Unreleased +* feat: a canister that uses a `recipe` can now declare its own `sync` steps, which previously was rejected outright. They run after the sync steps the recipe renders, so a recipe's post-deployment work stays intact and yours is appended to it. `recipe` and `build` remain mutually exclusive. * feat: `script` build steps now receive `ICP_CLI_ENVIRONMENT`, the name of the environment the canisters are being built for, so a build can vary by environment the way a sync step already could. * feat: `icp completions ` prints a shell completion script for `bash`, `zsh`, `fish`, `powershell`, or `elvish` to stdout. See the [installation guide](docs/guides/installation.md#shell-completions) for where to put it. * feat(sync-plugin): `dirs:` and `files:` on a `plugin` sync step may now name anything inside the project, not just paths below the canister's own directory. Entries are still written relative to the canister directory, but may rise out of it — `dirs: ["../shared/assets"]` — so several canisters can be handed the same tree without duplicating it. The project directory is the boundary: an entry that resolves above it is rejected before the plugin runs, as is an absolute one, and an entry that is (or traverses) a symlink is still rejected outright. The plugin sees each directory at the path the manifest wrote, `..` and all. diff --git a/crates/icp-cli/tests/recipe_tests.rs b/crates/icp-cli/tests/recipe_tests.rs index 85a889b2c..e10554f00 100644 --- a/crates/icp-cli/tests/recipe_tests.rs +++ b/crates/icp-cli/tests/recipe_tests.rs @@ -332,3 +332,67 @@ fn recipe_local_file_valid_checksum() { .assert() .success(); } + +/// A canister may declare sync steps alongside a recipe; they land after the +/// steps the recipe renders. +#[test] +fn recipe_with_manifest_sync_steps() { + let ctx = TestContext::new(); + + // Setup project + let project_dir = ctx.create_project_dir("icp"); + + // Recipe rendering a sync step of its own + write_string( + &project_dir.join("recipe.hbs"), // path + indoc! {r#" + build: + steps: + - type: script + command: echo "test" > "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: script + command: echo from-recipe + "#}, // contents + ) + .expect("failed to write recipe template"); + + let pm = indoc! {" + canisters: + - name: my-canister + recipe: + type: file://./recipe.hbs + sync: + steps: + - type: script + command: echo from-manifest + "}; + + write_string( + &project_dir.join("icp.yaml"), // path + pm, // contents + ) + .expect("failed to write project manifest"); + + // The effective configuration holds both steps, the recipe's first + let assert = ctx + .icp() + .current_dir(project_dir) + .args(["project", "show"]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()) + .expect("`icp project show` output is not UTF-8"); + + let recipe_at = stdout + .find("echo from-recipe") + .unwrap_or_else(|| panic!("recipe's sync step missing from:\n{stdout}")); + let manifest_at = stdout + .find("echo from-manifest") + .unwrap_or_else(|| panic!("manifest's sync step missing from:\n{stdout}")); + assert!( + recipe_at < manifest_at, + "the manifest's sync step should follow the recipe's, got:\n{stdout}" + ); +} diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index 8efab30de..fe19e41d6 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -153,27 +153,22 @@ impl<'de> Deserialize<'de> for CanisterManifest { let has_build = temp_map.contains_key(&build_key); let has_sync = temp_map.contains_key(&sync_key); - match (has_recipe, has_build, has_sync) { - (true, true, _) => { + match (has_recipe, has_build) { + (true, true) => { // Can't have a recipe and a build Err(Error::custom(format!( "Canister {name} cannot have both a `recipe` and a `build` section" ))) } - (true, false, true) => { - // Can't have a recipe and a sync sections - Err(Error::custom(format!( - "Canister {name} cannot have both a `recipe` and a `sync` section" - ))) - } - (false, false, _) => { + (false, false) => { // We must have recipe or build Err(Error::custom(format!( "Canister {name} must have a `recipe` or a `build` section" ))) } - (true, false, false) => { - // We have a a recipe + (true, false) => { + // We have a a recipe, optionally with sync steps of its + // own to run after the recipe's let recipe: Recipe = serde_yaml::from_value( temp_map .remove(&recipe_key) @@ -184,6 +179,23 @@ impl<'de> Deserialize<'de> for CanisterManifest { Error::custom(format!("Canister {name} failed to parse recipe: {}", e)) })?; + let sync: Option = if has_sync { + Some( + serde_yaml::from_value( + temp_map + .remove(&sync_key) + .ok_or_else(|| Error::custom("sync field not found"))?, + ) + .map_err(|e| { + Error::custom(format!( + "Canister {name} failed to parse sync instructions: {e}" + )) + })?, + ) + } else { + None + }; + if !temp_map.is_empty() { return Err(Error::custom(format!( "Unrecognized fields in canister `{name}`." @@ -194,10 +206,10 @@ impl<'de> Deserialize<'de> for CanisterManifest { name, settings, init_args, - instructions: Instructions::Recipe { recipe }, + instructions: Instructions::Recipe { recipe, sync }, }) } - (false, true, _) => { + (false, true) => { // We have a build section // Try to deserialize as BuildSync variant @@ -241,6 +253,10 @@ impl<'de> Deserialize<'de> for CanisterManifest { pub enum Instructions { Recipe { recipe: Recipe, + + /// Additional sync steps, run after the ones the recipe renders. + #[serde(skip_serializing_if = "Option::is_none")] + sync: Option, }, BuildSync { @@ -607,7 +623,8 @@ mod tests { recipe_type: RecipeType::File("my-recipe".to_string()), configuration: HashMap::new(), sha256: None, - } + }, + sync: None, }, }, ); @@ -636,7 +653,8 @@ mod tests { ("key-2".to_string(), "value-2".into()) ]), sha256: None, - } + }, + sync: None, }, }, ); @@ -667,7 +685,8 @@ mod tests { "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" .to_string() ), - } + }, + sync: None, }, }, ); @@ -697,12 +716,68 @@ mod tests { recipe_type: RecipeType::File("my-recipe".to_string()), configuration: HashMap::new(), sha256: None, - } + }, + sync: None, }, }, ); } + #[test] + fn recipe_with_sync() { + assert_eq!( + validate_canister_yaml(indoc! {r#" + name: my-canister + recipe: + type: file://my-recipe + sync: + steps: + - type: script + command: echo hi + "#}), + CanisterManifest { + name: "my-canister".to_string(), + settings: ManifestSettings::default(), + init_args: None, + instructions: Instructions::Recipe { + recipe: Recipe { + recipe_type: RecipeType::File("my-recipe".to_string()), + configuration: HashMap::new(), + sha256: None, + }, + sync: Some(SyncSteps { + steps: vec![SyncStep::Script(script::Adapter { + command: script::CommandField::Command("echo hi".to_string()), + })] + }), + }, + }, + ); + } + + #[test] + fn recipe_with_invalid_sync() { + match serde_yaml::from_str::(indoc! {r#" + name: my-canister + recipe: + type: file://my-recipe + sync: + steps: + - type: nonsense + "#}) + { + Ok(_) => panic!("an unknown sync step type should not deserialize"), + Err(err) => { + let err_msg = format!("{err}"); + if !err_msg.contains("Canister my-canister failed to parse sync instructions") { + panic!( + "expected 'Canister my-canister failed to parse sync instructions' error but got: {err}" + ); + } + } + }; + } + #[test] fn build_steps() { assert_eq!( diff --git a/crates/icp/src/project.rs b/crates/icp/src/project.rs index 1f5520def..a18bf28e5 100644 --- a/crates/icp/src/project.rs +++ b/crates/icp/src/project.rs @@ -376,7 +376,7 @@ async fn build_manifest_canisters( let registry_recipe = match &m.instructions { Instructions::BuildSync { .. } => None, - Instructions::Recipe { recipe } => match &recipe.recipe_type { + Instructions::Recipe { recipe, .. } => match &recipe.recipe_type { RecipeType::Registry { .. } => Some(recipe.recipe_type.to_string()), _ => None, }, @@ -393,7 +393,10 @@ async fn build_manifest_canisters( ), // Recipe - Instructions::Recipe { recipe } => { + Instructions::Recipe { + recipe, + sync: extra_sync, + } => { let fetched = recipe_resolver .resolve(recipe) @@ -422,7 +425,12 @@ async fn build_manifest_canisters( })?; } - steps + // The manifest's own sync steps run after the recipe's. + let (build, mut sync) = steps; + if let Some(extra_sync) = extra_sync { + sync.steps.extend(extra_sync.steps.iter().cloned()); + } + (build, sync) } }; @@ -1477,6 +1485,112 @@ pub async fn consolidate_manifest( }) } +#[cfg(test)] +mod recipe_sync_tests { + use super::*; + use crate::canister::recipe::{Fetched, Resolve, ResolveError}; + use crate::manifest::canister::SyncStep; + use crate::manifest::recipe::Recipe; + use camino_tempfile::Utf8TempDir; + + /// Hands back one fixed template for every recipe, without touching the + /// network or the cache. + struct FixedResolver(&'static str); + + #[async_trait::async_trait] + impl Resolve for FixedResolver { + async fn resolve(&self, _recipe: &Recipe) -> Result { + Ok(Fetched { + template: self.0.to_owned(), + pending_cache: None, + }) + } + } + + const TEMPLATE: &str = indoc::indoc! {r#" + build: + steps: + - type: script + command: build.sh + sync: + steps: + - type: script + command: echo recipe + "#}; + + async fn consolidate(pdir: &Path) -> Result { + let m: ProjectManifest = load_manifest_from_path(&pdir.join(PROJECT_MANIFEST)) + .await + .expect("failed to parse project manifest"); + consolidate_manifest(pdir, &FixedResolver(TEMPLATE), &m).await + } + + /// The commands of a canister's sync steps, which are all script steps here. + fn sync_commands(p: &Project, key: &str) -> Vec { + p.canisters + .get(key) + .expect("canister not found") + .1 + .sync + .steps + .iter() + .map(|s| match s { + SyncStep::Script(adapter) => adapter.command.as_vec().join(" "), + other => panic!("expected a script sync step, got {other:?}"), + }) + .collect() + } + + /// A canister may add sync steps of its own on top of a recipe's; they run + /// after the ones the recipe renders. + #[tokio::test] + async fn manifest_sync_steps_follow_the_recipes() { + let tmp = Utf8TempDir::new().unwrap(); + std::fs::write( + tmp.path().join(PROJECT_MANIFEST), + indoc::indoc! {r#" + canisters: + - name: backend + recipe: + type: file://recipe.hbs + sync: + steps: + - type: script + command: echo manifest + "#}, + ) + .unwrap(); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!( + sync_commands(&p, "backend"), + ["echo recipe", "echo manifest"] + ); + } + + /// Without a `sync` section, a recipe canister still gets exactly the + /// recipe's own sync steps. + #[tokio::test] + async fn recipe_sync_steps_alone_when_manifest_has_none() { + let tmp = Utf8TempDir::new().unwrap(); + std::fs::write( + tmp.path().join(PROJECT_MANIFEST), + indoc::indoc! {r#" + canisters: + - name: backend + recipe: + type: file://recipe.hbs + "#}, + ) + .unwrap(); + + let p = consolidate(tmp.path()).await.unwrap(); + + assert_eq!(sync_commands(&p, "backend"), ["echo recipe"]); + } +} + #[cfg(test)] mod dependency_tests { use super::*; diff --git a/docs/concepts/recipes.md b/docs/concepts/recipes.md index b8b31b89d..494db939a 100644 --- a/docs/concepts/recipes.md +++ b/docs/concepts/recipes.md @@ -39,6 +39,13 @@ canisters: - cp target/wasm32-unknown-unknown/release/my_backend.wasm "$ICP_WASM_OUTPUT_PATH" ``` +### Extra Sync Steps + +A recipe defines the canister's build, so `recipe` and `build` are mutually +exclusive. `sync` is not: a canister using a recipe may declare its own `sync` +steps, which run after the recipe's. See +[Using Recipes](../guides/using-recipes.md#adding-your-own-sync-steps). + ## Recipe Sources Recipes can come from three sources: diff --git a/docs/guides/using-recipes.md b/docs/guides/using-recipes.md index 9201dcce5..55388bec2 100644 --- a/docs/guides/using-recipes.md +++ b/docs/guides/using-recipes.md @@ -163,6 +163,27 @@ canisters: API_KEY: "secret" ``` +## Adding Your Own Sync Steps + +A recipe canister can also declare a `sync` section of its own, for +post-deployment work the recipe does not cover. Those steps run after the +recipe's own sync steps: + +```yaml +canisters: + - name: backend + recipe: + type: "@dfinity/rust@v3.0.0" + configuration: + package: backend + sync: + steps: + - type: script + command: ./scripts/seed-data.sh +``` + +`build` remains exclusive with `recipe`: the recipe is what defines the build. + ## Next Steps - [Recipes](../concepts/recipes.md) — Understand how recipes work diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index c8ccf0714..d9668055b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -60,7 +60,7 @@ canisters: | `sync` | object | No | Post-deployment sync configuration | | `settings` | object | No | Canister settings | | `init_args` | string or object | No | Initialization arguments (see [Init Args](#init-args)) | -| `recipe` | object | No | Recipe reference (alternative to build) | +| `recipe` | object | No | Recipe reference (alternative to build; may be combined with `sync`) | ## Build Steps @@ -218,6 +218,26 @@ canisters: | `sha256` | string | Conditional | Required for remote URLs | | `configuration` | object | No | Parameters passed to recipe template | +### Adding Sync Steps to a Recipe + +A canister that uses a recipe may declare a `sync` section of its own. Its steps +run after the ones the recipe renders, in the order written: + +```yaml +canisters: + - name: frontend + recipe: + type: "@dfinity/asset-canister@v2.2.1" + configuration: + dir: dist + sync: + steps: + - type: script + command: ./scripts/warm-cache.sh +``` + +A `recipe` still cannot be combined with `build` — the recipe defines the build. + ### Recipe Type Formats ```yaml diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 0c550ed58..d37d52a98 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -34,29 +34,6 @@ "type": "object" }, "Adapter2": { - "anyOf": [ - { - "$ref": "#/$defs/LocalSource", - "description": "Local path on-disk to read a WASM file from" - }, - { - "$ref": "#/$defs/RemoteSource", - "description": "Remote url to fetch a WASM file from" - } - ], - "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", - "properties": { - "sha256": { - "description": "Optional sha256 checksum of the WASM", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "Adapter3": { "anyOf": [ { "$ref": "#/$defs/LocalSource", @@ -125,6 +102,29 @@ }, "type": "object" }, + "Adapter3": { + "anyOf": [ + { + "$ref": "#/$defs/LocalSource", + "description": "Local path on-disk to read a WASM file from" + }, + { + "$ref": "#/$defs/RemoteSource", + "description": "Remote url to fetch a WASM file from" + } + ], + "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", + "properties": { + "sha256": { + "description": "Optional sha256 checksum of the WASM", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "ArgsFormat": { "description": "Format specifier for canister call/install args content.", "oneOf": [ @@ -163,7 +163,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter2", + "$ref": "#/$defs/Adapter3", "description": "Represents a pre-built canister.\nThis variant allows for retrieving a canister WASM from various sources.", "properties": { "type": { @@ -542,7 +542,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter3", + "$ref": "#/$defs/Adapter2", "description": "Represents a sync step executed by a WebAssembly plugin running inside\na wasmtime WASI sandbox. The plugin can call canister methods on exactly\nthe canister being synced and read files from the declared `dirs`.", "properties": { "type": { @@ -580,6 +580,17 @@ "properties": { "recipe": { "$ref": "#/$defs/Recipe" + }, + "sync": { + "anyOf": [ + { + "$ref": "#/$defs/SyncSteps" + }, + { + "type": "null" + } + ], + "description": "Additional sync steps, run after the ones the recipe renders." } }, "required": [ diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 32fece2fd..ce17ed659 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -34,29 +34,6 @@ "type": "object" }, "Adapter2": { - "anyOf": [ - { - "$ref": "#/$defs/LocalSource", - "description": "Local path on-disk to read a WASM file from" - }, - { - "$ref": "#/$defs/RemoteSource", - "description": "Remote url to fetch a WASM file from" - } - ], - "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", - "properties": { - "sha256": { - "description": "Optional sha256 checksum of the WASM", - "type": [ - "string", - "null" - ] - } - }, - "type": "object" - }, - "Adapter3": { "anyOf": [ { "$ref": "#/$defs/LocalSource", @@ -125,6 +102,29 @@ }, "type": "object" }, + "Adapter3": { + "anyOf": [ + { + "$ref": "#/$defs/LocalSource", + "description": "Local path on-disk to read a WASM file from" + }, + { + "$ref": "#/$defs/RemoteSource", + "description": "Remote url to fetch a WASM file from" + } + ], + "description": "Configuration for a wasm source — used by adapters that load a `.wasm` file\neither from a local path or from a remote URL.", + "properties": { + "sha256": { + "description": "Optional sha256 checksum of the WASM", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "ArgsFormat": { "description": "Format specifier for canister call/install args content.", "oneOf": [ @@ -163,7 +163,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter2", + "$ref": "#/$defs/Adapter3", "description": "Represents a pre-built canister.\nThis variant allows for retrieving a canister WASM from various sources.", "properties": { "type": { @@ -199,6 +199,17 @@ "properties": { "recipe": { "$ref": "#/$defs/Recipe" + }, + "sync": { + "anyOf": [ + { + "$ref": "#/$defs/SyncSteps" + }, + { + "type": "null" + } + ], + "description": "Additional sync steps, run after the ones the recipe renders." } }, "required": [ @@ -1065,7 +1076,7 @@ "type": "object" }, { - "$ref": "#/$defs/Adapter3", + "$ref": "#/$defs/Adapter2", "description": "Represents a sync step executed by a WebAssembly plugin running inside\na wasmtime WASI sandbox. The plugin can call canister methods on exactly\nthe canister being synced and read files from the declared `dirs`.", "properties": { "type": { From f5fec43462a7ee6431f5d360e334437c15537491 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Mon, 31 Aug 2026 12:14:10 -0700 Subject: [PATCH 51/51] add sync plugin setenv --- crates/icp-cli/tests/sync_tests.rs | 31 +++- crates/icp-sync-plugin/DESIGN.md | 39 +++- crates/icp-sync-plugin/src/runtime.rs | 170 +++++++++++++++--- crates/icp-sync-plugin/sync-plugin.wit | 30 +++- .../tests/fixtures/test-plugin/src/lib.rs | 14 ++ crates/icp/src/manifest/adapter/plugin.rs | 10 +- docs/concepts/sync-plugins.md | 37 +++- docs/guides/writing-sync-plugins.md | 19 +- docs/schemas/canister-yaml-schema.json | 2 +- docs/schemas/icp-yaml-schema.json | 2 +- examples/icp-sync-plugin/README.md | 24 ++- examples/icp-sync-plugin/plugin/src/lib.rs | 14 +- 12 files changed, 347 insertions(+), 45 deletions(-) diff --git a/crates/icp-cli/tests/sync_tests.rs b/crates/icp-cli/tests/sync_tests.rs index 14afac03d..13545871f 100644 --- a/crates/icp-cli/tests/sync_tests.rs +++ b/crates/icp-cli/tests/sync_tests.rs @@ -448,7 +448,8 @@ async fn sync_plugin_registers_seed_data() { .args(["deploy", "--environment", "random-environment"]) .assert() .success() - .stderr(contains("candid:service: absent")); + .stderr(contains("candid:service: absent")) + .stderr(contains("SEEDED_BY=random-environment")); // Query the canister to verify all three fruits were registered ctx.icp() @@ -470,6 +471,27 @@ async fn sync_plugin_registers_seed_data() { .and(contains("banana")) .and(contains("cherry")), ); + + // The plugin's environment variable really landed in the canister's + // settings, alongside the PUBLIC_CANISTER_ID binding deploy writes itself — + // proving the host read the current list and wrote it back with the new + // variable added, rather than replacing it. + ctx.icp() + .current_dir(&project_dir) + .args([ + "canister", + "settings", + "show", + "my-canister", + "--environment", + "random-environment", + ]) + .assert() + .success() + .stdout( + contains("SEEDED_BY: random-environment") + .and(contains("PUBLIC_CANISTER_ID:my-canister")), + ); } /// `dirs:` may be written as a map (name → path, or name → list of paths) @@ -953,6 +975,10 @@ async fn sync_plugin_routes_through_proxy() { // This manifest skips the example's ic-wasm step, so the section really is // missing — and the host must report the resulting rejection as an absent // section, the same answer a direct read proves from the certificate. + // + // Its environment-variable write is proxied as well: both the settings read + // and the settings write are made by the proxy, which is a controller, so + // the pair is checked against the proxy rather than the user identity. ctx.icp() .current_dir(&project_dir) .args([ @@ -964,7 +990,8 @@ async fn sync_plugin_routes_through_proxy() { ]) .assert() .success() - .stderr(contains("candid:service: absent")); + .stderr(contains("candid:service: absent")) + .stderr(contains("SEEDED_BY=random-environment")); // Query the canister to verify all three fruits were registered ctx.icp() diff --git a/crates/icp-sync-plugin/DESIGN.md b/crates/icp-sync-plugin/DESIGN.md index f80f04c26..30d99d516 100644 --- a/crates/icp-sync-plugin/DESIGN.md +++ b/crates/icp-sync-plugin/DESIGN.md @@ -43,6 +43,13 @@ docs; the *reasons* behind those choices are recorded here. a plugin probing for an optional section, not a failure it must recognize by parsing error text. The host pays for that guarantee on the proxied path — see *Metadata reads* below. +- **`canister-set-environment-variable` sets one variable, not a list** — the + management canister replaces a canister's environment variables wholesale, so + *some* read-modify-write has to happen; putting it in the host means a plugin + that wants to add one variable does not have to first learn the target's other + ones, which reading them itself would tell it. The cost is a round trip per + variable, paid by a plugin setting several. It takes the same `call-target` + and `direct` flag as the other two imports, for the same one mental model. - **`sync-exec-input` carries the canister ID table** — `canister-ids` exposes the project's name→principal map for the environment, so a plugin can resolve canister names it knows about. It is informational only; calling still @@ -188,10 +195,13 @@ exactly as `canister-call` chooses one: *proven* by the certificate rather than asserted. It requests `controllers` alongside the metadata path, since only that distinguishes a canister with no such section from one that was never created. -- **Proxied** — `ProxyArgs` aimed at the management canister's - `canister_metadata`, so the controller check runs against the proxy. This is - the same shape the CLI's own management calls take through - `update_or_proxy_raw`; the runtime inlines it rather than depending on the CLI. +- **Proxied** — `management_call` aimed at the management canister's + `canister_metadata`, so the controller check runs against the proxy. + +`management_call` is the shape the CLI's own management calls take through +`update_or_proxy_raw` — proxied via `ProxyArgs`, or direct with the target as +the effective canister ID — inlined here rather than depended on, and shared +with the environment-variable write below. Only a certificate can make a read `none`. The management canister answers a section that isn't there and one private to someone else with the same @@ -201,6 +211,27 @@ A plugin then sees one answer either way: no section by that name and no module installed at all are `none`; a private section it may not have, a canister that does not exist, and any other failure are errors. +### Environment-variable writes (read-modify-write) + +`update_settings` has no per-variable form: naming `environment_variables` at all +replaces the target's whole list, and omitting a setting is what leaves it +unchanged. So `canister-set-environment-variable` reads the target's current +settings with `canister_status`, overlays the one variable, and writes the list +back with every other field of `CanisterSettings` left `None`. + +Both calls go through `management_call` on the *same* route, chosen by `direct`. +They have to: each is controller-gated against whoever makes it, so a read as +the sync identity followed by a write as the proxy would demand both control the +target and buy nothing for it. The pair is not atomic — a settings update landing +between them is overwritten — which is inherent to the wholesale-replace API and +is documented in the WIT rather than papered over. + +The variable lives in the canister's settings, not in the manifest, so a later +`icp deploy` drops it: `set_binding_env_vars_many` rewrites the list from the +manifest's variables plus the `PUBLIC_CANISTER_ID:*` bindings without reading +what is there. A sync step that sets the variable on every sync restores it, +which is the ordinary case, since deploy runs the sync phase after that pass. + ### Interface versioning (parallel v0.1.0 / v0.2.0 support) A component built with wit-bindgen imports the interface it `use`s as a diff --git a/crates/icp-sync-plugin/src/runtime.rs b/crates/icp-sync-plugin/src/runtime.rs index 6e5cba9a4..f9e69a964 100644 --- a/crates/icp-sync-plugin/src/runtime.rs +++ b/crates/icp-sync-plugin/src/runtime.rs @@ -26,7 +26,10 @@ use camino::Utf8PathBuf; use candid::{Encode, Principal}; use ic_agent::Agent; use ic_agent::hash_tree::{Label, LookupResult}; -use ic_management_canister_types::{CanisterMetadataArgs, CanisterMetadataResult}; +use ic_management_canister_types::{ + CanisterIdRecord, CanisterMetadataArgs, CanisterMetadataResult, CanisterSettings, + CanisterStatusResult, EnvironmentVariable, UpdateSettingsArgs, +}; use icp_canister_interfaces::proxy::{ProxyArgs, ProxyResult}; use semver::{Version, VersionReq}; use snafu::prelude::*; @@ -155,6 +158,51 @@ async fn certified_metadata_section( } } +/// Call a controller-gated management-canister method about `target`, either +/// signed by the sync identity or made by the proxy canister on its behalf. +/// +/// This is the shape the CLI's own management calls take through +/// `update_or_proxy_raw`; the runtime inlines it rather than depending on the +/// CLI. Which caller the target sees is the whole point of the choice: the +/// method checks *it* against the target's controllers, so a plugin reaches a +/// canister the proxy controls but the sync identity does not, or the other way +/// around. +async fn management_call( + agent: &Agent, + proxy: Option, + target: Principal, + method: &str, + arg: Vec, +) -> Result, String> { + let Some(proxy_cid) = proxy else { + return agent + .update(&Principal::management_canister(), method) + .with_arg(arg) + .with_effective_canister_id(target) + .await + .map_err(|e| format!("{method} call failed: {e}")); + }; + + let proxy_args = ProxyArgs { + canister_id: Principal::management_canister(), + method: method.to_string(), + args: arg, + cycles: candid::Nat::from(0u8), + }; + let encoded = Encode!(&proxy_args).map_err(|e| format!("proxy encode failed: {e}"))?; + let raw = agent + .update(&proxy_cid, "proxy") + .with_arg(encoded) + .await + .map_err(|e| format!("proxy call failed: {e}"))?; + let (result,): (ProxyResult,) = + candid::decode_args(&raw).map_err(|e| format!("proxy decode failed: {e}"))?; + match result { + ProxyResult::Ok(ok) => Ok(ok.result), + ProxyResult::Err(err) => Err(err.format_error()), + } +} + /// Whether the management canister rejected a metadata read by claiming the /// target has no such section, rather than because the read itself failed. /// @@ -327,28 +375,21 @@ impl HostState { name: name.clone(), }) .map_err(|e| format!("metadata encode failed: {e}"))?; - let proxy_args = ProxyArgs { - canister_id: Principal::management_canister(), - method: "canister_metadata".to_string(), - args: metadata_args, - cycles: candid::Nat::from(0u8), - }; - let encoded = Encode!(&proxy_args).map_err(|e| format!("proxy encode failed: {e}"))?; - let raw = agent - .update(&proxy_cid, "proxy") - .with_arg(encoded) - .await - .map_err(|e| format!("proxy call failed: {e}"))?; - let (result,): (ProxyResult,) = - candid::decode_args(&raw).map_err(|e| format!("proxy decode failed: {e}"))?; - match result { - ProxyResult::Ok(ok) => { - let (metadata,): (CanisterMetadataResult,) = candid::decode_args(&ok.result) + match management_call( + &agent, + Some(proxy_cid), + target, + "canister_metadata", + metadata_args, + ) + .await + { + Ok(raw) => { + let (metadata,): (CanisterMetadataResult,) = candid::decode_args(&raw) .map_err(|e| format!("metadata decode failed: {e}"))?; Ok(Some(metadata.value)) } - ProxyResult::Err(err) => { - let message = err.format_error(); + Err(message) => { if !rejected_as_no_such_section(&message, target, &name) { return Err(format!("metadata read failed: {message}")); } @@ -370,6 +411,68 @@ impl HostState { result } + /// Set one environment variable on an already-resolved target principal, + /// leaving its other variables and the rest of its settings alone. + /// + /// The management canister has no per-variable update — `update_settings` + /// replaces a canister's environment variables wholesale — so this is a + /// read-modify-write: read the target's current settings, overlay the one + /// variable, write the whole list back. It is not atomic, so a settings + /// update by another party landing between the two calls is overwritten. + /// + /// Both halves take the same route. `direct` picks who the target sees + /// asking, and both `canister_status` and `update_settings` check that + /// caller against its controllers: reading as one principal and writing as + /// another would need both to control the target and would still be the + /// same round trip, so there is nothing to gain by splitting them. + fn do_set_environment_variable( + &mut self, + target: Principal, + name: String, + value: String, + direct: bool, + ) -> Result<(), String> { + let agent = Arc::clone(&self.agent); + let proxy = if direct { None } else { self.proxy }; + + let start = Instant::now(); + let result = tokio::runtime::Handle::current().block_on(async move { + let status_args = Encode!(&CanisterIdRecord { + canister_id: target + }) + .map_err(|e| format!("canister_status encode failed: {e}"))?; + let raw = management_call(&agent, proxy, target, "canister_status", status_args) + .await + .map_err(|e| format!("reading the target's environment variables failed: {e}"))?; + let (status,): (CanisterStatusResult,) = candid::decode_args(&raw) + .map_err(|e| format!("canister_status decode failed: {e}"))?; + + let mut variables = status.settings.environment_variables; + match variables.iter_mut().find(|variable| variable.name == name) { + Some(existing) => existing.value = value, + None => variables.push(EnvironmentVariable { name, value }), + } + + let update_args = Encode!(&UpdateSettingsArgs { + canister_id: target, + settings: CanisterSettings { + environment_variables: Some(variables), + // Every other setting is left `None`, which the management + // canister reads as "leave it as it is". + ..CanisterSettings::default() + }, + sender_canister_version: None, + }) + .map_err(|e| format!("update_settings encode failed: {e}"))?; + management_call(&agent, proxy, target, "update_settings", update_args) + .await + .map_err(|e| format!("setting the environment variable failed: {e}"))?; + Ok(()) + }); + self.refund_host_call_time(start); + result + } + /// Return the wall-clock time a host call spent off-wasm to the compute /// budget, so network latency doesn't count against the plugin's limit. fn refund_host_call_time(&self, start: Instant) { @@ -407,6 +510,14 @@ impl v2::SyncPluginImports for HostState { let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; self.do_canister_metadata_section(target, req.name, req.direct) } + + fn canister_set_environment_variable( + &mut self, + req: v2::icp::sync_plugin::types::SetEnvironmentVariableRequest, + ) -> Result<(), String> { + let target = resolve_call_target(&req.target, self.host_canister_id, &self.callable)?; + self.do_set_environment_variable(target, req.name, req.value, req.direct) + } } // -- v0.1.0 interface: calls always go to the canister being synced. ----------- @@ -1489,6 +1600,25 @@ mod tests { ); } + /// Setting an environment variable names its target the same way a call + /// does, so an undeclared target is refused before the host reads any + /// settings — no live canister needed. + #[test] + fn setting_env_var_on_undeclared_canister_is_rejected() { + let Some(wasm_path) = option_env!("TEST_PLUGIN_WASM") else { + return; + }; + let lines = + run_plugin(invocation(wasm_path, "set-env-undeclared")).expect("plugin should succeed"); + let [refusal] = &lines[..] else { + panic!("expected one refusal line, got: {lines:?}"); + }; + assert!( + refusal.contains("not permitted") && refusal.contains("undeclared"), + "got: {refusal}" + ); + } + /// The replica's own wording for the two ways a target reports it has no /// section, copied from `CanisterManagerError` in the IC repo. Both are /// absence, not failure, so both must reach the plugin as `none`. diff --git a/crates/icp-sync-plugin/sync-plugin.wit b/crates/icp-sync-plugin/sync-plugin.wit index 4af24d363..cb1107d14 100644 --- a/crates/icp-sync-plugin/sync-plugin.wit +++ b/crates/icp-sync-plugin/sync-plugin.wit @@ -165,11 +165,31 @@ interface types { /// configured the read goes directly either way. direct: bool, } + + /// A request to set one of a canister's environment variables. + record set-environment-variable-request { + /// Which canister to set the variable on. The same rule as + /// `canister-call-request.target` applies: `host` is always permitted, + /// a `name` must appear in the sync step's `canisters` list. + target: call-target, + /// Name of the environment variable, spelled as the canister reads it. + name: string, + /// Value to set it to, replacing whatever value the target currently + /// has under this name. + value: string, + /// When true, the update is signed by the sync identity, which must + /// control the target for it to be accepted. When false (the default), + /// it is made by the proxy canister configured via `--proxy`, and it is + /// the proxy that must control the target — the same arrangement + /// proxied update calls rely on. With no proxy configured the update is + /// signed by the sync identity either way. + direct: bool, + } } /// The complete interface of a sync plugin. world sync-plugin { - use types.{sync-exec-input, canister-call-request, metadata-section-request, call-target, canister-id-entry, dir-input, file-input, field-input}; + use types.{sync-exec-input, canister-call-request, metadata-section-request, set-environment-variable-request, call-target, canister-id-entry, dir-input, file-input, field-input}; // ------------------------------------------------------------------------- // Host functions (imports) — provided by icp-cli, called by the plugin @@ -198,6 +218,14 @@ world sync-plugin { /// read. The plugin is responsible for interpreting the bytes. import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; + /// Set an environment variable on a canister, leaving the target's other + /// environment variables — and the rest of its settings — as they are. + /// The `req.target` selects the canister under the same rule as + /// `canister-call`: the canister being synced (`host`), or one listed in + /// the sync step's `canisters` list, by name. + /// Returns an error message on failure. + import canister-set-environment-variable: func(req: set-environment-variable-request) -> result<_, string>; + // The plugin's stdout is captured and shown as transient progress in // the rolling step view of icp-cli; it is discarded when the step ends. // diff --git a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs index 116eb0bc8..b33bd64e8 100644 --- a/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs +++ b/crates/icp-sync-plugin/tests/fixtures/test-plugin/src/lib.rs @@ -76,6 +76,20 @@ impl Guest for TestPlugin { eprintln!("{err}"); Ok(()) } + // Ask to set an environment variable on a canister the step did not + // declare. Rejected on the same rule as a call or a metadata read, + // before any settings are read; echo the refusal. + "set-env-undeclared" => { + let err = canister_set_environment_variable(&SetEnvironmentVariableRequest { + target: CallTarget::Name("undeclared".to_string()), + name: "API_URL".to_string(), + value: "https://example.com".to_string(), + direct: true, + }) + .expect_err("host must reject an undeclared target"); + eprintln!("{err}"); + Ok(()) + } "spin" => { // Busy-loop forever to exercise the host's compute-time limit. // The epoch-interruption check at the loop back-edge traps this, diff --git a/crates/icp/src/manifest/adapter/plugin.rs b/crates/icp/src/manifest/adapter/plugin.rs index 579562503..739d5b964 100644 --- a/crates/icp/src/manifest/adapter/plugin.rs +++ b/crates/icp/src/manifest/adapter/plugin.rs @@ -253,16 +253,18 @@ pub struct Adapter { #[schemars(with = "Option>")] pub fields: Option>, - /// Canisters this plugin may call, or read metadata from, in addition to - /// the canister being synced. Each entry is a canister name resolved against + /// Canisters this plugin may call, read metadata from, or set environment + /// variables on, in addition to the canister being synced. Each entry is a + /// canister name resolved against /// the project's canister ID table for the environment being synced, written /// as this project spells it: a bare local name for one of its own canisters /// (e.g. `backend`), or a `:` key for a canister of /// something it depends on (e.g. `vendor/ledger:ledger`). The same spellings /// hold when the project is a workspace member, so vendoring it does not /// change them. The plugin picks a target per request via the `call-target` - /// in its `canister-call` or `canister-metadata-section` request; a target - /// not listed here is rejected by the host. + /// in its `canister-call`, `canister-metadata-section`, or + /// `canister-set-environment-variable` request; a target not listed here is + /// rejected by the host. pub canisters: Option>, } diff --git a/docs/concepts/sync-plugins.md b/docs/concepts/sync-plugins.md index 86646e197..7317870d8 100644 --- a/docs/concepts/sync-plugins.md +++ b/docs/concepts/sync-plugins.md @@ -3,7 +3,7 @@ title: Sync Plugins description: How sync plugins extend the sync phase with sandboxed WebAssembly components that run arbitrary post-deployment logic against a canister. --- -A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls, read canister metadata, and read declared files — nothing more. By default it can call only the canister being synced; it may call other canisters it lists in the sync step's `canisters:` list. +A **sync plugin** is a WebAssembly component that runs during the [sync phase](build-deploy-sync.md#sync-phase) to perform arbitrary post-deployment work. icp-cli loads the plugin into a sandboxed [wasmtime](https://wasmtime.dev/) WASI runtime, hands it the ID of the canister being synced (plus the project's canister ID table), and lets it make canister calls, read canister metadata, set canister environment variables, and read declared files — nothing more. By default it can reach only the canister being synced; it may reach other canisters it lists in the sync step's `canisters:` list. You declare a sync plugin in your manifest with a `plugin` sync step. For the exact manifest fields, see [Plugin Sync in the Configuration Reference](../reference/configuration.md#plugin-sync). To author your own plugin, see [Writing a Sync Plugin](../guides/writing-sync-plugins.md). @@ -15,7 +15,7 @@ Sync plugins fill that gap. A plugin is: - **Portable** — written in any language that compiles to `wasm32-wasip2`, distributed as one `.wasm` file (local path or remote URL + `sha256`). - **Sandboxed** — it cannot open network sockets, spawn subprocesses, or touch the filesystem outside the directories you explicitly grant it. -- **Scoped by declaration** — it can call update and query methods on the canister being synced, plus any canister listed in the manifest's `canisters:` list. A call to a canister that was not listed is rejected by the host. +- **Scoped by declaration** — it can call update and query methods on, read metadata from, and set environment variables on the canister being synced, plus any canister listed in the manifest's `canisters:` list. A request aimed at a canister that was not listed is rejected by the host. The most common way to get a sync plugin is through a [recipe](recipes.md). For example, the `@dfinity/asset-canister` recipe emits a `plugin` sync step (starting with `v2.2.1`) that uploads your built static files to the asset canister — so for everyday frontend deployment you never write a plugin yourself. @@ -38,15 +38,16 @@ icp sync │ canister-ids = │ dirs/files/fields = what you declared in the manifest │ - └─ plugin makes canister-call({ target, ... }) (× N) - and canister-metadata-section({ target, name }) + └─ plugin makes canister-call({ target, ... }) (× N), + canister-metadata-section({ target, name }), and + canister-set-environment-variable({ target, name, value }) target = host (the canister being synced), or a canister from `canisters:` by name ``` ## The Plugin Interface -The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides two imports (`canister-call` and `canister-metadata-section`); the plugin provides one export (`exec`): +The interface is defined as a [WIT](https://component-model.bytecodealliance.org/design/wit.html) world. The host provides three imports (`canister-call`, `canister-metadata-section`, and `canister-set-environment-variable`); the plugin provides one export (`exec`): ```wit world sync-plugin { @@ -56,6 +57,9 @@ world sync-plugin { // Host import: read a metadata section from one of those same canisters. import canister-metadata-section: func(req: metadata-section-request) -> result>, string>; + // Host import: set an environment variable on one of those same canisters. + import canister-set-environment-variable: func(req: set-environment-variable-request) -> result<_, string>; + // Plugin export: run the sync step. export exec: func(input: sync-exec-input) -> result<_, string>; } @@ -118,6 +122,24 @@ The two routes differ in who the target sees asking, which decides whether a **p With no proxy configured, both settings read directly. +### Setting an environment variable — `canister-set-environment-variable` + +The plugin writes one of a canister's [environment variables](../reference/environment-variables.md#canister-runtime-environment-variables) — configuration the canister's own code reads at runtime — through the `canister-set-environment-variable` import: + +| Request field | Meaning | +|---------------|---------| +| `target` | Which canister to set the variable on: `host`, or a canister declared in `canisters:` addressed by `name` — the same targets, and the same enforcement, as `canister-call` | +| `name` | Name of the environment variable, spelled as the canister reads it | +| `value` | Value to set it to, replacing whatever value the target has under that name | +| `direct` | When `false` (default), the update is made by the [proxy canister](../guides/proxy-canister.md) if one is configured; when `true`, it is signed by the sync identity | + +The target's other environment variables, and the rest of its settings, are left as they are. Setting a variable is **controller-gated**: whoever makes the update — the sync identity or the proxy, per `direct` — must control the target, the same requirement as `icp canister settings update`. + +Two things follow from how the management canister models environment variables, which it can only replace as a whole list: + +- **The update is a read-then-write, not an atomic one.** The host reads the target's current variables and writes them back with yours added. Another party's settings update that lands between the two is overwritten. +- **A later `icp deploy` drops the variable.** Deploy rewrites each canister's variables from the manifest plus the automatic `PUBLIC_CANISTER_ID:*` bindings, without preserving what a plugin added. In the ordinary case this is invisible: deploy runs the sync phase afterwards, so a plugin that sets the variable on every sync sets it again. To have a variable survive independently of the plugin, declare it in the manifest's [`environment_variables`](../reference/canister-settings.md#environment_variables) setting instead. + ### Logging — stdout and stderr The plugin's stdout and stderr are captured by the host (no logging import is needed — use ordinary `println!` / `eprintln!`): @@ -149,7 +171,8 @@ The plugin runs with a deliberately narrow capability surface. | `process::exit` / panics | yes | abort the guest cleanly; the host surfaces the error | | Canister calls | yes | to the canister being synced, and to canisters declared in `canisters:` | | Canister metadata reads | yes | the same set of canisters as calls | -| Environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | +| Canister environment-variable writes | yes | the same set of canisters as calls; the caller must control the target | +| The plugin's own environment variables / args | no | the WASI environment is empty; use `sync-exec-input.environment` | | Network sockets / DNS | blocked | treat the network as unavailable | | Filesystem writes | blocked | no writable preopens | | Spawning subprocesses | blocked | no process interface is linked | @@ -163,7 +186,7 @@ The plugin runs with a deliberately narrow capability surface. | Linear memory | wasm32 address space (≤ 4 GiB) | | stdout / stderr per stream | 1 MiB | -The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `canister-metadata-section`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. +The compute-time budget defaults to 60 seconds and is overridable with the [`ICP_CLI_PLUGIN_COMPUTE_LIMIT_SECS`](../reference/environment-variables.md#icp_cli_plugin_compute_limit_secs) environment variable — raise it for compute-heavy plugins (e.g. compressing a large asset bundle) that legitimately need more time, especially on slower CI runners. The budget counts only wasm instruction execution: time spent waiting for a host call (`canister-call`, `canister-metadata-section`, `canister-set-environment-variable`) to return over the network is **not** charged against it — the host grants that time back when the call completes. A plugin can make as many canister calls as it needs without the network latency eating into its compute limit. ## Next Steps diff --git a/docs/guides/writing-sync-plugins.md b/docs/guides/writing-sync-plugins.md index fde34ad84..0224ae347 100644 --- a/docs/guides/writing-sync-plugins.md +++ b/docs/guides/writing-sync-plugins.md @@ -37,7 +37,7 @@ wit-bindgen = { version = "0.56", features = ["realloc"] } ## Generate Bindings and Implement `exec` -`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `canister_metadata_section`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. +`wit_bindgen::generate!` reads the WIT at build time and produces the `Guest` trait you implement, the input/request types, and the host functions (`canister_call`, `canister_metadata_section`, `canister_set_environment_variable`). The `exec` export is your entry point — it returns `Ok(())` on success or `Err(message)` to fail the sync step. ```rust // src/lib.rs @@ -109,6 +109,23 @@ match interface { `direct` picks who the target sees asking, which is what decides whether a *private* section is readable: a direct read is signed by the sync identity, a proxied one is made by the proxy canister on your behalf. See [Reading canister metadata](../concepts/sync-plugins.md#reading-canister-metadata--canister-metadata-section) for the full semantics. +## Set a Canister Environment Variable + +`canister_set_environment_variable` writes one of a canister's [environment variables](../reference/environment-variables.md#canister-runtime-environment-variables) — configuration the canister reads at runtime — leaving its other variables and settings alone: + +```rust +canister_set_environment_variable(&SetEnvironmentVariableRequest { + target: CallTarget::Host, // same targets, same rules, as canister_call + name: "SEEDED_BY".to_string(), + value: input.environment.clone(), + direct: false, // let the proxy make the update if one is configured +})?; +``` + +The update is controller-gated, so whoever makes it must control the target: with `direct: true` that is the sync identity, with `direct: false` and a proxy configured it is the proxy canister. + +Set the variable on every sync rather than once. The management canister can only replace a canister's variables as a whole list, so the host reads them and writes them back with yours added — and a later `icp deploy` rewrites that list from the manifest plus the automatic `PUBLIC_CANISTER_ID:*` bindings, dropping anything a plugin added. Deploy runs the sync phase afterwards, so a plugin that always sets it always restores it. For a variable that should not depend on the plugin running, declare it in the manifest's [`environment_variables`](../reference/canister-settings.md#environment_variables) setting instead. + ## Read Declared Files and Directories A plugin can't see the filesystem freely — only what you grant it in the manifest's `dirs:` and `files:`. diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index d37d52a98..9bf62818c 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -47,7 +47,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call` or `canister-metadata-section` request; a target\nnot listed here is rejected by the host.", + "description": "Canisters this plugin may call, read metadata from, or set environment\nvariables on, in addition to the canister being synced. Each entry is a\ncanister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call`, `canister-metadata-section`, or\n`canister-set-environment-variable` request; a target not listed here is\nrejected by the host.", "items": { "type": "string" }, diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index ce17ed659..7f160518c 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -47,7 +47,7 @@ "description": "Configuration for a sync plugin step.\n\nA sync plugin is a WebAssembly module invoked during `icp sync` for a\nspecific canister. It runs inside a WASI sandbox whose filesystem access\nis limited to the directories listed in `dirs` (preopened read-only) plus\nthe contents of any files listed in `files` (read by the host and passed\ninline to the plugin). Both are written relative to the canister directory\nand may name anything inside the project, but nothing outside it.\n\nExample (local path):\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # optional for path\n dirs: # directories preopened read-only\n - assets/seed-data\n files: # files read by the host and passed inline\n - config.txt\n fields: # key-value fields passed inline\n api_url: https://example.com\n retries: 3\n```\n\n`dirs` and `files` may instead be written as a map, tagging each entry with a\n`key` surfaced to the plugin; a key may map to a single path or a list:\n```yaml\n- type: plugin\n path: ./plugins/populate-data.wasm\n dirs:\n seed: assets/seed-data # keyed single path\n migrations: # keyed list — entries share the key\n - migrations/2025\n - migrations/2026\n```\n\nExample (remote URL — `sha256` is required):\n```yaml\n- type: plugin\n url: https://example.com/plugins/populate-data.wasm\n sha256: e3b0c44298fc1c149afb... # required for url\n```", "properties": { "canisters": { - "description": "Canisters this plugin may call, or read metadata from, in addition to\nthe canister being synced. Each entry is a canister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call` or `canister-metadata-section` request; a target\nnot listed here is rejected by the host.", + "description": "Canisters this plugin may call, read metadata from, or set environment\nvariables on, in addition to the canister being synced. Each entry is a\ncanister name resolved against\nthe project's canister ID table for the environment being synced, written\nas this project spells it: a bare local name for one of its own canisters\n(e.g. `backend`), or a `:` key for a canister of\nsomething it depends on (e.g. `vendor/ledger:ledger`). The same spellings\nhold when the project is a workspace member, so vendoring it does not\nchange them. The plugin picks a target per request via the `call-target`\nin its `canister-call`, `canister-metadata-section`, or\n`canister-set-environment-variable` request; a target not listed here is\nrejected by the host.", "items": { "type": "string" }, diff --git a/examples/icp-sync-plugin/README.md b/examples/icp-sync-plugin/README.md index 8f2c8ee2e..f8f7b42ae 100644 --- a/examples/icp-sync-plugin/README.md +++ b/examples/icp-sync-plugin/README.md @@ -27,14 +27,15 @@ A simple Rust canister with three methods: A Rust Wasm component that implements the `sync-plugin` world defined in `crates/icp-sync-plugin/sync-plugin.wit`. The host runtime calls its `exec` -export and provides the `canister-call` and `canister-metadata-section` imports the -plugin uses to reach the canister. +export and provides the `canister-call`, `canister-metadata-section`, and +`canister-set-environment-variable` imports the plugin uses to reach the +canister. ## How the plugin system is exercised This example is designed to demonstrate both routing modes of the `canister-call` import — the `direct` flag — in a single sync run, plus a -metadata read that follows the same routing. +metadata read and an environment-variable write that follow the same routing. ### Read — `candid:service` via proxy (`direct: false`) @@ -60,6 +61,19 @@ canister is listed as a controller of the target, the controller guard passes. This models a pattern where privileged, one-time setup calls must come from a known controller — not directly from an end-user identity. +### Write — `SEEDED_BY` environment variable via proxy (`direct: false`) + +The plugin then records which environment seeded the canister as a canister +environment variable, so the canister's own code can read it back at runtime. +Setting settings is controller-gated exactly like `set_uploader`, so it takes +the same route: the proxy makes both the settings read and the settings write, +and it is the proxy's control over the canister that they are checked against. + +The management canister can only replace a canister's environment variables as a +whole list, so the host reads the current ones and writes them back with +`SEEDED_BY` added — the `PUBLIC_CANISTER_ID:*` variables `icp deploy` writes +itself are still there afterwards. + ### Call 2 — `register` directly (`direct: true`) For each file under `seed-data/`, the plugin calls `register` with @@ -85,6 +99,10 @@ icp sync ├─ canister-call set_uploader() direct=false → proxy → canister │ canister stores uploader = │ + ├─ canister-set-environment-variable SEEDED_BY= + │ direct=false → proxy → mgmt canister + │ read current settings, write them back with SEEDED_BY added + │ └─ canister-call register(name, content) direct=true → canister (× N files) canister checks caller == uploader ✓ ``` diff --git a/examples/icp-sync-plugin/plugin/src/lib.rs b/examples/icp-sync-plugin/plugin/src/lib.rs index 09595a47d..723c75ed4 100644 --- a/examples/icp-sync-plugin/plugin/src/lib.rs +++ b/examples/icp-sync-plugin/plugin/src/lib.rs @@ -46,7 +46,19 @@ impl Guest for Plugin { })?; println!("set_uploader ({}): ok", input.identity_principal); - // 3. Register every file found by traversing the preopened dirs. + // 3. Record which environment seeded the canister as an environment + // variable, so the canister's own code can read it back. Setting + // settings is controller-gated like set_uploader, so it takes the + // same route (direct: false). + canister_set_environment_variable(&SetEnvironmentVariableRequest { + target: CallTarget::Host, + name: "SEEDED_BY".to_string(), + value: input.environment.clone(), + direct: false, + })?; + eprintln!("SEEDED_BY={}", input.environment); + + // 4. Register every file found by traversing the preopened dirs. // Direct calls (direct: true) because register is gated on the // uploader principal, which is the current identity — not the proxy. let mut registered = 0u32;