Add fault recovery

.github/workflows/k3s.yaml

@@ -26,6 +26,7 @@ name: k3s
       - services/ws-web-runner/Dockerfile
       - services/ws-test-server/src/bin/verify-math1.rs
       - utilities/cli/src/deployment_types/k3s.rs
+      - utilities/cli/src/deployment_types/k3s_recovery.rs
       # What decides which machine runs what, and the compose stack `compose-verify` starts from it.
       - utilities/cli/src/deployment_types/docker_compose.rs
       - utilities/cli/src/placement.rs

.mise/config.coverage.toml

@@ -3,8 +3,7 @@
 # and the *-all tasks never touch it. Selected only by the coverage workflow, stacked on the language envs it also
 # loads (rust + the guest langs whose ws-modules the coverage build exercises, so the module-loading tests run
 # rather than self-skip). Houses the cargo-llvm-cov task so it stays out of the always-loaded config.toml.
-# cargo-llvm-cov is not a mise [tool] -- the workflow co-installs it via taiki-e/install-action. (cargo-nextest IS
-# a mise tool, from config.toml's `github:nextest-rs/nextest` entry.)
+# The focused recovery task also runs locally, so its prebuilt cargo-llvm-cov is available through mise.
 
 # conda-forge clang is a full LLVM clang with the wasm32 backend that the host clang (Apple's) lacks.
 # The wasm coverage builds need it: minicov's build.rs hardcodes bare `clang` to compile its C profiler runtime
@@ -31,6 +30,10 @@
 "aqua:hhatto/gocloc" = "latest"
 "conda:clang" = { version = "latest", os = ["linux", "macos"] }
 
+[tools."github:taiki-e/cargo-llvm-cov"]
+asset_pattern = "{{ vars.musl_pattern }}"
+version = "0.9.1"
+
 [vars]
 # The libs/ crates each branch-coverage check is answerable for, as directory names under libs/.
 # The split is which pipeline carries the crate's coverage, not a judgement about how well tested it is.
@@ -39,7 +42,10 @@
 # tasks share (wasm-cov for the guests, wasm-agent-cov and pic-viewer-cov for the browser tests) and so can
 # only be asserted where that path runs. A crate belongs to exactly one list: naming it in both would let a
 # gap in one pipeline be answered by the other.
-native_cov_libs = "actix-web-config edge-toolkit et-otlp path test-helpers test-nats test-otlp ws-runner-common"
+native_cov_libs = """
+{{ "actix-web-config edge-toolkit et-otlp path test-helpers " ~
+  "test-nats test-otlp test-recovery ws-runner-common" -}}
+"""
 wasm_cov_libs = "wasi-guest web"
 # The third kind: a crate whose source is `const` and `macro_rules!` only.
 # llvm-cov emits no records at all for one -- there is no executable code to instrument, so its tests can pass
@@ -85,10 +91,7 @@ _.path = "{{ env.HOME }}/.local/share/mise/installs/conda-clang/latest/bin"
 
 [tasks.cargo-llvm-cov]
 description = "Rust coverage (cargo-llvm-cov) + JUnit test results (nextest); emits lcov.info + JUnit reports"
-# cargo-llvm-cov is deliberately NOT a mise [tool] (like cargo-unmaintained).
-# The coverage workflow installs it via taiki-e/install-action; cargo-nextest comes from config.toml's mise tool.
-# This task just runs them. cargo-llvm-cov needs the `llvm-tools-preview` rustup component, which it adds on demand
-# against the mise-managed toolchain.
+# cargo-llvm-cov installs `llvm-tools-preview` on demand for the mise-managed toolchain.
 # `show-env` exports the coverage instrumentation env (rustc wrapper, LLVM_PROFILE_FILE, target dir) so BOTH the
 # nextest run and the mise regen tasks build + run instrumented into one profile set, then a single `report`
 # merges them. The regens -- `gen-specs` (et-int-gen) and `regen-verification` (et-cli) -- exercise the code
@@ -386,6 +389,14 @@ mise run _cov-branch-assert "$report" "{{ vars.wasm_cov_libs }}"
 """
 shell = "{{ vars.task_shell }}"
 
+[tasks.recovery-cov]
+description = "Check recovery service coverage and require complete coverage of its shared test fixtures"
+run = """
+cargo llvm-cov -p et-recovery-supervisor -p et-test-recovery --summary-only --fail-under-lines 90
+cargo llvm-cov report -p et-test-recovery --summary-only --fail-under-lines 100
+"""
+shell = "{{ vars.task_shell }}"
+
 [tasks.pytest-cov]
 description = "Combined Python coverage (pytest + web-runner Pyodide runs) -> Cobertura, plus pytest JUnit results"
 # One Python report is combined from several data sources: the pytest runs plus the web-runner Pyodide runs.
@@ -405,6 +416,12 @@ for module in pyface1 pydemo1 pyeye1 pyspeech1; do
   coreutils cp "$dir/.coverage" "$pycov/.coverage.${module}_pytest"
 done
 pyface=services/ws-modules/pyface1
+sdk=libs/python-edge-toolkit
+junit="$PWD/$pycov/junit-edge-toolkit.xml"
+cov_args="--cov=edge_toolkit --cov-report=term-missing --cov-fail-under=100"
+# shellcheck disable=SC2086
+uv run --directory "$sdk" pytest $cov_args --junitxml="$junit" -o junit_family=legacy
+coreutils cp "$sdk/.coverage" "$pycov/.coverage.edge_toolkit_pytest"
 bridge=services/ws-modules/pyo3-mlflow
 junit="$PWD/$pycov/junit-pyo3-mlflow.xml"
 uv run --project "$pyface" pytest "$bridge" --cov="$bridge/pkg" --cov-report= --junitxml="$junit" -o junit_family=legacy

.mise/config.maint.toml

@@ -618,7 +618,7 @@ docker run --rm --platform linux/amd64 -v "$mnt" et-mise-linux mise run oci-buil
 """
 shell = "{{ vars.task_shell }}"
 
-# Publish the hub and runner images to GHCR as multi-arch manifests (maintainer-only).
+# Publish the hub, runner and recovery images to GHCR as multi-arch manifests (maintainer-only).
 # A generated k3s deployment names these images rather than building them, so the cluster pulls
 # ghcr.io/edge-toolkit/core/et-<service>:latest and only the scenario image -- the one carrying that
 # deployment's module set -- is still built by whoever deploys it. Written ONLY from a maintainer's machine,
@@ -637,7 +637,7 @@ shell = "{{ vars.task_shell }}"
 # needs a `docker-container` builder -- the default `docker` driver builds a single platform and fails the
 # moment two are asked for -- and binfmt registered for the foreign architecture. The foreign leg is emulated,
 # and ws-web-runner links V8, so expect that one to take hours rather than minutes; pass image names to publish
-# a subset rather than rebuilding all four.
+# a subset rather than rebuilding every service.
 #
 # `IMAGE_PLATFORMS` narrows that list for a host that cannot emulate the other architecture. On Apple silicon
 # the emulator decides whether it can at all: under Rosetta the x86 build works, while plain QEMU cannot run
@@ -657,10 +657,10 @@ shell = "{{ vars.task_shell }}"
 # run, so it picks up an architecture a different machine published months earlier. The cost is one extra tag
 # per architecture, which shares every blob with the image it points at.
 [tasks.push-service-images]
-description = "Build and push the hub and runner images to GHCR for linux/amd64 + linux/arm64 (maintainer-only)"
+description = "Build and push service images to GHCR for linux/amd64 + linux/arm64 (maintainer-only)"
 # docker is a host daemon rather than a mise tool, and these images install nothing through mise.
 run = """
-images="${usage_image:-ws-server ws-web-runner ws-wasi-runner ws-pyo3-runner}"
+images="${usage_image:-ws-server ws-web-runner ws-wasi-runner ws-pyo3-runner recovery-supervisor}"
 every_platform="linux/amd64,linux/arm64"
 platforms="${IMAGE_PLATFORMS:-$every_platform}"
 tag=latest
@@ -710,7 +710,7 @@ for name in $images; do
 done
 """
 shell = "{{ vars.task_shell }}"
-usage = 'arg "[image]..." var=#true help="Service directory names to publish (default: the hub and every runner)"'
+usage = 'arg "[image]..." var=#true help="Service directory names to publish (default: all service images)"'
 
 # Verify every GHCR package is public, so unauthenticated users can pull a tool store or a service image.
 # GHCR has NO API to change package visibility -- the packages REST surface is read/delete/restore only,

.mise/config.toml

@@ -3045,6 +3045,15 @@ alias = "cargo-nextest"
 description = "Run every Rust test via nextest (web-runner excluded on gnullvm)"
 run = "cargo nextest run --workspace {{ vars.cargo_ws_excludes }} --config-file config/nextest.toml"
 
+[tasks.test-recovery-supervisor]
+description = "Test recovery policy, Kubernetes observations, persistence, and hub lifecycle transport"
+run = "cargo test -p et-recovery-supervisor"
+tools = { "aqua:kubernetes/kubernetes/kubectl" = "latest" }
+
+[tasks.recovery-supervisor]
+description = "Run the opt-in Core workload recovery supervisor"
+run = "cargo run -p et-recovery-supervisor"
+
 # The self-contained slice of the suite, for platforms still being brought up.
 # These three crates' tests need nothing the host has to provide: no built WASM module, no ws-server, no
 # docker, no network. That is what makes them meaningful on a lane whose job is to prove the toolchain

CHANGELOG.md

@@ -7,6 +7,8 @@ Changelog 1.1.0.
 
 ### Added
 
+- An opt-in `et-recovery-supervisor` service observes application progress and coordinates bounded Kubernetes recovery.
+- The Python SDK provides recovery lifecycle messages and integrity-checked checkpoint storage helpers.
 - A scenario can list `machines:` to span several hosts. The first is the server, running the hub and the collector,
   and each agent runs on the machine that lists it. k3s pins each `Deployment` with a node selector on
   `edge-toolkit/machine`, compose gives each machine a profile, mise adds a `machine-<name>` task per machine, and

Cargo.toml

@@ -14,6 +14,7 @@ members = [
   "libs/test-helpers",
   "libs/test-nats",
   "libs/test-otlp",
+  "libs/test-recovery",
   "libs/wasi-guest",
   "libs/web",
   "libs/ws-runner-common",
@@ -41,6 +42,7 @@ members = [
   "services/hub-backplane",
   "services/modules",
   "services/storage",
+  "services/recovery-supervisor",
   "services/websockify",
   "services/ws",
   "services/ws-pyo3-runner",
@@ -103,11 +105,13 @@ et-onnx = { path = "utilities/onnx", version = "0.2.0" }
 et-org = { path = "libs/org", version = "0.2.0" }
 et-otlp = { path = "libs/et-otlp", version = "0.3.0" }
 et-path = { path = "libs/path", version = "0.3.0" }
+et-recovery-supervisor = { path = "services/recovery-supervisor", version = "0.1.0" }
 et-rest-client = { path = "generated/rust-rest", version = "0.2.0", default-features = false }
 et-storage-service = { path = "services/storage", version = "0.2.0", default-features = false }
 et-test-helpers = { path = "libs/test-helpers", version = "0.2.0" }
 et-test-nats = { path = "libs/test-nats", version = "0.2.0" }
 et-test-otlp = { path = "libs/test-otlp", version = "0.2.0" }
+et-test-recovery = { path = "libs/test-recovery", version = "0.1.0" }
 et-wasi-guest = { path = "libs/wasi-guest", version = "0.2.0" }
 et-web = { path = "libs/web", version = "0.3.0" }
 et-websockify-service = { path = "services/websockify", version = "0.2.0" }

config/coverage.toml

@@ -12,6 +12,7 @@ branch = true
 relative_files = true
 
 [tool.coverage.paths]
+edge_toolkit = ["libs/python-edge-toolkit/edge_toolkit", "edge_toolkit", "**/edge_toolkit"]
 pydata1 = ["services/ws-modules/pydata1/pydata1", "pydata1", "**/pydata1"]
 pydemo1 = ["services/ws-modules/pydemo1/pydemo1", "pydemo1", "**/pydemo1"]
 pyeye1 = ["services/ws-modules/pyeye1/pyeye1", "pyeye1", "**/pyeye1"]

config/lychee.toml

@@ -50,6 +50,8 @@ exclude = [
   '\{|%7B',
   '^file:///app/storage$',
   '^https://github\.com/[^/]+/[^/]+/actions/runs/',
+  # Kubernetes API address; reachable only from inside the cluster.
+  '^https://kubernetes\.default\.svc/?$',
   '^https://openhub\.net/p/edge-toolkit(/|$)',
   '^https?://host[:/]',
 ]

config/nextest.toml

@@ -36,5 +36,7 @@ or binary_id(edge-toolkit::pipx_site_packages)
 or binary_id(et-test-helpers::child_guard) or binary_id(et-test-helpers::path_prefix)
 or binary_id(et-ws-runner-common::fetch_main_field) or binary_id(et-ws-runner-common::module_file_missing)
 or binary_id(et-ws-runner-common::register_frames)
-or binary_id(et-storage-service::s3_backend) or binary_id(et-ws-pyo3-runner::modules)'''
+or binary_id(et-storage-service::s3_backend) or binary_id(et-ws-pyo3-runner::modules)
+or binary_id(et-test-recovery::command) or binary_id(et-recovery-supervisor::runtime)
+or binary_id(et-recovery-supervisor::hub_frames) or binary_id(et-recovery-supervisor::kubernetes)'''
 test-group = 'integration'

config/pyrefly.toml

@@ -55,6 +55,7 @@ replace-imports-with-any = [
   "et_ws",
   "et_ws.*",
   "pyodide_http",
+  "pytest",
   "torch",
   "wit_world",
   "wit_world.*",