diff --git a/.github/workflows/k3s.yaml b/.github/workflows/k3s.yaml index 4c07951b..83208ece 100644 --- a/.github/workflows/k3s.yaml +++ b/.github/workflows/k3s.yaml @@ -26,6 +26,7 @@ name: k3s - services/ws-web-runner/Dockerfile - services/ws-test-server/src/bin/verify-math1.rs - utilities/cli/src/deployment_types/k3s.rs + - utilities/cli/src/deployment_types/k3s_recovery.rs # What decides which machine runs what, and the compose stack `compose-verify` starts from it. - utilities/cli/src/deployment_types/docker_compose.rs - utilities/cli/src/placement.rs diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 54f58ac3..f4650541 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -3,8 +3,7 @@ # and the *-all tasks never touch it. Selected only by the coverage workflow, stacked on the language envs it also # loads (rust + the guest langs whose ws-modules the coverage build exercises, so the module-loading tests run # rather than self-skip). Houses the cargo-llvm-cov task so it stays out of the always-loaded config.toml. -# cargo-llvm-cov is not a mise [tool] -- the workflow co-installs it via taiki-e/install-action. (cargo-nextest IS -# a mise tool, from config.toml's `github:nextest-rs/nextest` entry.) +# The focused recovery task also runs locally, so its prebuilt cargo-llvm-cov is available through mise. # conda-forge clang is a full LLVM clang with the wasm32 backend that the host clang (Apple's) lacks. # The wasm coverage builds need it: minicov's build.rs hardcodes bare `clang` to compile its C profiler runtime @@ -31,6 +30,10 @@ "aqua:hhatto/gocloc" = "latest" "conda:clang" = { version = "latest", os = ["linux", "macos"] } +[tools."github:taiki-e/cargo-llvm-cov"] +asset_pattern = "{{ vars.musl_pattern }}" +version = "0.9.1" + [vars] # The libs/ crates each branch-coverage check is answerable for, as directory names under libs/. # The split is which pipeline carries the crate's coverage, not a judgement about how well tested it is. @@ -39,7 +42,10 @@ # tasks share (wasm-cov for the guests, wasm-agent-cov and pic-viewer-cov for the browser tests) and so can # only be asserted where that path runs. A crate belongs to exactly one list: naming it in both would let a # gap in one pipeline be answered by the other. -native_cov_libs = "actix-web-config edge-toolkit et-otlp path test-helpers test-nats test-otlp ws-runner-common" +native_cov_libs = """ +{{ "actix-web-config edge-toolkit et-otlp path test-helpers " ~ + "test-nats test-otlp test-recovery ws-runner-common" -}} +""" wasm_cov_libs = "wasi-guest web" # The third kind: a crate whose source is `const` and `macro_rules!` only. # llvm-cov emits no records at all for one -- there is no executable code to instrument, so its tests can pass @@ -85,10 +91,7 @@ _.path = "{{ env.HOME }}/.local/share/mise/installs/conda-clang/latest/bin" [tasks.cargo-llvm-cov] description = "Rust coverage (cargo-llvm-cov) + JUnit test results (nextest); emits lcov.info + JUnit reports" -# cargo-llvm-cov is deliberately NOT a mise [tool] (like cargo-unmaintained). -# The coverage workflow installs it via taiki-e/install-action; cargo-nextest comes from config.toml's mise tool. -# This task just runs them. cargo-llvm-cov needs the `llvm-tools-preview` rustup component, which it adds on demand -# against the mise-managed toolchain. +# cargo-llvm-cov installs `llvm-tools-preview` on demand for the mise-managed toolchain. # `show-env` exports the coverage instrumentation env (rustc wrapper, LLVM_PROFILE_FILE, target dir) so BOTH the # nextest run and the mise regen tasks build + run instrumented into one profile set, then a single `report` # merges them. The regens -- `gen-specs` (et-int-gen) and `regen-verification` (et-cli) -- exercise the code @@ -386,6 +389,14 @@ mise run _cov-branch-assert "$report" "{{ vars.wasm_cov_libs }}" """ shell = "{{ vars.task_shell }}" +[tasks.recovery-cov] +description = "Check recovery service coverage and require complete coverage of its shared test fixtures" +run = """ +cargo llvm-cov -p et-recovery-supervisor -p et-test-recovery --summary-only --fail-under-lines 90 +cargo llvm-cov report -p et-test-recovery --summary-only --fail-under-lines 100 +""" +shell = "{{ vars.task_shell }}" + [tasks.pytest-cov] description = "Combined Python coverage (pytest + web-runner Pyodide runs) -> Cobertura, plus pytest JUnit results" # One Python report is combined from several data sources: the pytest runs plus the web-runner Pyodide runs. @@ -405,6 +416,12 @@ for module in pyface1 pydemo1 pyeye1 pyspeech1; do coreutils cp "$dir/.coverage" "$pycov/.coverage.${module}_pytest" done pyface=services/ws-modules/pyface1 +sdk=libs/python-edge-toolkit +junit="$PWD/$pycov/junit-edge-toolkit.xml" +cov_args="--cov=edge_toolkit --cov-report=term-missing --cov-fail-under=100" +# shellcheck disable=SC2086 +uv run --directory "$sdk" pytest $cov_args --junitxml="$junit" -o junit_family=legacy +coreutils cp "$sdk/.coverage" "$pycov/.coverage.edge_toolkit_pytest" bridge=services/ws-modules/pyo3-mlflow junit="$PWD/$pycov/junit-pyo3-mlflow.xml" uv run --project "$pyface" pytest "$bridge" --cov="$bridge/pkg" --cov-report= --junitxml="$junit" -o junit_family=legacy diff --git a/.mise/config.maint.toml b/.mise/config.maint.toml index a96e8a91..2d35d63b 100644 --- a/.mise/config.maint.toml +++ b/.mise/config.maint.toml @@ -618,7 +618,7 @@ docker run --rm --platform linux/amd64 -v "$mnt" et-mise-linux mise run oci-buil """ shell = "{{ vars.task_shell }}" -# Publish the hub and runner images to GHCR as multi-arch manifests (maintainer-only). +# Publish the hub, runner and recovery images to GHCR as multi-arch manifests (maintainer-only). # A generated k3s deployment names these images rather than building them, so the cluster pulls # ghcr.io/edge-toolkit/core/et-:latest and only the scenario image -- the one carrying that # deployment's module set -- is still built by whoever deploys it. Written ONLY from a maintainer's machine, @@ -637,7 +637,7 @@ shell = "{{ vars.task_shell }}" # needs a `docker-container` builder -- the default `docker` driver builds a single platform and fails the # moment two are asked for -- and binfmt registered for the foreign architecture. The foreign leg is emulated, # and ws-web-runner links V8, so expect that one to take hours rather than minutes; pass image names to publish -# a subset rather than rebuilding all four. +# a subset rather than rebuilding every service. # # `IMAGE_PLATFORMS` narrows that list for a host that cannot emulate the other architecture. On Apple silicon # the emulator decides whether it can at all: under Rosetta the x86 build works, while plain QEMU cannot run @@ -657,10 +657,10 @@ shell = "{{ vars.task_shell }}" # run, so it picks up an architecture a different machine published months earlier. The cost is one extra tag # per architecture, which shares every blob with the image it points at. [tasks.push-service-images] -description = "Build and push the hub and runner images to GHCR for linux/amd64 + linux/arm64 (maintainer-only)" +description = "Build and push service images to GHCR for linux/amd64 + linux/arm64 (maintainer-only)" # docker is a host daemon rather than a mise tool, and these images install nothing through mise. run = """ -images="${usage_image:-ws-server ws-web-runner ws-wasi-runner ws-pyo3-runner}" +images="${usage_image:-ws-server ws-web-runner ws-wasi-runner ws-pyo3-runner recovery-supervisor}" every_platform="linux/amd64,linux/arm64" platforms="${IMAGE_PLATFORMS:-$every_platform}" tag=latest @@ -710,7 +710,7 @@ for name in $images; do done """ shell = "{{ vars.task_shell }}" -usage = 'arg "[image]..." var=#true help="Service directory names to publish (default: the hub and every runner)"' +usage = 'arg "[image]..." var=#true help="Service directory names to publish (default: all service images)"' # Verify every GHCR package is public, so unauthenticated users can pull a tool store or a service image. # GHCR has NO API to change package visibility -- the packages REST surface is read/delete/restore only, diff --git a/.mise/config.toml b/.mise/config.toml index 4784b23e..6bd88758 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -3045,6 +3045,15 @@ alias = "cargo-nextest" description = "Run every Rust test via nextest (web-runner excluded on gnullvm)" run = "cargo nextest run --workspace {{ vars.cargo_ws_excludes }} --config-file config/nextest.toml" +[tasks.test-recovery-supervisor] +description = "Test recovery policy, Kubernetes observations, persistence, and hub lifecycle transport" +run = "cargo test -p et-recovery-supervisor" +tools = { "aqua:kubernetes/kubernetes/kubectl" = "latest" } + +[tasks.recovery-supervisor] +description = "Run the opt-in Core workload recovery supervisor" +run = "cargo run -p et-recovery-supervisor" + # The self-contained slice of the suite, for platforms still being brought up. # These three crates' tests need nothing the host has to provide: no built WASM module, no ws-server, no # docker, no network. That is what makes them meaningful on a lane whose job is to prove the toolchain diff --git a/.mise/mise.coverage.lock b/.mise/mise.coverage.lock index ae36e19d..ca0eab5c 100644 --- a/.mise/mise.coverage.lock +++ b/.mise/mise.coverage.lock @@ -739,3 +739,40 @@ conda_deps = [ "libiconv-1.18-hc1393d2_3", "liblzma-5.8.3-hfd05255_1", ] + +[[tools."github:taiki-e/cargo-llvm-cov"]] +version = "0.9.1" +backend = "github:taiki-e/cargo-llvm-cov" + +[tools."github:taiki-e/cargo-llvm-cov".options] +asset_pattern = '*-{{ x86_64_arch }}-*{{ os(linux="musl.t", macos="darwin.t", windows="msvc.z") }}*' + +[tools."github:taiki-e/cargo-llvm-cov"."platforms.linux-arm64"] +checksum = "sha256:49c0085628392a8d727c4464d91ec417c613c5015f5197cd16d5c05aa3fb5ad1" +url = "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.9.1/cargo-llvm-cov-aarch64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/taiki-e/cargo-llvm-cov/releases/assets/547403283" +provenance = "github-attestations" + +[tools."github:taiki-e/cargo-llvm-cov"."platforms.linux-x64"] +checksum = "sha256:3fca950394a3c49457657c158b1619cec8dfd2647ae5b48746734c0ab969a522" +url = "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.9.1/cargo-llvm-cov-x86_64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/taiki-e/cargo-llvm-cov/releases/assets/547403186" +provenance = "github-attestations" + +[tools."github:taiki-e/cargo-llvm-cov"."platforms.macos-arm64"] +checksum = "sha256:7e821a6c90c0884f79f759f5d9be50799ade67bb82515186d922f26032ef43b8" +url = "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.9.1/cargo-llvm-cov-aarch64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/taiki-e/cargo-llvm-cov/releases/assets/547403359" +provenance = "github-attestations" + +[tools."github:taiki-e/cargo-llvm-cov"."platforms.macos-x64"] +checksum = "sha256:ddf949b82a63017441ea76aa068bdd7e2dde8055f47cd3ef736fdbbd633bc33e" +url = "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.9.1/cargo-llvm-cov-x86_64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/taiki-e/cargo-llvm-cov/releases/assets/547403156" +provenance = "github-attestations" + +[tools."github:taiki-e/cargo-llvm-cov"."platforms.windows-x64"] +checksum = "sha256:656590cfa13bb7478595bbfdd3bc6bc075a4f7f3f0df5a439c1856cb03c5a4d2" +url = "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.9.1/cargo-llvm-cov-x86_64-pc-windows-msvc.zip" +url_api = "https://api.github.com/repos/taiki-e/cargo-llvm-cov/releases/assets/547404215" +provenance = "github-attestations" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d064a12..0c201e04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Changelog 1.1.0. ### Added +- An opt-in `et-recovery-supervisor` service observes application progress and coordinates bounded Kubernetes recovery. +- The Python SDK provides recovery lifecycle messages and integrity-checked checkpoint storage helpers. - A scenario can list `machines:` to span several hosts. The first is the server, running the hub and the collector, and each agent runs on the machine that lists it. k3s pins each `Deployment` with a node selector on `edge-toolkit/machine`, compose gives each machine a profile, mise adds a `machine-` task per machine, and diff --git a/Cargo.lock b/Cargo.lock index fffb30aa..7a1fa7bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4460,6 +4460,7 @@ dependencies = [ "edge-toolkit", "et-org", "et-path", + "et-recovery-supervisor", "et-test-helpers", "et-ws-test-server", "fake", @@ -4470,6 +4471,7 @@ dependencies = [ "rand 0.10.2", "rand_chacha 0.10.0", "serde", + "serde-env", "serde_json", "serde_yaml", "sha2 0.11.0", @@ -4511,6 +4513,7 @@ dependencies = [ "et-cli", "et-modules-service", "et-onnx", + "et-recovery-supervisor", "et-storage-service", "et-test-helpers", "et-ws-runner-common", @@ -4612,6 +4615,35 @@ dependencies = [ "tempfile", ] +[[package]] +name = "et-recovery-supervisor" +version = "0.1.0" +dependencies = [ + "document-features", + "edge-toolkit", + "et-otlp", + "et-test-helpers", + "et-test-recovery", + "et-ws-runner-common", + "et-ws-test-server", + "fs-err", + "futures-util", + "k8s-openapi", + "schemars 1.2.2", + "serde", + "serde-env", + "serde_json", + "serde_yaml", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", + "url", + "uuid", +] + [[package]] name = "et-rest-client" version = "0.2.0" @@ -4696,6 +4728,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "et-test-recovery" +version = "0.1.0" +dependencies = [ + "chrono", + "clap", + "edge-toolkit", + "et-recovery-supervisor", + "futures-util", + "serde-env", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "et-wasi-guest" version = "0.2.0" @@ -11700,7 +11748,11 @@ checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71" dependencies = [ "futures-util", "log", + "rustls", + "rustls-native-certs 0.8.4", + "rustls-pki-types", "tokio", + "tokio-rustls", "tungstenite", ] @@ -12090,6 +12142,8 @@ dependencies = [ "httparse", "log", "rand 0.10.2", + "rustls", + "rustls-pki-types", "sha1 0.11.0", "thiserror 2.0.20", ] diff --git a/Cargo.toml b/Cargo.toml index e2d974c1..0bc3c5d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "libs/test-helpers", "libs/test-nats", "libs/test-otlp", + "libs/test-recovery", "libs/wasi-guest", "libs/web", "libs/ws-runner-common", @@ -41,6 +42,7 @@ members = [ "services/hub-backplane", "services/modules", "services/storage", + "services/recovery-supervisor", "services/websockify", "services/ws", "services/ws-pyo3-runner", @@ -103,11 +105,13 @@ et-onnx = { path = "utilities/onnx", version = "0.2.0" } et-org = { path = "libs/org", version = "0.2.0" } et-otlp = { path = "libs/et-otlp", version = "0.3.0" } et-path = { path = "libs/path", version = "0.3.0" } +et-recovery-supervisor = { path = "services/recovery-supervisor", version = "0.1.0" } et-rest-client = { path = "generated/rust-rest", version = "0.2.0", default-features = false } et-storage-service = { path = "services/storage", version = "0.2.0", default-features = false } et-test-helpers = { path = "libs/test-helpers", version = "0.2.0" } et-test-nats = { path = "libs/test-nats", version = "0.2.0" } et-test-otlp = { path = "libs/test-otlp", version = "0.2.0" } +et-test-recovery = { path = "libs/test-recovery", version = "0.1.0" } et-wasi-guest = { path = "libs/wasi-guest", version = "0.2.0" } et-web = { path = "libs/web", version = "0.3.0" } et-websockify-service = { path = "services/websockify", version = "0.2.0" } diff --git a/config/coverage.toml b/config/coverage.toml index dbf33e74..78a088a2 100644 --- a/config/coverage.toml +++ b/config/coverage.toml @@ -12,6 +12,7 @@ branch = true relative_files = true [tool.coverage.paths] +edge_toolkit = ["libs/python-edge-toolkit/edge_toolkit", "edge_toolkit", "**/edge_toolkit"] pydata1 = ["services/ws-modules/pydata1/pydata1", "pydata1", "**/pydata1"] pydemo1 = ["services/ws-modules/pydemo1/pydemo1", "pydemo1", "**/pydemo1"] pyeye1 = ["services/ws-modules/pyeye1/pyeye1", "pyeye1", "**/pyeye1"] diff --git a/config/lychee.toml b/config/lychee.toml index 09976a17..43e92ed7 100644 --- a/config/lychee.toml +++ b/config/lychee.toml @@ -50,6 +50,8 @@ exclude = [ '\{|%7B', '^file:///app/storage$', '^https://github\.com/[^/]+/[^/]+/actions/runs/', + # Kubernetes API address; reachable only from inside the cluster. + '^https://kubernetes\.default\.svc/?$', '^https://openhub\.net/p/edge-toolkit(/|$)', '^https?://host[:/]', ] diff --git a/config/nextest.toml b/config/nextest.toml index 42b486b6..847fb5d3 100644 --- a/config/nextest.toml +++ b/config/nextest.toml @@ -36,5 +36,8 @@ or binary_id(edge-toolkit::pipx_site_packages) or binary_id(et-test-helpers::child_guard) or binary_id(et-test-helpers::path_prefix) or binary_id(et-ws-runner-common::fetch_main_field) or binary_id(et-ws-runner-common::module_file_missing) or binary_id(et-ws-runner-common::register_frames) -or binary_id(et-storage-service::s3_backend) or binary_id(et-ws-pyo3-runner::modules)''' +or binary_id(et-storage-service::s3_backend) or binary_id(et-ws-pyo3-runner::modules) +or binary_id(et-test-recovery::command) or binary_id(et-recovery-supervisor::runtime) +or binary_id(et-recovery-supervisor::hub_frames) or binary_id(et-recovery-supervisor::kubernetes) +or binary_id(et-recovery-supervisor::shutdown)''' test-group = 'integration' diff --git a/config/pyrefly.toml b/config/pyrefly.toml index 1625e780..cdb6b9c3 100644 --- a/config/pyrefly.toml +++ b/config/pyrefly.toml @@ -55,6 +55,7 @@ replace-imports-with-any = [ "et_ws", "et_ws.*", "pyodide_http", + "pytest", "torch", "wit_world", "wit_world.*", diff --git a/libs/python-edge-toolkit/edge_toolkit/recovery.py b/libs/python-edge-toolkit/edge_toolkit/recovery.py new file mode 100644 index 00000000..8f22bdb0 --- /dev/null +++ b/libs/python-edge-toolkit/edge_toolkit/recovery.py @@ -0,0 +1,203 @@ +"""Application lifecycle evidence and integrity-checked checkpoints for Core recovery supervision.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import threading +import time +import uuid +from collections.abc import Callable +from dataclasses import dataclass +from typing import Protocol + +from et_ws.messages import WsSendAgentMessage + +from edge_toolkit.events import client_event + + +class Storage(Protocol): + """Agent-scoped storage operations provided by the native Python runner.""" + + @property + def agent_id(self) -> str | None: + """Return the runner's current write namespace, or None before registration.""" + + def put(self, key: str, data: bytes, /) -> None: + """Persist a whole object in this agent's storage area.""" + + def get(self, agent_id: str, key: str, /) -> bytes | None: + """Retrieve a whole object, or return None when it does not exist.""" + + +class Lifecycle: + """Publish ordered recovery evidence as direct CloudEvents over the existing hub message transport.""" + + def __init__(self, send: Callable[[str], object], supervisor: str, source: str, job_id: str): + """Bind one application process to a supervisor, module source and job identity.""" + if not supervisor or not source.startswith("/modules/") or not job_id: + raise ValueError("supervisor, module source and job identity are required") + self.send = send + self.supervisor = supervisor + self.source = source + self.job_id = job_id + self.instance = str(uuid.uuid4()) + self.sequence = 0 + self.lock = threading.Lock() + + def emit(self, action: str, progress: int, *, bytes_count: int = 0, duration_millis: int = 0) -> None: + """Report lifecycle or checkpoint evidence; progress counts committed useful work, not attempts.""" + if action not in { + "started", + "progress", + "failed", + "completed", + "checkpoint-saved", + "checkpoint-restored", + "resume-completed", + }: + raise ValueError("unsupported recovery lifecycle action") + if isinstance(progress, bool) or not isinstance(progress, int) or not 0 <= progress <= 2**64 - 1: + raise ValueError("progress must be an unsigned 64-bit integer") + for value in (bytes_count, duration_millis): + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 2**64 - 1: + raise ValueError("checkpoint measurements must be unsigned 64-bit integers") + with self.lock: + if self.sequence == 0 and action != "started": + raise ValueError("the process must announce started before reporting other lifecycle events") + sequence = self.sequence + 1 + event = client_event( + self.source, + "recovery", + action, + { + "schema_version": 1, + "job_id": self.job_id, + "instance": self.instance, + "sequence": sequence, + "progress": progress, + "bytes_count": bytes_count, + "duration_millis": duration_millis, + }, + ).event + message = WsSendAgentMessage( + type="et-send-agent-message", + to_agent_id=self.supervisor, + message=event.model_dump(mode="json"), + ) + self.send(message.model_dump_json()) + self.sequence = sequence + + +@dataclass(frozen=True) +class _Pointer: + """Validated pointer fields, confined to this checkpoint's payload namespace.""" + + filename: str + digest: str + progress: int + + +def _progress(value: object) -> int: + """Reject booleans and out-of-range values before using a checkpoint progress counter.""" + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 2**64 - 1: + raise ValueError("checkpoint progress must be an unsigned 64-bit integer") + return value + + +def _digest(value: object) -> str: + """Accept only a complete lowercase SHA-256 digest.""" + if not isinstance(value, str) or len(value) != 64 or any(c not in "0123456789abcdef" for c in value): + raise ValueError("invalid checkpoint digest") + return value + + +class Checkpoints: + """Commit alternating payload slots through a latest pointer; one application writer owns each key.""" + + def __init__( + self, + storage: Storage, + agent_id: str, + key: str, + max_bytes: int = 64 * 1024 * 1024, + lifecycle: Lifecycle | None = None, + ): + """Bind checkpoints to the producing agent's storage and a bounded application-specific key.""" + if not agent_id or not key or len(key) > 64 or not all(c.isascii() and (c.isalnum() or c in "-_") for c in key): + raise ValueError("checkpoint identity and a simple key of at most 64 characters are required") + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0: + raise ValueError("max_bytes must be a positive integer") + self.storage = storage + self.agent_id = agent_id + self.key = key + self.max_bytes = max_bytes + self.lifecycle = lifecycle + self.lock = threading.Lock() + + def save(self, payload: bytes, progress: int) -> None: + """Write bytes before the pointer so a failed write cannot replace the previous committed checkpoint.""" + started = time.monotonic_ns() + if not isinstance(payload, bytes) or len(payload) > self.max_bytes: + raise ValueError("checkpoint payload must be bytes within the configured size limit") + progress = _progress(progress) + digest = hashlib.sha256(payload).hexdigest() + with self.lock: + if self.storage.agent_id != self.agent_id: + raise ValueError("checkpoint saves require the runner's current storage agent identity") + previous = self._read_pointer() + first = f"{self.key}-slot-0.bin" + filename = f"{self.key}-slot-1.bin" if previous is not None and previous.filename == first else first + self.storage.put(filename, payload) + pointer = {"schema_version": 1, "filename": filename, "sha256": digest, "progress": progress} + self.storage.put(f"{self.key}-latest.json", json.dumps(pointer).encode()) + self._observe("checkpoint-saved", progress, len(payload), started) + + def load(self) -> tuple[bytes, int] | None: + """Verify the latest committed payload; return None only when no checkpoint pointer exists.""" + started = time.monotonic_ns() + with self.lock: + pointer = self._read_pointer() + if pointer is None: + return None + payload = self.storage.get(self.agent_id, pointer.filename) + if ( + payload is None + or len(payload) > self.max_bytes + or hashlib.sha256(payload).hexdigest() != pointer.digest + ): + raise ValueError("checkpoint payload is missing, oversized or corrupt") + self._observe("checkpoint-restored", pointer.progress, len(payload), started) + return bytes(payload), pointer.progress + + def _read_pointer(self) -> _Pointer | None: + """Read a bounded pointer, distinguishing an absent checkpoint from malformed state.""" + raw = self.storage.get(self.agent_id, f"{self.key}-latest.json") + if raw is None: + return None + if len(raw) > 4096: + raise ValueError("checkpoint pointer exceeds its size limit") + pointer = json.loads(raw) + if ( + not isinstance(pointer, dict) + or type(pointer.get("schema_version")) is not int + or pointer["schema_version"] != 1 + ): + raise ValueError("unsupported checkpoint pointer") + progress, digest = _progress(pointer.get("progress")), _digest(pointer.get("sha256")) + filenames = {f"{self.key}-slot-0.bin", f"{self.key}-slot-1.bin", f"{self.key}-{progress}-{digest}.bin"} + filename = pointer.get("filename") + if not isinstance(filename, str) or filename not in filenames: + raise ValueError("checkpoint pointer does not reference its own payload namespace") + return _Pointer(filename, digest, progress) + + def _observe(self, action: str, progress: int, size: int, started: int) -> None: + """Keep telemetry transport failures from invalidating a successful checkpoint operation.""" + if self.lifecycle is not None: + try: + self.lifecycle.emit( + action, progress, bytes_count=size, duration_millis=(time.monotonic_ns() - started) // 1_000_000 + ) + except (OSError, RuntimeError, ValueError): + logging.getLogger(__name__).exception("checkpoint succeeded but lifecycle delivery failed") diff --git a/libs/python-edge-toolkit/tests/test_recovery.py b/libs/python-edge-toolkit/tests/test_recovery.py new file mode 100644 index 00000000..42043627 --- /dev/null +++ b/libs/python-edge-toolkit/tests/test_recovery.py @@ -0,0 +1,297 @@ +"""Modules publish lifecycle evidence and preserve committed checkpoints through storage failures.""" + +import hashlib +import json +from typing import Any +from unittest.mock import Mock + +import pytest +from edge_toolkit.recovery import Checkpoints, Lifecycle + + +class MemoryStorage: + """Model the runner storage contract and allow precise failure injection.""" + + def __init__(self): + """Create isolated namespaces and controls for checkpoint write failures.""" + self.agent_id: str | None = "worker" + self.values = {} + self.namespaces = {"worker": self.values} + self.reads = [] + self.fail_pointer = False + + def put(self, key, value): + """Write to the current agent namespace or inject a failed pointer commit.""" + if self.agent_id is None: + raise RuntimeError("storage agent identity is not assigned") + if self.fail_pointer and key.endswith("-latest.json"): + raise OSError("pointer write failed") + self.namespaces.setdefault(self.agent_id, {})[key] = value + + def get(self, agent_id, key): + """Record the requested namespace and return its stored object when present.""" + self.reads.append((agent_id, key)) + return self.namespaces.get(agent_id, {}).get(key) + + +def test_lifecycle_uses_existing_direct_cloudevent_transport(): + """Verify ordered lifecycle events use the existing acknowledged agent-message envelope.""" + sent = [] + lifecycle = Lifecycle(sent.append, "supervisor", "/modules/example", "job") + for action, progress in [("started", 0), ("progress", 1), ("failed", 1), ("completed", 2)]: + lifecycle.emit(action, progress) + messages = [json.loads(payload) for payload in sent] + assert all(message["to_agent_id"] == "supervisor" for message in messages) + assert all(message["type"] == "et-send-agent-message" for message in messages) + events = [message["message"] for message in messages] + assert [event["data"]["sequence"] for event in events] == [1, 2, 3, 4] + assert [event["type"] for event in events] == [ + "et.recovery.started", + "et.recovery.progress", + "et.recovery.failed", + "et.recovery.completed", + ] + assert all(event["source"] == "/modules/example" and event["specversion"] == "1.0" for event in events) + assert len({event["data"]["instance"] for event in events}) == 1 + assert Lifecycle(sent.append, "supervisor", "/modules/example", "job").instance != lifecycle.instance + + +@pytest.mark.parametrize( + ("supervisor", "source", "job"), [("", "/modules/x", "job"), ("s", "x", "job"), ("s", "/modules/x", "")] +) +def test_lifecycle_requires_explicit_identity(supervisor, source, job): + """Reject incomplete lifecycle bindings before publishing any evidence.""" + with pytest.raises(ValueError, match="required"): + Lifecycle(Mock(), supervisor, source, job) + + +def test_lifecycle_rejects_invalid_or_unannounced_evidence(): + """Reject unsupported actions, invalid counters and progress reported before startup.""" + send = Mock() + lifecycle = Lifecycle(send, "supervisor", "/modules/example", "job") + with pytest.raises(ValueError, match="announce started"): + lifecycle.emit("progress", 1) + with pytest.raises(ValueError, match="unsupported"): + lifecycle.emit("restart-arbitrary-deployment", 1) + invalid_progress: list[Any] = [True, -1, 1.5, 2**64] + for progress in invalid_progress: + with pytest.raises(ValueError, match="unsigned"): + lifecycle.emit("started", progress) + send.assert_not_called() + + +def test_checkpoint_measurements_share_lifecycle_and_failed_send_does_not_advance_sequence(): + """Preserve sequence order across delivery failures and attach checkpoint measurements to lifecycle evidence.""" + sent = [] + send = Mock(side_effect=OSError("transport unavailable")) + lifecycle = Lifecycle(send, "supervisor", "/modules/example", "job") + with pytest.raises(OSError): + lifecycle.emit("started", 0) + assert lifecycle.sequence == 0 + lifecycle.send = sent.append + lifecycle.emit("started", 0) + invalid_measurements: list[Any] = [True, -1, "bad", 2**64] + for value in invalid_measurements: + with pytest.raises(ValueError, match="measurements"): + lifecycle.emit("checkpoint-saved", 0, bytes_count=value) + checkpoints = Checkpoints(MemoryStorage(), "worker", "model", lifecycle=lifecycle) + checkpoints.save(b"weights", 1) + assert checkpoints.load() == (b"weights", 1) + evidence = [json.loads(payload)["message"] for payload in sent] + assert [item["type"] for item in evidence] == [ + "et.recovery.started", + "et.recovery.checkpoint-saved", + "et.recovery.checkpoint-restored", + ] + assert all(item["data"]["bytes_count"] == 7 for item in evidence[1:]) + assert all(item["data"]["duration_millis"] >= 0 for item in evidence[1:]) + + +def test_checkpoint_roundtrip_and_no_pointer_are_distinct(): + """Distinguish an absent checkpoint from a successfully verified payload.""" + storage = MemoryStorage() + checkpoints = Checkpoints(storage, "worker", "model") + assert checkpoints.load() is None + checkpoints.save(b"model weights", 3) + assert checkpoints.load() == (b"model weights", 3) + assert all(agent == "worker" for agent, _ in storage.reads) + pointer = json.loads(storage.values["model-latest.json"]) + assert pointer["sha256"] == hashlib.sha256(b"model weights").hexdigest() + + +def test_checkpoint_success_does_not_depend_on_lifecycle_delivery(caplog): + """Keep committed checkpoints usable when lifecycle telemetry cannot be delivered.""" + lifecycle = Lifecycle(Mock(), "supervisor", "/modules/example", "job") + lifecycle.emit("started", 0) + lifecycle.send = Mock(side_effect=OSError("hub unavailable")) + checkpoints = Checkpoints(MemoryStorage(), "worker", "model", lifecycle=lifecycle) + checkpoints.save(b"valid", 1) + assert checkpoints.load() == (b"valid", 1) + assert caplog.text.count("checkpoint succeeded but lifecycle delivery failed") == 2 + + +def test_failed_pointer_commit_retains_previous_checkpoint(): + """Preserve the committed payload when writing the next latest pointer fails.""" + storage = MemoryStorage() + checkpoints = Checkpoints(storage, "worker", "model") + checkpoints.save(b"committed", 1) + storage.fail_pointer = True + with pytest.raises(OSError, match="pointer write failed"): + checkpoints.save(b"uncommitted", 2) + assert checkpoints.load() == (b"committed", 1) + + +@pytest.mark.parametrize("current_agent", [None, "replacement"]) +def test_mismatched_or_unregistered_checkpoint_writer_is_rejected_before_storage_access(current_agent): + """Reject stale writer identities before reading or changing either storage namespace.""" + storage = MemoryStorage() + checkpoints = Checkpoints(storage, "worker", "model") + checkpoints.save(b"previous agent", 1) + storage.agent_id = current_agent + before = {agent: values.copy() for agent, values in storage.namespaces.items()} + reads = storage.reads.copy() + with pytest.raises(ValueError, match="current storage agent identity"): + checkpoints.save(b"wrong namespace", 2) + assert storage.reads == reads + assert storage.namespaces == before + assert checkpoints.load() == (b"previous agent", 1) + + +def test_cross_agent_restore_does_not_corrupt_new_agents_checkpoint_on_failed_commit(): + """Restore old-agent state without allowing stale writers to corrupt the new agent's committed checkpoint.""" + storage = MemoryStorage() + previous = Checkpoints(storage, "worker", "model") + previous.save(b"old agent", 1) + storage.agent_id = "replacement" + assert previous.load() == (b"old agent", 1) + current = Checkpoints(storage, "replacement", "model") + current.save(b"new agent committed", 2) + storage.fail_pointer = True + with pytest.raises(ValueError, match="current storage agent identity"): + previous.save(b"would overwrite current slot", 3) + with pytest.raises(OSError, match="pointer write failed"): + current.save(b"new agent uncommitted", 3) + assert current.load() == (b"new agent committed", 2) + assert previous.load() == (b"old agent", 1) + + +def test_repeated_saves_and_failed_commits_keep_storage_bounded(): + """Bound payload retention even when saves repeatedly fail before pointer commitment.""" + storage = MemoryStorage() + checkpoints = Checkpoints(storage, "worker", "model") + for progress in range(20): + checkpoints.save(str(progress).encode(), progress) + assert checkpoints.load() == (str(progress).encode(), progress) + assert len(storage.values) <= 3 + storage.fail_pointer = True + for progress in range(20, 30): + with pytest.raises(OSError, match="pointer write failed"): + checkpoints.save(str(progress).encode(), progress) + assert checkpoints.load() == (b"19", 19) + assert len(storage.values) == 3 + storage.fail_pointer = False + checkpoints.save(b"resumed", 30) + assert Checkpoints(storage, "worker", "model").load() == (b"resumed", 30) + assert len(storage.values) == 3 + + +def test_legacy_pointer_restores_and_next_save_uses_bounded_slots(): + """Read legacy content-addressed checkpoints while bounding storage used by subsequent saves.""" + storage = MemoryStorage() + payload = b"legacy" + digest = hashlib.sha256(payload).hexdigest() + filename = f"model-1-{digest}.bin" + storage.values[filename] = payload + storage.values["model-latest.json"] = json.dumps( + {"schema_version": 1, "filename": filename, "sha256": digest, "progress": 1} + ).encode() + checkpoints = Checkpoints(storage, "worker", "model") + assert checkpoints.load() == (payload, 1) + for progress in range(2, 10): + checkpoints.save(b"current", progress) + assert checkpoints.load() == (b"current", progress) + assert len(storage.values) == 4 + + +def test_save_does_not_overwrite_malformed_pointer_or_other_payloads(): + """Reject malformed saved pointers without overwriting existing storage objects.""" + storage = MemoryStorage() + storage.values["model-latest.json"] = b"invalid pointer" + before = storage.values.copy() + with pytest.raises(ValueError): + Checkpoints(storage, "worker", "model").save(b"new", 1) + assert storage.values == before + + +@pytest.mark.parametrize( + ("agent", "key"), [("", "model"), ("worker", ""), ("worker", "../state"), ("w", "x" * 65), ("w", "\u00e9")] +) +def test_checkpoint_names_cannot_escape_the_application_namespace(agent, key): + """Reject checkpoint keys that could address another application namespace.""" + with pytest.raises(ValueError, match="simple key"): + Checkpoints(MemoryStorage(), agent, key) + + +@pytest.mark.parametrize("limit", [0, -1, True, 1.5]) +def test_checkpoint_size_limit_is_a_positive_integer(limit): + """Reject disabled, fractional and Boolean checkpoint size limits.""" + with pytest.raises(ValueError, match="positive integer"): + Checkpoints(MemoryStorage(), "worker", "model", max_bytes=limit) + + +def test_checkpoint_save_enforces_payload_and_progress_limits(): + """Reject oversized payloads and counters outside the unsigned checkpoint range.""" + checkpoints = Checkpoints(MemoryStorage(), "worker", "model", max_bytes=3) + invalid_payloads: list[Any] = ["not bytes", b"too large"] + for payload in invalid_payloads: + with pytest.raises(ValueError, match="payload"): + checkpoints.save(payload, 1) + invalid_progress: list[Any] = [True, -1, "one", 2**64] + for progress in invalid_progress: + with pytest.raises(ValueError, match="progress"): + checkpoints.save(b"ok", progress) + + +@pytest.mark.parametrize( + "pointer", + [ + [], + {"schema_version": 2}, + {"schema_version": True}, + {"schema_version": 1, "progress": True}, + {"schema_version": 1, "progress": -1}, + {"schema_version": 1, "progress": 2**64}, + {"schema_version": 1, "progress": 0, "sha256": "bad"}, + {"schema_version": 1, "progress": 0, "sha256": "z" * 64}, + {"schema_version": 1, "progress": 0, "sha256": "a" * 64, "filename": "../private"}, + {"schema_version": 1, "progress": 0, "sha256": "a" * 64, "filename": []}, + ], +) +def test_invalid_checkpoint_pointers_fail_before_reading_payload(pointer): + """Reject unsupported or malformed pointers before attempting a payload read.""" + storage = MemoryStorage() + storage.values["model-latest.json"] = json.dumps(pointer).encode() + with pytest.raises(ValueError): + Checkpoints(storage, "worker", "model").load() + assert storage.reads == [("worker", "model-latest.json")] + + +def test_unreadable_and_oversized_checkpoint_pointers_fail_explicitly(): + """Report invalid pointer encoding and excessive pointer size as errors rather than absent checkpoints.""" + storage = MemoryStorage() + for payload in [b"not json", b"x" * 4097]: + storage.values["model-latest.json"] = payload + with pytest.raises(ValueError): + Checkpoints(storage, "worker", "model").load() + + +@pytest.mark.parametrize("payload", [None, b"corrupt", b"x" * 20]) +def test_missing_corrupt_or_oversized_payloads_cannot_be_restored(payload): + """Reject payloads that are absent, too large or inconsistent with the committed digest.""" + storage = MemoryStorage() + checkpoints = Checkpoints(storage, "worker", "model", max_bytes=10) + checkpoints.save(b"valid", 1) + pointer = json.loads(storage.values["model-latest.json"]) + storage.values[pointer["filename"]] = payload + with pytest.raises(ValueError, match="missing, oversized or corrupt"): + checkpoints.load() diff --git a/libs/test-recovery/Cargo.toml b/libs/test-recovery/Cargo.toml new file mode 100644 index 00000000..c8f14ad6 --- /dev/null +++ b/libs/test-recovery/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "et-test-recovery" +publish = true +description = "Domain-specific fixtures for recovery supervisor integration tests" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +doctest = false + +[[bin]] +bench = false +doc = false +name = "recovery-command-probe" +path = "src/bin/command-probe.rs" +test = false + +[dependencies] +chrono.workspace = true +clap.workspace = true +edge-toolkit.workspace = true +et-recovery-supervisor.workspace = true +serde-env.workspace = true +serde_json.workspace = true + +[dev-dependencies] +futures-util.workspace = true +tokio = { workspace = true, features = ["io-util", "macros", "process", "rt", "time"] } +tracing.workspace = true +tracing-subscriber.workspace = true + +[lints] +workspace = true diff --git a/libs/test-recovery/README.md b/libs/test-recovery/README.md new file mode 100644 index 00000000..82db307b --- /dev/null +++ b/libs/test-recovery/README.md @@ -0,0 +1,5 @@ +# Recovery test helpers + +`et-test-recovery` provides the example configuration, lifecycle event builders and policy fixtures used by +recovery integration tests. It is a development dependency, not part of the deployed supervisor. Recovery-specific +fixtures stay here so the general WebSocket test server does not depend on recovery policy. diff --git a/libs/test-recovery/src/bin/command-probe.rs b/libs/test-recovery/src/bin/command-probe.rs new file mode 100644 index 00000000..3eacea52 --- /dev/null +++ b/libs/test-recovery/src/bin/command-probe.rs @@ -0,0 +1,18 @@ +//! Recovery command fixture: emit both streams, then complete or wait for termination. + +use std::io::{self, Write as _}; + +use clap::Parser as _; +use et_test_recovery::command::Arguments; + +/// Keep the delayed mode finite so an interrupted parent cannot leave an indefinitely running fixture. +fn main() -> io::Result<()> { + let arguments = Arguments::parse(); + let delay = arguments.delay(); + io::stdout().write_all(b"fixture stdout\n")?; + io::stderr().write_all(b"fixture stderr\n")?; + io::stdout().flush()?; + io::stderr().flush()?; + std::thread::sleep(delay); + Ok(()) +} diff --git a/libs/test-recovery/src/command.rs b/libs/test-recovery/src/command.rs new file mode 100644 index 00000000..f48f8338 --- /dev/null +++ b/libs/test-recovery/src/command.rs @@ -0,0 +1,37 @@ +//! Typed modes for the recovery command subprocess fixture. + +use std::time::Duration; + +use clap::{Parser, ValueEnum}; + +/// Emit subprocess diagnostics and select when the fixture exits. +#[derive(Parser)] +#[command(name = "recovery-command-probe")] +pub struct Arguments { + /// Completion behaviour used by the parent command test. + #[arg(value_enum)] + mode: Mode, +} + +/// Accepted subprocess behaviours; arbitrary arguments cannot select additional actions. +#[derive(Clone, Copy, ValueEnum)] +enum Mode { + /// Exit immediately after flushing the diagnostic streams. + Complete, + /// Exit after a short delay to exercise normal asynchronous collection. + Pause, + /// Wait for the parent's deadline while remaining finite if the parent disappears. + Hang, +} + +impl Arguments { + /// Return the selected finite delay without sleeping so every mode can be verified directly. + #[must_use] + pub const fn delay(&self) -> Duration { + match self.mode { + Mode::Complete => Duration::ZERO, + Mode::Pause => Duration::from_millis(25), + Mode::Hang => Duration::from_secs(120), + } + } +} diff --git a/libs/test-recovery/src/lib.rs b/libs/test-recovery/src/lib.rs new file mode 100644 index 00000000..461d0f07 --- /dev/null +++ b/libs/test-recovery/src/lib.rs @@ -0,0 +1,81 @@ +//! Shared inputs for recovery supervisor integration tests. +#![expect( + clippy::unwrap_used, + reason = "invalid test fixtures must fail the calling integration test" +)] + +use std::collections::BTreeMap; + +use chrono::Utc; +use edge_toolkit::ws::CloudEvent; +use et_recovery_supervisor::config::{Config, EnvConfig, Mode}; +use et_recovery_supervisor::policy::{Budget, Supervisor}; + +pub mod command; + +/// Parse a deterministic test environment using the supervisor's production configuration shape. +#[must_use] +pub fn config() -> Config { + let config: EnvConfig = serde_env::from_iter(environment()).unwrap(); + config.recovery +} + +/// Build test settings without mutating the test process's environment or requiring deployment files. +#[must_use] +pub fn environment() -> BTreeMap { + let values = [ + ("RECOVERY_AGENT_ID", "recovery-supervisor"), + ("RECOVERY_COMMAND_TIMEOUT_SECONDS", "10"), + ("RECOVERY_CONTEXT", "k3d-example"), + ("RECOVERY_JOB_ID", "example-job"), + ("RECOVERY_MODE", "observe-only"), + ("RECOVERY_NAMESPACE", "example"), + ("RECOVERY_POLICY_DEADLINE_SECONDS", "240"), + ("RECOVERY_POLICY_DETECTION_GRACE_SECONDS", "3"), + ("RECOVERY_POLICY_MAX_RESTARTS", "2"), + ("RECOVERY_POLICY_RESTART_AFTER_SECONDS", "20"), + ("RECOVERY_POLICY_STALL_SECONDS", "180"), + ("RECOVERY_POLL_SECONDS", "2"), + ("RECOVERY_STATE_FILE", "target/recovery-state.json"), + // Intentionally default to a loopback-only test endpoint; configuration parsing does not open a connection. + // nosemgrep: config.semgrep.no-semgrep-mentions -- external rule annotation + // nosemgrep: javascript.lang.security.detect-insecure-websocket.detect-insecure-websocket + ("RECOVERY_WS_URL", "ws://127.0.0.1:8080/ws"), + ]; + let mut env: BTreeMap = values + .into_iter() + .map(|(key, value)| (key.into(), value.into())) + .collect(); + let workloads = serde_json::json!([{ + "deployment": "example-worker", "agent_id": "example-worker", "source": "/modules/example-worker" + }]); + let _previous = env.insert("RECOVERY_WORKLOADS".into(), workloads.to_string()); + env +} + +/// Construct an isolated recovery policy using the example workload. +#[must_use] +pub fn supervisor(mode: Mode) -> Supervisor { + let mut config = config(); + config.mode = mode; + Supervisor::new(config.clone(), Budget::new(config.job_id)).unwrap() +} + +/// Build a lifecycle event for the example application identity and job. +#[must_use] +pub fn event(action: &str, progress: u64, sequence: u64, instance: &str) -> CloudEvent { + CloudEvent::new( + format!("event-{sequence}"), + "/modules/example-worker".into(), + format!("et.recovery.{action}"), + Utc::now(), + serde_json::json!({ "schema_version": 1_u64, "job_id": "example-job", "instance": instance, + "sequence": sequence, "progress": progress }), + ) +} + +/// Convert a typed report into its wire representation for protocol assertions. +#[must_use] +pub fn report(supervisor: &Supervisor) -> serde_json::Value { + serde_json::to_value(supervisor.report()).unwrap() +} diff --git a/libs/test-recovery/tests/command.rs b/libs/test-recovery/tests/command.rs new file mode 100644 index 00000000..a8184e40 --- /dev/null +++ b/libs/test-recovery/tests/command.rs @@ -0,0 +1,198 @@ +//! Exercise recovery command output, bounded diagnostics and deadlines using a Cargo-built subprocess fixture. +#![cfg(test)] + +use std::io; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use std::time::Duration; + +use clap::Parser as _; +use et_recovery_supervisor::RecoveryError; +use et_recovery_supervisor::command::{capture_stderr, output}; +use et_test_recovery::command::Arguments; +use tokio::io::AsyncWriteExt as _; +use tokio::process::Command; +use tracing::subscriber::{NoSubscriber, set_default}; +use tracing_subscriber::fmt::MakeWriter; + +/// Capture diagnostic fields without using a global subscriber shared by parallel tests. +#[derive(Clone, Default)] +struct Logs(Arc>>); + +impl io::Write for Logs { + /// Append each formatted event atomically to the fixture's captured log. + fn write(&mut self, buf: &[u8]) -> io::Result { + let Ok(mut bytes) = self.0.lock() else { + return Err(io::Error::other("diagnostic log mutex poisoned")); + }; + bytes.extend_from_slice(buf); + Ok(buf.len()) + } + + /// Memory-backed capture has no pending writes to flush. + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl<'writer> MakeWriter<'writer> for Logs { + type Writer = Self; + + /// Share the fixture's buffer across event-formatting calls. + fn make_writer(&'writer self) -> Self::Writer { + self.clone() + } +} + +/// Launch the Cargo-built fixture; an empty mode omits the argument to exercise missing-input rejection. +fn probe(mode: &str) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_recovery-command-probe")); + if !mode.is_empty() { + let _command = command.arg(mode); + } + command +} + +/// Verify long-wait mode selection without waiting for its fallback termination delay. +#[test] +fn fixture_modes_select_finite_completion_delays() { + for (mode, delay) in [ + ("complete", Duration::ZERO), + ("pause", Duration::from_millis(25)), + ("hang", Duration::from_secs(120)), + ] { + let arguments = Arguments::try_parse_from(["recovery-command-probe", mode]).unwrap(); + assert_eq!(arguments.delay(), delay, "mode={mode}"); + } +} + +/// Collect both streams on a normal process exit. +#[tokio::test] +async fn successful_commands_preserve_stdout_and_stderr() { + let _logging = set_default(NoSubscriber::default()); + for mode in ["complete", "pause"] { + let result = output(&mut probe(mode), Duration::from_secs(60)).await.unwrap(); + assert!(result.status.success(), "mode={mode}"); + assert_eq!(result.stdout, b"fixture stdout\n"); + assert_eq!(result.stderr, b"fixture stderr\n"); + } +} + +/// Reject a mistyped fixture mode rather than silently running the wrong subprocess behaviour. +#[tokio::test] +async fn probe_rejects_missing_or_unknown_modes() { + for (mode, expected) in [("", "required arguments"), ("invalid", "invalid value")] { + let result = output(&mut probe(mode), Duration::from_secs(60)).await.unwrap(); + assert_eq!(result.status.code(), Some(2_i32), "mode={mode}"); + assert_eq!(result.stdout, b""); + let error = String::from_utf8(result.stderr).unwrap(); + assert!(error.contains(expected), "mode={mode}: {error}"); + assert!(error.contains("--help"), "mode={mode}: {error}"); + } +} + +/// Reject trailing arguments and unknown flags before emitting any fixture output. +#[tokio::test] +async fn probe_rejects_unexpected_arguments() { + for argument in ["unexpected", "--unknown"] { + let mut command = probe("complete"); + let _command = command.arg(argument); + let result = output(&mut command, Duration::from_secs(60)).await.unwrap(); + assert_eq!(result.status.code(), Some(2_i32), "argument={argument}"); + assert_eq!(result.stdout, b""); + let error = String::from_utf8(result.stderr).unwrap(); + assert!(error.contains("unexpected argument"), "{error}"); + assert!(error.contains(argument), "{error}"); + } +} + +/// Describe every accepted mode without running the subprocess behaviour. +#[tokio::test] +async fn probe_help_lists_modes_without_emitting_diagnostics() { + let result = output(&mut probe("--help"), Duration::from_secs(60)).await.unwrap(); + assert!(result.status.success()); + assert_eq!(result.stderr, b""); + let help = String::from_utf8(result.stdout).unwrap(); + for mode in ["complete", "pause", "hang"] { + assert!(help.contains(mode), "missing mode {mode}: {help}"); + } + assert!(!help.contains("fixture stdout")); +} + +/// Report the actual PID, spawn timing, process state and stderr field when a child exceeds its deadline. +#[tokio::test] +async fn timeout_retains_its_error_type_and_logs_child_diagnostics() { + let logs = Logs::default(); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_writer(logs.clone()) + .finish(); + let _logging = set_default(subscriber); + let error = output(&mut probe("hang"), Duration::from_secs(1)).await.unwrap_err(); + assert!(matches!(error, RecoveryError::CommandTimeout(_))); + let text = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + for expected in [ + "kubectl process spawned", + "pid=", + "state=", + "stderr=", + "stderr_limit_reached=false", + ] { + assert!(text.contains(expected), "missing {expected}: {text}"); + } + for field in ["spawn_seconds", "elapsed_seconds"] { + let prefix = format!("{field}="); + let value = text + .split_whitespace() + .find_map(|word| word.strip_prefix(&prefix)) + .unwrap() + .parse::() + .unwrap(); + assert!(value.is_finite() && value >= 0.0_f64, "invalid {field}: {text}"); + } +} + +/// Disabling diagnostics must not change the deadline error or prevent termination of the child. +#[tokio::test] +async fn timeout_remains_an_error_without_a_logging_subscriber() { + let _logging = set_default(NoSubscriber::default()); + let error = output(&mut probe("hang"), Duration::from_secs(1)).await.unwrap_err(); + assert!(matches!(error, RecoveryError::CommandTimeout(_))); +} + +/// Retain bytes already received even when an open pipe prevents the read future from completing. +#[tokio::test] +async fn cancelled_stderr_capture_keeps_the_partial_prefix() { + let (mut reader, mut writer) = tokio::io::duplex(128); + writer.write_all(b"partial diagnostic").await.unwrap(); + let mut bytes = Vec::new(); + { + let read = capture_stderr(&mut reader, &mut bytes); + tokio::pin!(read); + assert!(matches!(futures_util::poll!(read.as_mut()), Poll::Pending)); + } + assert_eq!(bytes, b"partial diagnostic"); +} + +/// Drain oversized stderr to EOF while retaining only the configured diagnostic prefix. +#[tokio::test] +async fn stderr_capture_is_bounded_without_blocking_the_writer() { + let (mut reader, mut writer) = tokio::io::duplex(128); + let mut bytes = Vec::new(); + let (read, ()) = tokio::join!(capture_stderr(&mut reader, &mut bytes), async move { + writer.write_all(&vec![b'x'; 70 * 1024]).await.unwrap(); + }); + read.unwrap(); + assert_eq!(bytes.len(), 64 * 1024); + assert!(bytes.iter().all(|byte| *byte == b'x')); +} + +/// Reject clock-overflowing deadlines before attempting to spawn an executable. +#[tokio::test] +async fn unrepresentable_deadlines_fail_before_process_creation() { + let mut command = Command::new("et-intentionally-absent-kubectl-probe"); + let error = output(&mut command, Duration::from_secs(u64::MAX)).await.unwrap_err(); + assert!(matches!(error, RecoveryError::Invalid(_))); + assert!(error.to_string().contains("clock limits")); +} diff --git a/services/recovery-supervisor/Cargo.toml b/services/recovery-supervisor/Cargo.toml new file mode 100644 index 00000000..82dc242d --- /dev/null +++ b/services/recovery-supervisor/Cargo.toml @@ -0,0 +1,59 @@ +[package] +name = "et-recovery-supervisor" +default-run = "et-recovery-supervisor" +publish = true +description = "Application-neutral workload recovery supervision for Edge Toolkit" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[package.metadata.docs.rs] +all-features = true + +[lib] +doctest = false + +[[bin]] +name = "et-recovery-supervisor" +path = "src/main.rs" + +[[bin]] +name = "et-recovery-healthcheck" +path = "src/healthcheck.rs" + +[features] +## Internal: derives the environment configuration schema for generated help. +env-schema = ["dep:schemars"] +## Internal: renders this feature list into the crate documentation. +docs = ["dep:document-features"] + +[dependencies] +document-features = { workspace = true, optional = true } +edge-toolkit.workspace = true +et-ws-runner-common.workspace = true +fs-err.workspace = true +futures-util.workspace = true +schemars = { workspace = true, optional = true } +serde.workspace = true +serde-env.workspace = true +serde_json.workspace = true +tempfile.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["io-util", "macros", "process", "rt-multi-thread", "signal", "time"] } +tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] } +tracing.workspace = true +tracing-subscriber.workspace = true +url.workspace = true +uuid.workspace = true + +[dev-dependencies] +et-otlp.workspace = true +et-test-helpers.workspace = true +et-test-recovery.workspace = true +et-ws-test-server.workspace = true +k8s-openapi.workspace = true +serde_yaml.workspace = true + +[lints] +workspace = true diff --git a/services/recovery-supervisor/Dockerfile b/services/recovery-supervisor/Dockerfile new file mode 100644 index 00000000..99dd5315 --- /dev/null +++ b/services/recovery-supervisor/Dockerfile @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1 + +FROM registry.k8s.io/kubectl:v1.35.0 AS kubectl +FROM rust:1-bookworm AS builder +RUN bash <<'EOF' +set -euo pipefail +apt-get update +apt-get install -y --no-install-recommends clang cmake pkg-config +apt-get clean +rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb +EOF + +WORKDIR /workspace +COPY . . +RUN cargo build --release --locked -p et-recovery-supervisor + +FROM debian:bookworm-slim +LABEL org.opencontainers.image.source="https://github.com/edge-toolkit/core" +RUN bash <<'EOF' +set -euo pipefail +apt-get update +apt-get install -y --no-install-recommends ca-certificates +apt-get clean +rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb +useradd --create-home --uid 10001 --user-group app +EOF + +COPY --from=kubectl /bin/kubectl /usr/local/bin/kubectl +COPY --from=builder /workspace/target/release/et-recovery-supervisor /usr/local/bin/et-recovery-supervisor +COPY --from=builder /workspace/target/release/et-recovery-healthcheck /usr/local/bin/et-recovery-healthcheck +ENV HOME=/app/runtime +WORKDIR /app +USER 10001 +# The service persists its final report on Ctrl-C; use that signal for container shutdown too. +STOPSIGNAL SIGINT +HEALTHCHECK --interval=10s --timeout=3s --start-period=60s --retries=3 CMD ["et-recovery-healthcheck"] +CMD ["et-recovery-supervisor"] diff --git a/services/recovery-supervisor/HELP.md b/services/recovery-supervisor/HELP.md new file mode 100644 index 00000000..c47faf8b --- /dev/null +++ b/services/recovery-supervisor/HELP.md @@ -0,0 +1,95 @@ +# Help for `et-recovery-supervisor` + +`et-recovery-supervisor` takes no command-line arguments; it reads its configuration from the environment. + +## Environment variables + +### `RECOVERY_AGENT_ID` + +Type: string. Required. + +Stable identity reserved for this supervisor. + +### `RECOVERY_COMMAND_TIMEOUT_SECONDS` + +Type: integer. Required. + +Bound on each kubectl process, including credential-plugin execution. + +### `RECOVERY_CONTEXT` + +Type: string. Required. + +Explicit Kubernetes context; never relies on the operator's current context. + +### `RECOVERY_JOB_ID` + +Type: string. Required. + +Application job whose lifecycle events are eligible. + +### `RECOVERY_MODE` + +Type: one of `observe-only`, `controller-assisted`. Default: `observe-only`. + +Observe-only unless explicitly enabled. + +### `RECOVERY_NAMESPACE` + +Type: string. Required. + +Namespace containing every allowlisted deployment. + +### `RECOVERY_POLICY_DEADLINE_SECONDS` + +Type: integer. Required. + +Maximum time from first unhealthy observation to useful recovery. + +### `RECOVERY_POLICY_DETECTION_GRACE_SECONDS` + +Type: integer. Required. + +Continuous unhealthy observation before declaring a failure. + +### `RECOVERY_POLICY_MAX_RESTARTS` + +Type: integer. Required. + +Lifetime restart request budget per configured workload and job, persisted before executing an action. + +### `RECOVERY_POLICY_RESTART_AFTER_SECONDS` + +Type: integer. Required. + +Minimum time between detection and an action, and between subsequent attempts. + +### `RECOVERY_POLICY_STALL_SECONDS` + +Type: integer. Required. + +Active work without advancing progress is considered stalled after this interval. + +### `RECOVERY_POLL_SECONDS` + +Type: integer. Required. + +Time between infrastructure observation passes. + +### `RECOVERY_STATE_FILE` + +Type: string. Required. + +Durable restart budget and latest recovery report; parent directory must exist. + +### `RECOVERY_WORKLOADS` + +Type: string. Required. + +Explicit restart allowlist and sender bindings. + +### `RECOVERY_WS_URL` + +Type: string. Required. + +Hub WebSocket URL. diff --git a/services/recovery-supervisor/README.md b/services/recovery-supervisor/README.md new file mode 100644 index 00000000..fb816b82 --- /dev/null +++ b/services/recovery-supervisor/README.md @@ -0,0 +1,242 @@ +# Recovery supervisor + +`et-recovery-supervisor` is an opt-in Core service for observing application failures and coordinating bounded recovery. +It is separate from the WebSocket hub: the hub routes messages and serves storage, while the supervisor owns recovery +policy. Kubernetes performs deployment restarts; application modules decide how their work is checkpointed and resumed. +No machine-learning library, application adapter, dataset, or fault-injection tool is required by the service. + +## Scenario deployment + +Add an optional `recovery:` block to a k3s scenario. Omit it for an ordinary scenario without a supervisor: + +```yaml +cluster_name: example +deployment_type: k3s +agents: + - name: example-worker + runner: pyo3 + resources: + - type: pyo3-math1 +recovery: + job_id: example-job + mode: observe-only + poll_seconds: 2 + command_timeout_seconds: 10 + policy: + detection_grace_seconds: 3 + restart_after_seconds: 20 + stall_seconds: 180 + deadline_seconds: 240 + max_restarts: 2 + workloads: + - deployment: example-worker + agent_id: example-worker + source: /modules/@edge-toolkit/et-ws-pyo3-math1 +``` + +Generate and deploy the scenario using the normal CLI workflow. The output includes the supervisor Deployment, +`RECOVERY_*` settings, ServiceAccount, named-Deployment Role and RoleBinding, projected-token kubeconfig and persistent +state claim. Observe-only grants `get`; controller-assisted also grants `patch`. No host kubeconfig is mounted. +The generated README includes image preparation and deployment commands. Recovery currently requires k3s output; +other formats are rejected rather than silently omitting supervision. Optional `recovery.limits` uses the standard +scenario resource-limit fields. + +Supervised targets must be declared, continuously running native-Python workloads. Their generated settings include +`PYO3_AGENT_ID`, `RECOVERY_JOB_ID` and `RECOVERY_SUPERVISOR_AGENT_ID`. The module must use the shared job/supervisor +settings, emit matching lifecycle evidence and implement its own restoration logic. The math1 example above illustrates +deployment wiring only: math1 does not itself report recovery progress or restore checkpoints. Generating its supervisor +does not add these application behaviors automatically. Scheduled and other runner types are not supported targets. +Application dependencies remain the module author's responsibility. Modules importing Core's Python recovery helpers +need the SDK installed in their runner image; scenario generation does not install Python packages. + +Supervisor and supervised Deployments use `Recreate` replacement. The supervisor has one replica and persistent budget +state; it is not a highly available controller. A different job cannot reuse an old job's state: archive the old report +and deliberately provision fresh state before changing the job. Do not run a second recovery controller over the same +workloads. Hub or physical-node loss is not covered by this deployment integration. + +## Supervisor health + +The image includes `et-recovery-healthcheck`, which reads the same `RECOVERY_*` settings as the service. It performs +local checks only: no network listener, Kubernetes command, or recovery action is involved. Success requires a held +budget lease and a recent completed supervision cycle from its current job and session. Startup, stale or malformed +evidence, old sessions, and an exited supervisor fail the probe. A sibling `.health.json` file records the +heartbeat only after observations, policy actions and report persistence finish; shutdown writes do not refresh it. +The `.lock` file holds exclusive ownership only; readable session identity lives in +`.session`, written after the lease is acquired. Keeping these separate lets health probes work with +Windows mandatory file locks. A leftover identity or heartbeat without a live lease never establishes health. + +The freshness allowance includes the poll interval, bounded observation and potential restart commands for every +configured workload, and ten seconds of I/O slack. Future timestamps fail closed; this local freshness test does not +change the monotonic clocks used for recovery metrics. A supervised workload failure or an unavailable Kubernetes +observation does not itself make the supervisor unhealthy while its loop continues advancing. Unknown observations +remain visible in its recovery report rather than being presented as healthy applications. + +Docker runs the probe every ten seconds, with a sixty-second startup grace and three attempts before reporting an +unhealthy container. Adjust the container's startup grace for unusually large workload sets. Generated k3s deployments +use the same executable for startup, readiness and liveness. Their startup budget includes registration and a bounded +cycle; after startup, three failed liveness probes can restart the supervisor. Kubernetes does not inherit Docker's +health check. Such a restart preserves the durable action budget but begins a new observer session, not a continuation +of its incident history. + +## Standalone example + +The binary reads environment settings through serde-env and takes no configuration-file argument. Supply real identities +and a surviving state directory, then start it separately from the application: + +```sh +export RECOVERY_AGENT_ID=recovery-supervisor +export RECOVERY_WS_URL=ws://127.0.0.1:8080/ws +export RECOVERY_CONTEXT=k3d-example +export RECOVERY_NAMESPACE=example +export RECOVERY_JOB_ID=example-job +export RECOVERY_MODE=observe-only +export RECOVERY_STATE_FILE=target/recovery-state.json +export RECOVERY_POLL_SECONDS=2 +export RECOVERY_COMMAND_TIMEOUT_SECONDS=10 +export RECOVERY_POLICY_DETECTION_GRACE_SECONDS=3 +export RECOVERY_POLICY_RESTART_AFTER_SECONDS=20 +export RECOVERY_POLICY_STALL_SECONDS=180 +export RECOVERY_POLICY_DEADLINE_SECONDS=240 +export RECOVERY_POLICY_MAX_RESTARTS=2 +export RECOVERY_WORKLOADS='[ + {"deployment":"example-worker","agent_id":"example-worker","source":"/modules/example-worker"} +]' +mise run recovery-supervisor +``` + +`kubectl` must be on PATH with credentials for the explicitly configured context. Standalone startup does not apply +deployment manifests. The state directory must already exist and survive supervisor restarts. + +`RECOVERY_WORKLOADS` is a JSON array of objects with `deployment`, `agent_id`, and `source` fields. It remains an array +for a single workload. The loader explicitly decodes this JSON; malformed entries and unknown workload fields fail +startup. Policy settings use `RECOVERY_POLICY_*`, such as `RECOVERY_POLICY_STALL_SECONDS`. Runtime targets and policy +are validated before observing or changing deployments. + +The default is `observe-only`. Set `RECOVERY_MODE=controller-assisted` to permit restarts. Observe-only still writes +reports and its state file, but never requests deployment changes. A single supervisor owns each job and workload set. +A file lock prevents concurrent owners of the same state file; separate state files do not provide distributed leader +election, so operators must not run overlapping supervisors against the same deployments. + +## Recovery policy + +The supervisor combines deployment readiness with application lifecycle evidence received over the existing hub +agent-message transport. An unhealthy deployment, an explicit application failure, or active work that stops advancing +can open an incident. Modules that have not started work do not fail merely for being idle. Completed work does not +acquire another processing obligation, although an unhealthy deployment remains observable. + +The detection grace period filters brief disruptions. Once failure is detected, the supervisor waits before requesting +a restart, allowing native Kubernetes recovery to act first. Restart requests have a cooldown and a per-workload budget +per job. Each request consumes its budget before execution, even if Kubernetes rejects it. The reservation +is persisted before the external command; a persistence failure stops the service without executing that command. +Serialization and state-file writes run off the async transport task. Saves remain ordered even when their caller is +cancelled, so a final shutdown report cannot be overwritten by an older in-flight save. +An unreadable or mismatched saved budget fails closed. Do not delete the state file to bypass an exhausted budget. +On Unix, successful state replacement also syncs the parent directory so the renamed budget survives a filesystem +crash. Windows retains atomic replacement and file syncing; it does not claim equivalent directory-sync durability. + +API errors are unknown observations, not evidence that an application failed. They cannot trigger restarts. Kubectl +processes have deadlines and are killed when cancelled; commands use explicit argument lists, not shell interpolation. +Supervised deployments must have exactly one desired replica. This initial backend does not choose replacement nodes, +change replica counts, move datasets, or repair a physical machine. + +A Ready deployment is not sufficient to close a detected incident. Application progress must advance beyond its +pre-failure value, and the workload must no longer be unhealthy. Reporting a restored checkpoint alone does not prove +unfinished work continued. Missing evidence remains incomplete; crossing the recovery deadline stops further actions. +Later useful recovery is recorded with its actual duration, not presented as meeting the deadline. + +## Application contract + +Modules send a CloudEvent as the payload of an `et-send-agent-message` addressed to the configured supervisor agent. +This uses the existing direct-message path and acknowledgements without changing the hub wire protocol. A hub +`et-client-event` alone is logged by the hub, not delivered to the supervisor. The SDK supplies the envelope: + +```python +from edge_toolkit.recovery import Checkpoints, Lifecycle + +# Run after on_connect supplies the producing agent identity. +lifecycle = Lifecycle(send.text, "recovery-supervisor", "/modules/example-worker", "example-job") +lifecycle.emit("started", 0) +checkpoints = Checkpoints(storage, agent_id, "example-job-model", lifecycle=lifecycle) +restored = checkpoints.load() +# Reconstruct application state from restored bytes when a checkpoint exists. +# After committing one new unit of useful work: +checkpoints.save(b"application-defined state", 1) +lifecycle.emit("progress", 1) +# At the end of the job: +lifecycle.emit("completed", 1) +``` + +Every lifecycle payload has `schema_version = 1`, `job_id`, `instance`, `sequence`, and unsigned `progress` fields. +The SDK creates a new process instance and increasing sequence numbers. Event types are `et.recovery.started`, +`et.recovery.progress`, `et.recovery.failed`, `et.recovery.completed`, `et.recovery.checkpoint-saved`, +`et.recovery.checkpoint-restored`, and `et.recovery.resume-completed`. Checkpoint events also carry `bytes_count` and +`duration_millis`. The application can emit `resume-completed` after reconstructing its in-memory state; resuming useful +processing is verified separately through progress. Progress units are application-defined committed work, not attempted +operations: training rounds, completed batches, or completed inference tasks, for example. + +The exact configured sender, module source, and job must match. The first observed instance can be adopted mid-run, +with its existing counter establishing a baseline rather than proving recovery. Replacement instances announce +`started` first; replayed sequence numbers and retired instances are rejected within the observer session. These +bindings reduce accidental cross-talk, but hub agent IDs and CloudEvent sources are not authenticated identities. Use +controller-assisted mode only on a trusted hub/network. This is not malicious-node detection or a security boundary +against another hub participant claiming an allowlisted agent ID. Kubernetes credentials belong to the supervisor, +never to browser modules. + +Use `wss` across untrusted networks. The supervisor validates TLS certificates using the operating system's trusted +roots; a self-signed server certificate must be trusted there too, not just accepted in a browser. Plain `ws` remains +supported for loopback and trusted internal deployments. URL validation does not turn an unencrypted connection into a +secure one. When the bounded evidence queue fills, delivery waits for capacity before transport acknowledgement +instead of discarding lifecycle events. Outbound keepalive pings continue while application data reads are +backpressured. Acknowledgement confirms queue admission, not successful policy processing or durable persistence. +Observation cycles drain a bounded batch so sustained producers cannot indefinitely postpone health reporting. +Keep reporting rates below the observer's capacity; backpressure does not increase processing throughput. + +Checkpoints alternate between two bounded payload slots and an integrity-checked latest pointer in the producing agent +namespace. The inactive slot is written before the pointer changes, so a failed commit leaves the previous checkpoint +available. Repeated or failed saves create at most two payloads and one pointer per key. Loading rejects corrupt, +missing, oversized, or foreign-namespace pointers/payloads. The application owns serialization, safe commit boundaries, +optimizer state, participation rules and duplicate-work handling. Never deserialize untrusted executable formats. Use +one writer per checkpoint key and one shared helper for concurrent access within a process. Legacy content-addressed +checkpoints remain readable; future saves use the bounded slots, but orphaned legacy objects require operator cleanup. +Saving requires the checkpoint namespace to match `storage.agent_id`, the runner's current registered identity. +After an identity change, an old helper may still load the previous agent's checkpoint, but saves are rejected before +storage access. Create a helper bound to the new identity before saving resumed work in its namespace. +Integrity hashes do not authenticate producers. Guarantees depend on whole-object pointer writes and storage surviving +worker restarts; hub/storage loss and power-loss durability are not solved here. Telemetry delivery failures are logged +without undoing a successful checkpoint operation. + +## Evidence and measurements + +The state file contains the durable restart budget and latest JSON report. Reports distinguish suspected/transient, +detected, recovered, and unrecovered incidents, action requests and failures, unknown infrastructure observations, and +checkpoint save/load/resume measurements. `failure_detection_seconds` measures grace-qualified detection from first +unhealthy observation. `recovery_seconds` measures from that observation to verified useful progress. `mttr_seconds` +averages recovered incidents only and is null when none recovered; incomplete incidents remain in the report. These are +observer-relative measurements, not injection-to-recovery times or continuous uptime guarantees. Polling and command +latency limit resolution. Checkpoint timing is producer-reported; cross-node clock synchronisation is not needed for +those durations. Resume efficiency requires application-specific work-attempt evidence and is not inferred here. + +Incident detection and recovery timestamps are null until observed, rather than using zero as a missing value. +Action records contain a `kind` (`restart-reserved`, `restart-requested`, or `restart-failed`); failed actions also +contain an `error` description. Checkpoint records retain their operation kind, instance, progress, size and duration. + +Budget counters survive supervisor restarts. Lifecycle sequence tracking and incident histories belong to one observer +session and are not restored; archive reports before restarting if historical measurements are needed. A lost hub +connection stops the service and its actions rather than continuing to act on stale application evidence. Restart the +service using the same state file after connectivity returns; this first version does not reconnect its live session. + +## Permissions and verification + +Restrict Kubernetes access to `get` and `patch` on the named Deployments in the configured namespace. `rollout restart` +uses a patch; no Pod deletion, secret reads, cluster-wide access, or arbitrary command execution is required. Operators +must provide appropriately scoped credentials. Run the supervisor outside the supervised workloads, with persistent +state storage. Generated deployments provide these permissions and persistent storage automatically. + +```sh +mise run test-recovery-supervisor +MISE_ENV=python mise run test-edge-toolkit +``` + +The tests use deterministic policy observations, a fake deployment backend, and real in-process hub transport. They do +not inject faults or mutate a live cluster. Fault injection, workload-specific benchmark scenarios, repetition +statistics and experimental SLO evaluation remain independent consumers of Core recovery evidence. diff --git a/services/recovery-supervisor/src/command.rs b/services/recovery-supervisor/src/command.rs new file mode 100644 index 00000000..f3dc7fdb --- /dev/null +++ b/services/recovery-supervisor/src/command.rs @@ -0,0 +1,112 @@ +//! Preserve bounded subprocess diagnostics when Kubernetes commands exceed their configured deadline. + +use std::process::{Output, Stdio}; +use std::time::Duration; + +use tokio::io::{AsyncRead, AsyncReadExt as _}; +use tokio::process::{Child, ChildStderr, ChildStdout, Command}; +use tokio::time::{Instant, timeout_at}; +use tracing::{info, warn}; + +use crate::RecoveryError; + +const STDERR_LIMIT: usize = 64 * 1024; + +/// Drain stderr without blocking the child, retaining a bounded prefix even if this future is cancelled. +pub async fn capture_stderr(reader: &mut R, bytes: &mut Vec) -> std::io::Result<()> +where + R: AsyncRead + Unpin, +{ + let mut chunk = [0_u8; 4096]; + loop { + let count = reader.read(&mut chunk).await?; + if count == 0 { + return Ok(()); + } + let remaining = STDERR_LIMIT.saturating_sub(bytes.len()); + bytes.extend(chunk.iter().take(count.min(remaining)).copied()); + } +} + +/// Collect kubectl output within one deadline, logging actual child state and partial stderr on timeout. +pub async fn output(command: &mut Command, timeout: Duration) -> Result { + let started = Instant::now(); + let deadline = started + .checked_add(timeout) + .ok_or_else(|| RecoveryError::Invalid("kubectl command deadline exceeds clock limits".into()))?; + let (mut child, mut stdout, mut stderr) = spawn_piped(command, started)?; + let mut stdout_bytes = Vec::new(); + let mut stderr_bytes = Vec::new(); + let result = timeout_at(deadline, async { + tokio::try_join!( + child.wait(), + stdout.read_to_end(&mut stdout_bytes), + capture_stderr(&mut stderr, &mut stderr_bytes), + ) + }) + .await; + match result { + Ok(outcome) => { + let (status, _, ()) = outcome?; + Ok(Output { + status, + stdout: stdout_bytes, + stderr: stderr_bytes, + }) + } + Err(error) => Err(timeout_error(&mut child, started, &stderr_bytes, error)), + } +} + +/// Start the child with owned pipes and record startup timing before collecting its output. +#[expect( + clippy::single_call_fn, + reason = "process setup is distinct from deadline-bound output collection" +)] +fn spawn_piped(command: &mut Command, started: Instant) -> Result<(Child, ChildStdout, ChildStderr), RecoveryError> { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn()?; + let pid = child.id(); + let spawn_seconds = started.elapsed().as_secs_f64(); + info!(?pid, spawn_seconds, "kubectl process spawned"); + let stdout = child + .stdout + .take() + .ok_or_else(|| RecoveryError::Invalid("kubectl stdout pipe is unavailable".into()))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| RecoveryError::Invalid("kubectl stderr pipe is unavailable".into()))?; + Ok((child, stdout, stderr)) +} + +/// Preserve timeout semantics even if the operating system cannot report the child's current state. +#[expect( + clippy::single_call_fn, + reason = "timeout diagnostics are distinct from concurrent output collection" +)] +fn timeout_error( + child: &mut Child, + started: Instant, + stderr: &[u8], + error: tokio::time::error::Elapsed, +) -> RecoveryError { + let pid = child.id(); + let state = child.try_wait(); + let elapsed_seconds = started.elapsed().as_secs_f64(); + let stderr_limit_reached = stderr.len() == STDERR_LIMIT; + let stderr = String::from_utf8_lossy(stderr); + warn!( + ?pid, + ?state, + elapsed_seconds, + stderr = %stderr, + stderr_limit_reached, + "kubectl deadline exceeded; dropping the child requests termination" + ); + error.into() +} diff --git a/services/recovery-supervisor/src/config.rs b/services/recovery-supervisor/src/config.rs new file mode 100644 index 00000000..921d4fe4 --- /dev/null +++ b/services/recovery-supervisor/src/config.rs @@ -0,0 +1,248 @@ +//! Validated, opt-in recovery configuration with one application job per supervisor. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; + +use crate::RecoveryError; + +/// Whether the supervisor may request workload restarts. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "env-schema", derive(schemars::JsonSchema))] +#[non_exhaustive] +#[serde(rename_all = "kebab-case")] +pub enum Mode { + /// Record incidents and recovery evidence without changing deployments. + #[default] + ObserveOnly, + /// Request restarts within the configured budget after the grace period. + ControllerAssisted, +} + +/// Explicit timing and action bounds, in seconds measured by the supervisor's monotonic clock. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg_attr(feature = "env-schema", derive(schemars::JsonSchema))] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct Policy { + /// Continuous unhealthy observation before declaring a failure. + pub detection_grace_seconds: u64, + /// Minimum time between detection and an action, and between subsequent attempts. + pub restart_after_seconds: u64, + /// Active work without advancing progress is considered stalled after this interval. + pub stall_seconds: u64, + /// Maximum time from first unhealthy observation to useful recovery. + pub deadline_seconds: u64, + /// Lifetime restart request budget per configured workload and job, persisted before executing an action. + pub max_restarts: u32, +} + +/// One deployment and the application identity permitted to report its useful progress. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg_attr(feature = "env-schema", derive(schemars::JsonSchema))] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct Workload { + /// Kubernetes Deployment name within the configured namespace. + pub deployment: String, + /// Hub-assigned sender identity expected on lifecycle messages. + pub agent_id: String, + /// Exact `CloudEvent` source, normally the producing module's served path. + pub source: String, +} + +/// Inputs for a single supervisor; namespace and targets cannot be supplied by application messages. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg_attr(feature = "env-schema", derive(schemars::JsonSchema))] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct Config { + /// Hub WebSocket URL. + pub ws_url: String, + /// Stable identity reserved for this supervisor. + pub agent_id: String, + /// Application job whose lifecycle events are eligible. + pub job_id: String, + /// Explicit Kubernetes context; never relies on the operator's current context. + pub context: String, + /// Namespace containing every allowlisted deployment. + pub namespace: String, + /// Observe-only unless explicitly enabled. + #[serde(default)] + pub mode: Mode, + /// Time between infrastructure observation passes. + pub poll_seconds: u64, + /// Bound on each kubectl process, including credential-plugin execution. + pub command_timeout_seconds: u64, + /// Durable restart budget and latest recovery report; parent directory must exist. + pub state_file: PathBuf, + /// Shared recovery policy. + pub policy: Policy, + /// Explicit restart allowlist and sender bindings. + #[serde(deserialize_with = "deserialize_workloads")] + #[cfg_attr(feature = "env-schema", schemars(with = "String"))] + pub workloads: Vec, +} + +/// Environment configuration grouped under the `RECOVERY_*` prefix. +#[derive(Debug, Deserialize)] +#[cfg_attr(feature = "env-schema", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub struct EnvConfig { + /// Recovery connection settings, bounded policy, and workload allowlist. + pub recovery: Config, +} + +/// Decode the environment's JSON workload list while preserving structured configuration round trips. +fn deserialize_workloads<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum Input { + Encoded(String), + Structured(Vec), + } + match Input::deserialize(deserializer)? { + Input::Encoded(value) => match serde_json::from_str(&value) { + Ok(workloads) => Ok(workloads), + Err(error) => Err(serde::de::Error::custom(error)), + }, + Input::Structured(value) => Ok(value), + } +} + +/// Validate Kubernetes DNS identifiers without permitting flags, path fragments, or selectors. +#[must_use] +pub fn valid_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 63 + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && value.starts_with(|character: char| character.is_ascii_alphanumeric()) + && value.ends_with(|character: char| character.is_ascii_alphanumeric()) +} + +impl Config { + /// Build in-cluster settings with observe-only mode, two-second polling and ten-second command deadlines. + /// + /// Customize these defaults as needed and validate the completed settings before starting supervision. + #[must_use] + pub fn in_cluster( + ws_url: String, + namespace: String, + job_id: String, + policy: Policy, + workloads: Vec, + ) -> Self { + Self { + ws_url, + agent_id: "recovery-supervisor".into(), + job_id, + context: "in-cluster".into(), + namespace, + mode: Mode::ObserveOnly, + poll_seconds: 2, + command_timeout_seconds: 10, + state_file: PathBuf::from("/app/state/recovery.json"), + policy, + workloads, + } + } + + /// Reject ambiguous identities, unsafe targets, and timing settings that cannot support the policy. + pub fn validate(&self) -> Result<(), RecoveryError> { + self.validate_identity()?; + self.policy.validate()?; + if self.poll_seconds == 0 || self.command_timeout_seconds == 0 || self.workloads.is_empty() { + return Err(RecoveryError::Invalid( + "positive polling and command deadlines and at least one workload are required".into(), + )); + } + let mut agents = BTreeSet::new(); + let mut deployments = BTreeSet::new(); + for workload in &self.workloads { + workload.validate(&self.agent_id)?; + if !agents.insert(&workload.agent_id) || !deployments.insert(&workload.deployment) { + return Err(RecoveryError::Invalid( + "workload targets and sender identities must be unique".into(), + )); + } + } + Ok(()) + } + + /// Reject ambiguous job, deployment namespace and durable-state identities before connecting. + fn validate_identity(&self) -> Result<(), RecoveryError> { + if self.context.is_empty() + || self.context.starts_with('-') + || !valid_name(&self.namespace) + || self.agent_id.is_empty() + || self.job_id.is_empty() + || self.state_file.as_os_str().is_empty() + { + return Err(RecoveryError::Invalid( + "invalid hub, job, namespace, context, or state location".into(), + )); + } + validate_endpoint(&self.ws_url) + } +} + +/// Validate transport syntax without forbidding trusted-cluster deployments that terminate TLS elsewhere. +#[expect( + clippy::single_call_fn, + reason = "connection syntax is independent of recovery identity and timing" +)] +fn validate_endpoint(value: &str) -> Result<(), RecoveryError> { + let endpoint = url::Url::parse(value)?; + // Plain WebSockets support trusted cluster networks; use TLS across untrusted network boundaries. + if !["ws", "wss"].contains(&endpoint.scheme()) + || endpoint.host_str().is_none() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.fragment().is_some() + { + return Err(RecoveryError::Invalid("invalid recovery WebSocket endpoint".into())); + } + Ok(()) +} + +impl Policy { + /// Validate positive recovery timings and a deadline that permits recovery. + fn validate(&self) -> Result<(), RecoveryError> { + if [ + self.detection_grace_seconds, + self.restart_after_seconds, + self.stall_seconds, + self.deadline_seconds, + ] + .contains(&0) + || self.deadline_seconds <= self.detection_grace_seconds.saturating_add(self.restart_after_seconds) + { + return Err(RecoveryError::Invalid( + "recovery timings must be positive and permit detection followed by recovery".into(), + )); + } + Ok(()) + } +} + +impl Workload { + /// Validate one deployment target and its permitted lifecycle sender. + fn validate(&self, supervisor: &str) -> Result<(), RecoveryError> { + if !valid_name(&self.deployment) + || self.agent_id.is_empty() + || self.agent_id == supervisor + || !self.source.starts_with("/modules/") + { + return Err(RecoveryError::Invalid( + "workload target and sender identity must be valid".into(), + )); + } + Ok(()) + } +} diff --git a/services/recovery-supervisor/src/health.rs b/services/recovery-supervisor/src/health.rs new file mode 100644 index 00000000..6f99807d --- /dev/null +++ b/services/recovery-supervisor/src/health.rs @@ -0,0 +1,130 @@ +//! Local, session-bound supervision-cycle evidence; no listener, credentials, or workload mutations in a probe. + +// fs-err wraps filesystem operations but uses the standard library's lock error type. +use std::fs::TryLockError; +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use fs_err as fs; +use serde::{Deserialize, Serialize}; + +use crate::RecoveryError; +use crate::config::Config; +use crate::persistence::StateStore; + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Evidence { + schema_version: u32, + job_id: String, + session: String, + completed_at: SystemTime, +} + +/// Own the budget lease and publish health only after a complete, durably recorded observation cycle. +#[derive(Debug)] +pub struct Session { + // This guard holds the exclusive budget lock until the supervisor exits, including cancelled futures. + _lease: fs::File, + store: StateStore, + job_id: String, + identity: String, +} + +impl Session { + /// Acquire exclusive ownership before changing the session identity or publishing health evidence. + pub fn acquire(config: &Config) -> Result { + config.validate()?; + let lease = fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(config.state_file.with_extension("lock"))?; + lease.try_lock()?; + let identity = uuid::Uuid::new_v4().to_string(); + // Windows mandatory locks reject reads through another handle with "os error 33". Keep the identity outside the + // locked file, and replace it only after acquiring ownership. + fs::write(identity_path(config), identity.as_bytes())?; + Ok(Self { + _lease: lease, + store: StateStore::new(evidence_path(config)), + job_id: config.job_id.clone(), + identity, + }) + } + + /// Refresh evidence only when the policy loop has finished observing, acting, and persisting its report. + pub async fn advance(&self) -> Result<(), RecoveryError> { + self.store + .persist(Evidence { + schema_version: 1, + job_id: self.job_id.clone(), + session: self.identity.clone(), + completed_at: SystemTime::now(), + }) + .await + } +} + +/// Bound a full supervision cycle using configured command deadlines and I/O slack. +/// +/// Includes polling, one observation and potential restart per workload, and ten seconds for persistence. +#[must_use] +pub fn silence_limit(config: &Config) -> Duration { + let commands = config.workloads.iter().fold(0_u64, |budget, _workload| { + budget.saturating_add(config.command_timeout_seconds.saturating_mul(2)) + }); + Duration::from_secs(config.poll_seconds.saturating_add(commands).saturating_add(10)) +} + +/// Reject absent owners, old sessions, malformed evidence, stale cycles, and timestamps ahead of the local clock. +pub fn check(config: &Config) -> Result<(), RecoveryError> { + config.validate()?; + let session = current_session(config)?; + let evidence: Evidence = serde_json::from_slice(&fs::read(evidence_path(config))?)?; + evidence.validate(config, &session) +} + +/// Require a live owner before reading the session identity; a stale lock file is not proof of liveness. +#[expect( + clippy::single_call_fn, + reason = "lease ownership is checked separately from completed-cycle evidence" +)] +fn current_session(config: &Config) -> Result { + let lock_path = config.state_file.with_extension("lock"); + let lease = fs::OpenOptions::new().read(true).write(true).open(&lock_path)?; + match lease.try_lock() { + Ok(()) => return Err(RecoveryError::Invalid("recovery supervisor is not running".into())), + Err(TryLockError::WouldBlock) => {} + // Unsupported filesystems and other OS lock failures are errors, not evidence of a live owner. + Err(error) => return Err(error.into()), + } + Ok(fs::read_to_string(identity_path(config))?) +} + +impl Evidence { + /// Match the live session and reject incomplete or expired supervision-cycle evidence. + fn validate(&self, config: &Config, session: &str) -> Result<(), RecoveryError> { + if self.schema_version != 1 || self.job_id != config.job_id || self.session != session { + return Err(RecoveryError::Invalid( + "no completed cycle for the current recovery session".into(), + )); + } + let age = SystemTime::now().duration_since(self.completed_at)?; + if age > silence_limit(config) { + return Err(RecoveryError::Invalid("recovery supervision cycle is stale".into())); + } + Ok(()) + } +} + +/// Keep the local heartbeat beside the durable budget without overwriting that budget. +fn evidence_path(config: &Config) -> PathBuf { + config.state_file.with_extension("health.json") +} + +/// Keep readable session metadata separate from the exclusively locked ownership file. +fn identity_path(config: &Config) -> PathBuf { + config.state_file.with_extension("session") +} diff --git a/services/recovery-supervisor/src/healthcheck.rs b/services/recovery-supervisor/src/healthcheck.rs new file mode 100644 index 00000000..b5450370 --- /dev/null +++ b/services/recovery-supervisor/src/healthcheck.rs @@ -0,0 +1,10 @@ +//! Exit successfully only when the configured supervisor owns its lease and has completed a recent cycle. + +use et_recovery_supervisor::config::EnvConfig; + +/// Load the configured job and validate its local supervision heartbeat. +fn main() -> Result<(), Box> { + let config = serde_env::from_env::()?.recovery; + et_recovery_supervisor::health::check(&config)?; + Ok(()) +} diff --git a/services/recovery-supervisor/src/kubernetes.rs b/services/recovery-supervisor/src/kubernetes.rs new file mode 100644 index 00000000..4e0559cc --- /dev/null +++ b/services/recovery-supervisor/src/kubernetes.rs @@ -0,0 +1,134 @@ +//! Namespace-scoped Kubernetes operations; targets come only from validated operator configuration. + +use std::time::Duration; + +use serde::Deserialize; +use tokio::process::Command; + +use crate::RecoveryError; +use crate::config::{Config, Workload}; +use crate::policy::Health; + +/// Deployment-specific operations consumed by the supervisor, independently testable with an in-memory backend. +pub trait Backend: Send { + /// Observe readiness; failures must remain errors, not be interpreted as application failure. + fn health(&mut self, workload: &Workload) -> impl Future> + Send; + /// Request recovery of an allowlisted workload; success means accepted, not recovered. + fn restart(&mut self, workload: &Workload) -> impl Future> + Send; +} + +/// Kubernetes backend using the installed kubectl credentials and an explicit context and namespace. +#[derive(Debug)] +pub struct Kubernetes { + config: Config, +} + +impl Kubernetes { + /// Validate targets before any external command is possible. + pub fn new(config: Config) -> Result { + config.validate()?; + Ok(Self { config }) + } + + /// Execute kubectl directly, without shell interpolation, bounding both the API request and child lifetime. + pub async fn execute(&self, workload: &Workload, arguments: &[&str]) -> Result, RecoveryError> { + if !self + .config + .workloads + .iter() + .any(|item| item.deployment == workload.deployment) + { + return Err(RecoveryError::Invalid("Kubernetes target is not allowlisted".into())); + } + let target = format!("deployment/{}", workload.deployment); + let mut command = Command::new("kubectl"); + let _command = command + .args([ + "--context", + &self.config.context, + "--namespace", + &self.config.namespace, + "--request-timeout=5s", + ]) + .args(arguments) + .arg(target); + let output = + crate::command::output(&mut command, Duration::from_secs(self.config.command_timeout_seconds)).await?; + if !output.status.success() { + return Err(RecoveryError::Invalid(format!( + "kubectl {arguments:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + Ok(output.stdout) + } +} + +#[derive(Deserialize)] +struct Deployment { + metadata: Metadata, + spec: Spec, + #[serde(default)] + status: DeploymentStatus, +} + +#[derive(Deserialize)] +struct Metadata { + generation: u64, +} + +#[derive(Deserialize)] +struct Spec { + replicas: u32, +} + +#[derive(Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DeploymentStatus { + #[serde(default)] + observed_generation: u64, + #[serde(default)] + available_replicas: u32, + #[serde(default)] + updated_replicas: u32, + #[serde(default)] + replicas: u32, + #[serde(default)] + unavailable_replicas: u32, +} + +/// Decode readiness without assuming that an HTTP-available process has resumed application work. +pub fn deployment_health(bytes: &[u8]) -> Result { + let deployment: Deployment = serde_json::from_slice(bytes)?; + if deployment.spec.replicas != 1 { + return Err(RecoveryError::Invalid( + "supervised deployments must have exactly one desired replica".into(), + )); + } + let status = deployment.status; + Ok( + if status.observed_generation >= deployment.metadata.generation + && status.available_replicas == 1 + && status.updated_replicas == 1 + && status.replicas == 1 + && status.unavailable_replicas == 0 + { + Health::Ready + } else { + Health::Unready + }, + ) +} + +impl Backend for Kubernetes { + /// Query and decode readiness for one allowlisted deployment. + async fn health(&mut self, workload: &Workload) -> Result { + deployment_health(&self.execute(workload, &["get", "-o", "json"]).await?) + } + + /// Request a bounded rollout restart for one allowlisted deployment. + async fn restart(&mut self, workload: &Workload) -> Result<(), RecoveryError> { + let _output = self.execute(workload, &["rollout", "restart"]).await?; + Ok(()) + } +} diff --git a/services/recovery-supervisor/src/lib.rs b/services/recovery-supervisor/src/lib.rs new file mode 100644 index 00000000..11846d5c --- /dev/null +++ b/services/recovery-supervisor/src/lib.rs @@ -0,0 +1,66 @@ +//! Bounded, application-neutral recovery supervision, separate from fault injection and the WebSocket hub. +#![cfg_attr(feature = "docs", doc = document_features::document_features!())] + +// fs-err retains the standard library's lock error type. +use std::fs::TryLockError; + +pub mod command; +pub mod config; +pub mod health; +pub mod kubernetes; +pub mod persistence; +pub mod policy; +pub mod runtime; + +/// Errors preserve the failed operation instead of treating missing observations as healthy workloads. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum RecoveryError { + /// Invalid configuration, evidence, or command outcome. + #[error("{0}")] + Invalid(String), + /// The configured WebSocket endpoint is not a URL. + #[error("invalid recovery WebSocket URL: {0}")] + Url(#[from] url::ParseError), + /// Filesystem or process creation failed. + #[error("I/O failure: {0}")] + Io(#[from] std::io::Error), + /// A heartbeat timestamp lies ahead of the local wall clock. + #[error("recovery health timestamp is in the future: {0}")] + HealthClock(#[from] std::time::SystemTimeError), + /// A complete state document could not replace its predecessor. + #[error("state persistence failed: {0}")] + Persist(#[from] tempfile::PersistError), + /// A blocking persistence task could not complete. + #[error("state persistence task failed: {0}")] + PersistenceTask(#[from] tokio::task::JoinError), + /// The configured budget is already owned or cannot be locked on this filesystem. + #[error("cannot acquire budget lock: {0}")] + Lease(#[from] TryLockError), + /// The external Kubernetes command exceeded its allowed lifetime. + #[error("kubectl process exceeded its deadline: {0}")] + CommandTimeout(#[from] tokio::time::error::Elapsed), + /// A JSON payload could not be decoded or encoded. + #[error("JSON failure: {0}")] + Json(#[from] serde_json::Error), + /// Connecting and registering with the hub failed. + #[error("hub registration failed: {0}")] + Connect(#[from] et_ws_runner_common::ConnectError), + /// A live hub connection failed. + #[error("hub connection failed: {0}")] + Socket(Box), +} + +impl From for RecoveryError { + /// Preserve the WebSocket error while keeping the public error enum compact. + fn from(error: tokio_tungstenite::tungstenite::Error) -> Self { + Self::Socket(Box::new(error)) + } +} + +impl From> for RecoveryError { + /// Retain evidence-queue failure context in the supervisor error. + fn from(error: tokio::sync::mpsc::error::TrySendError<(String, edge_toolkit::ws::CloudEvent)>) -> Self { + Self::Invalid(format!("recovery evidence queue unavailable: {error}")) + } +} diff --git a/services/recovery-supervisor/src/main.rs b/services/recovery-supervisor/src/main.rs new file mode 100644 index 00000000..621fe98e --- /dev/null +++ b/services/recovery-supervisor/src/main.rs @@ -0,0 +1,16 @@ +//! Recovery service entrypoint; policy and backend behavior are exercised through the library. + +use et_recovery_supervisor::config::EnvConfig; +use et_recovery_supervisor::kubernetes::Kubernetes; + +/// Load environment configuration and run recovery supervision. +#[tokio::main] +async fn main() -> Result<(), Box> { + let config = serde_env::from_env::()?.recovery; + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init()?; + let backend = Kubernetes::new(config.clone())?; + et_recovery_supervisor::runtime::run(config, backend).await?; + Ok(()) +} diff --git a/services/recovery-supervisor/src/persistence.rs b/services/recovery-supervisor/src/persistence.rs new file mode 100644 index 00000000..787c7d79 --- /dev/null +++ b/services/recovery-supervisor/src/persistence.rs @@ -0,0 +1,79 @@ +//! Ordered state replacement without blocking the async transport or racing a cancelled save. + +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +#[cfg(unix)] +use fs_err as fs; +use serde::Serialize; +use tokio::sync::Mutex; + +use crate::RecoveryError; + +/// One state file's ordered writer. Clones share the same write ordering and must be used for that file's saves. +#[derive(Clone, Debug)] +pub struct StateStore { + path: PathBuf, + serial: Arc>, +} + +impl StateStore { + /// Bind a writer to its operator-configured state file without creating or modifying it. + #[must_use] + pub fn new(path: PathBuf) -> Self { + Self { + path, + serial: Arc::new(Mutex::new(())), + } + } + + /// Serialize and atomically replace state on a blocking thread, completing before recovery may be requested. + pub async fn persist(&self, value: T) -> Result<(), RecoveryError> + where + T: Serialize + Send + 'static, + { + // The blocking task owns the permit: cancelling its waiter cannot let a later save overtake this write. + let permit = Arc::clone(&self.serial).lock_owned().await; + let path = self.path.clone(); + tokio::task::spawn_blocking(move || { + let _permit = permit; + persist(&path, &value) + }) + .await? + } +} + +/// Stage a flushed document and atomically commit it at the configured state path. +#[expect( + clippy::single_call_fn, + reason = "atomic filesystem operation is isolated from async scheduling" +)] +fn persist(path: &Path, value: &T) -> Result<(), RecoveryError> +where + T: Serialize, +{ + let absolute = std::path::absolute(path)?; + let parent = absolute + .parent() + .ok_or_else(|| RecoveryError::Invalid("state path must name a file within a directory".into()))?; + let bytes = serde_json::to_vec(value)?; + let temporary = stage(parent, &bytes)?; + let _file = temporary.persist(&absolute)?; + // Persist the directory entry too: syncing only the payload does not make a rename crash-durable on Unix. + #[cfg(unix)] + fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +/// Flush a complete replacement before making it visible at the durable state's path. +#[expect( + clippy::single_call_fn, + reason = "staging a flushed file is distinct from committing its directory entry" +)] +fn stage(parent: &Path, bytes: &[u8]) -> Result { + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; + temporary.write_all(bytes)?; + temporary.as_file().sync_all()?; + Ok(temporary) +} diff --git a/services/recovery-supervisor/src/policy.rs b/services/recovery-supervisor/src/policy.rs new file mode 100644 index 00000000..7da8531f --- /dev/null +++ b/services/recovery-supervisor/src/policy.rs @@ -0,0 +1,624 @@ +//! Deterministic recovery decisions driven by observer time and application evidence, not by benchmark actions. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use edge_toolkit::ws::CloudEvent; +use serde::{Deserialize, Serialize}; + +use crate::RecoveryError; +use crate::config::{Config, Mode, Workload}; + +/// Infrastructure observations distinguish API failure from an unhealthy deployment. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum Health { + /// The desired single replica is available at its current generation. + Ready, + /// The deployment exists but is not ready, or has no running replica. + Unready, + /// Observation failed; no new recovery action is safe on that evidence alone. + Unknown, +} + +/// Decision emitted by a policy step; only the trusted runtime executes restarts. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum Decision { + /// Observe without modifying the deployment. + Observe, + /// A restart request is eligible, subject to durable budget reservation. + Restart, +} + +/// An incident's measurable lifecycle; incomplete incidents are never counted as successful recoveries. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[non_exhaustive] +#[serde(rename_all = "kebab-case")] +pub enum Status { + /// Unhealthy evidence has not yet satisfied the detection grace period. + Suspected, + /// Failure detected; useful recovery has not been verified. + Detected, + /// Application useful progress advanced beyond its pre-failure value. + Recovered, + /// Recovery deadline passed without useful progress. + Unrecovered, + /// Unhealthy observation cleared before the detection grace period elapsed. + Transient, +} + +/// A missing observation is distinct from a real timestamp at observer second zero. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[non_exhaustive] +#[serde(untagged)] +pub enum ObservationTime { + /// This transition has not been observed; serialized as null. + Pending, + /// Elapsed observer seconds when the transition occurred. + Recorded(u64), +} + +/// A recovery action's outcome is separate from evidence that useful work resumed. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[non_exhaustive] +#[serde(tag = "kind", rename_all = "kebab-case")] +pub enum Action { + /// Budget was reserved before attempting an external operation. + RestartReserved, + /// Kubernetes accepted the restart request. + RestartRequested, + /// The backend rejected or could not deliver the restart request. + RestartFailed { + /// Operational error returned by the backend. + error: String, + }, +} + +/// Checkpoint operations reported by an application, not inferred from container readiness. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[non_exhaustive] +pub enum CheckpointKind { + /// A checkpoint was committed to storage. + #[serde(rename = "checkpoint-saved")] + Saved, + /// Saved bytes were loaded and validated. + #[serde(rename = "checkpoint-restored")] + Restored, + /// The application reconstructed its in-memory state. + #[serde(rename = "resume-completed")] + ResumeCompleted, +} + +/// One validated application checkpoint measurement. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[non_exhaustive] +pub struct CheckpointMeasurement { + /// Deployment reporting the operation. + pub workload: String, + /// Checkpoint operation performed. + pub kind: CheckpointKind, + /// Application process instance. + pub instance: String, + /// Observer-relative receipt time in seconds. + pub observed_at: u64, + /// Application-defined completed work counter. + pub progress: u64, + /// Payload size in bytes. + pub bytes: u64, + /// Producer-measured operation duration. + pub duration_millis: u64, +} + +/// An unavailable measurement must not be interpreted as a zero-duration success. +#[derive(Clone, Copy, Debug, Serialize)] +#[non_exhaustive] +#[serde(untagged)] +pub enum MeanSeconds { + /// No eligible observations have been recorded; serialized as null. + Unavailable, + /// Arithmetic mean of eligible durations, in seconds. + Measured(f64), +} + +/// Owned report snapshot, safe to serialize off the async control task. +#[derive(Clone, Debug, Serialize)] +#[non_exhaustive] +pub struct Report { + /// Persisted report format version. + pub schema_version: u32, + /// Job supervised by this observer. + pub job_id: String, + /// Whether external recovery actions are permitted. + pub mode: Mode, + /// Time coordinate used for incident observations. + pub clock_basis: &'static str, + /// Incidents recorded during this observer session. + pub incidents: Vec, + /// Failed infrastructure observations, distinct from workload failures. + pub unknown_observations: u64, + /// Durations from first unhealthy observation to verified useful recovery. + pub recovery_seconds: Vec, + /// Number of incidents with verified recovery. + pub recovered_incidents: usize, + /// Durable job-level recovery action counters. + pub restart_budget: Budget, + /// Durations from first unhealthy observation to grace-qualified detection. + pub failure_detection_seconds: Vec, + /// Mean detection duration, or null without eligible observations. + pub mean_failure_detection_seconds: MeanSeconds, + /// Mean useful recovery duration, or null without verified recoveries. + pub mttr_seconds: MeanSeconds, + /// Number of grace-qualified failures, including those not recovered. + pub detected_incidents: usize, + /// Application-reported checkpoint operations. + pub checkpoint_measurements: Vec, +} + +/// Incident times are elapsed observer seconds within one supervisor process, not producer timestamps. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[non_exhaustive] +pub struct Incident { + /// Deployment affected by the incident. + pub workload: String, + /// First unhealthy observation. + pub onset: u64, + /// Grace-qualified detection time, or pending before detection. + pub detected: ObservationTime, + /// Verified useful recovery time, or pending before recovery. + pub recovered: ObservationTime, + /// Useful work completed before the incident. + pub baseline: u64, + /// Incident outcome. + pub status: Status, + /// Executed action outcomes, distinct from Kubernetes' own process recovery. + pub actions: Vec, +} + +#[derive(Debug, Default)] +enum Phase { + #[default] + Healthy, + Incident(usize), +} + +#[derive(Debug, Default)] +struct State { + active: bool, + failed: bool, + completed: bool, + progress: u64, + progress_at: u64, + instance: String, + retired: BTreeSet, + sequence: u64, + phase: Phase, + last_action: u64, +} + +/// Durable action budget. An attempted restart consumes budget even if the backend fails or the process exits. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[non_exhaustive] +#[serde(deny_unknown_fields)] +pub struct Budget { + /// Persisted format version. + pub schema_version: u32, + /// Job identity prevents accidentally reusing another job's budget. + pub job_id: String, + /// Lifetime restart requests per deployment. + pub restarts: BTreeMap, +} + +impl Budget { + /// Begin a new job with no recovery actions consumed. + #[must_use] + pub const fn new(job_id: String) -> Self { + Self { + schema_version: 1, + job_id, + restarts: BTreeMap::new(), + } + } +} + +/// A policy instance supervises exactly one configured job; no application event can change its target allowlist. +#[derive(Debug)] +pub struct Supervisor { + config: Config, + states: BTreeMap, + /// Measured incidents for this observer process. + incidents: Vec, + /// Durable lifetime action counters. + budget: Budget, + /// API failures counted separately from workload failures. + unknown_observations: u64, + checkpoints: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum EventKind { + Started, + Progress, + Failed, + Completed, + Checkpoint(CheckpointKind), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Evidence { + schema_version: u32, + job_id: String, + instance: String, + sequence: u64, + progress: u64, + #[serde(default)] + bytes_count: u64, + #[serde(default)] + duration_millis: u64, +} + +impl Supervisor { + /// Incidents observed in this process, including incomplete and transient observations. + #[must_use] + pub fn incidents(&self) -> &[Incident] { + &self.incidents + } + + /// Durable restart counts; callers persist this budget before executing a reserved action. + #[must_use] + pub const fn budget(&self) -> &Budget { + &self.budget + } + + /// Construct policy state after validating both configuration and any restored action budget. + pub fn new(config: Config, budget: Budget) -> Result { + config.validate()?; + if budget.schema_version != 1 || budget.job_id != config.job_id { + return Err(RecoveryError::Invalid( + "restart budget version or job identity does not match configuration".into(), + )); + } + let states = config + .workloads + .iter() + .map(|item| (item.deployment.clone(), State::default())) + .collect(); + Ok(Self { + config, + states, + incidents: Vec::new(), + budget, + unknown_observations: 0, + checkpoints: Vec::new(), + }) + } + + /// Consume an allowlisted sender's lifecycle `CloudEvent`, rejecting invalid, replayed, or foreign evidence. + pub fn event(&mut self, sender: &str, event: &CloudEvent, now: u64) -> Result<(), RecoveryError> { + let workload = self + .config + .workloads + .iter() + .find(|item| item.agent_id == sender && item.source == event.source) + .ok_or_else(|| RecoveryError::Invalid("lifecycle sender/source is not allowlisted".into()))?; + let (kind, evidence) = event_evidence(event, &self.config.job_id)?; + let state = self + .states + .get_mut(&workload.deployment) + .ok_or_else(|| RecoveryError::Invalid("workload state is absent".into()))?; + state.accept_evidence(kind, &evidence, &mut self.incidents, now)?; + if let EventKind::Checkpoint(kind) = kind { + self.checkpoints.push(CheckpointMeasurement { + workload: workload.deployment.clone(), + kind, + instance: state.instance.clone(), + observed_at: now, + progress: evidence.progress, + bytes: evidence.bytes_count, + duration_millis: evidence.duration_millis, + }); + } + Ok(()) + } + + /// Observe one workload at monotonic elapsed seconds. Readiness alone never closes a detected incident. + pub fn observe(&mut self, workload: &Workload, health: Health, now: u64) -> Result { + let state = self + .states + .get_mut(&workload.deployment) + .ok_or_else(|| RecoveryError::Invalid("workload is not configured".into()))?; + if health == Health::Unknown { + self.unknown_observations = self.unknown_observations.saturating_add(1); + return Ok(Decision::Observe); + } + // An idle module is not stalled, and completing a job does not create another work obligation. + let unhealthy = state.unhealthy(health, &self.config.policy, now); + if matches!(state.phase, Phase::Healthy) && unhealthy { + let index = self.incidents.len(); + self.incidents.push(Incident { + workload: workload.deployment.clone(), + onset: now, + detected: ObservationTime::Pending, + recovered: ObservationTime::Pending, + baseline: state.progress, + status: Status::Suspected, + actions: Vec::new(), + }); + state.phase = Phase::Incident(index); + } + let Phase::Incident(index) = state.phase else { + return Ok(Decision::Observe); + }; + let incident = self + .incidents + .get_mut(index) + .ok_or_else(|| RecoveryError::Invalid("incident state is absent".into()))?; + state.advance(incident, health, unhealthy, &self.config.policy, now); + let attempts = self + .budget + .restarts + .get(&workload.deployment) + .copied() + .unwrap_or_default(); + Ok(state.decision(incident, &self.config, attempts, now)) + } + + /// Reserve an action before persisting the budget and executing the backend operation. + pub fn reserve_restart(&mut self, workload: &Workload, now: u64) -> Result<(), RecoveryError> { + let state = self + .states + .get_mut(&workload.deployment) + .ok_or_else(|| RecoveryError::Invalid("restart target is not configured".into()))?; + let Phase::Incident(index) = state.phase else { + return Err(RecoveryError::Invalid("no incident permits this restart".into())); + }; + let count = self.budget.restarts.entry(workload.deployment.clone()).or_default(); + let incident = self + .incidents + .get_mut(index) + .ok_or_else(|| RecoveryError::Invalid("incident state is absent".into()))?; + if self.config.mode != Mode::ControllerAssisted + || *count >= self.config.policy.max_restarts + || now.saturating_sub(state.last_action) < self.config.policy.restart_after_seconds + || incident.status != Status::Detected + { + return Err(RecoveryError::Invalid( + "restart is not eligible under the configured policy".into(), + )); + } + *count = count.saturating_add(1); + state.last_action = now; + incident.actions.push(Action::RestartReserved); + Ok(()) + } + + /// Record whether a reserved backend request succeeded; this is not evidence of useful recovery. + pub fn action_result(&mut self, workload: &Workload, outcome: Action) -> Result<(), RecoveryError> { + let state = self + .states + .get(&workload.deployment) + .ok_or_else(|| RecoveryError::Invalid("action target is not configured".into()))?; + let Phase::Incident(index) = state.phase else { + return Err(RecoveryError::Invalid("action has no incident".into())); + }; + self.incidents + .get_mut(index) + .ok_or_else(|| RecoveryError::Invalid("incident state is absent".into()))? + .actions + .push(outcome); + Ok(()) + } + + /// Serialize evidence with explicit unavailable measurements rather than zero-duration successful recoveries. + #[must_use] + pub fn report(&self) -> Report { + let durations: Vec<_> = self + .incidents + .iter() + .filter_map(|item| match item.recovered { + ObservationTime::Recorded(time) => Some(time.saturating_sub(item.onset)), + ObservationTime::Pending => None, + }) + .collect(); + let detection: Vec<_> = self + .incidents + .iter() + .filter_map(|item| match item.detected { + ObservationTime::Recorded(time) => Some(time.saturating_sub(item.onset)), + ObservationTime::Pending => None, + }) + .collect(); + Report { + schema_version: 1, + job_id: self.config.job_id.clone(), + mode: self.config.mode, + clock_basis: "observer-monotonic-seconds", + incidents: self.incidents.clone(), + unknown_observations: self.unknown_observations, + recovered_incidents: durations.len(), + restart_budget: self.budget.clone(), + mean_failure_detection_seconds: mean_seconds(&detection), + mttr_seconds: mean_seconds(&durations), + detected_incidents: detection.len(), + failure_detection_seconds: detection, + recovery_seconds: durations, + checkpoint_measurements: self.checkpoints.clone(), + } + } +} + +/// Represent missing observations separately from the mean duration of recorded events. +fn mean_seconds(values: &[u64]) -> MeanSeconds { + let total = values.iter().fold(0_u64, |total, value| total.saturating_add(*value)); + // A process cannot practically retain more than u32::MAX incidents; checked_div preserves missing evidence as null. + let mean = Duration::from_secs(total).checked_div(u32::try_from(values.len()).unwrap_or(u32::MAX)); + mean.map_or(MeanSeconds::Unavailable, |duration| { + MeanSeconds::Measured(duration.as_secs_f64()) + }) +} + +/// Decode and validate lifecycle evidence before permitting a policy-state transition. +#[expect(clippy::single_call_fn, reason = "separate wire validation from state transitions")] +fn event_evidence(event: &CloudEvent, job_id: &str) -> Result<(EventKind, Evidence), RecoveryError> { + let evidence: Evidence = serde_json::from_value(event.data.clone())?; + evidence.validate(job_id, &event.id)?; + let kind = event + .event_type + .strip_prefix("et.recovery.") + .ok_or_else(|| RecoveryError::Invalid("not a recovery lifecycle event".into()))?; + Ok((EventKind::parse(kind)?, evidence)) +} + +impl Evidence { + /// Validate message identity independently of lifecycle event-name decoding. + fn validate(&self, job_id: &str, event_id: &str) -> Result<(), RecoveryError> { + if self.schema_version != 1 + || self.job_id != job_id + || self.instance.is_empty() + || self.instance.len() > 128 + || self.sequence == 0 + || event_id.is_empty() + { + return Err(RecoveryError::Invalid("invalid lifecycle identity or job".into())); + } + Ok(()) + } +} + +impl EventKind { + /// Decode only supported recovery lifecycle action names. + #[expect( + clippy::single_call_fn, + reason = "wire event names have a single explicit decoding table" + )] + fn parse(name: &str) -> Result { + match name { + "started" => Ok(Self::Started), + "progress" => Ok(Self::Progress), + "failed" => Ok(Self::Failed), + "completed" => Ok(Self::Completed), + "checkpoint-saved" => Ok(Self::Checkpoint(CheckpointKind::Saved)), + "checkpoint-restored" => Ok(Self::Checkpoint(CheckpointKind::Restored)), + "resume-completed" => Ok(Self::Checkpoint(CheckpointKind::ResumeCompleted)), + _ => Err(RecoveryError::Invalid("unsupported recovery lifecycle event".into())), + } + } +} + +impl State { + /// Adopt a first instance's baseline only after validating its sequence and identity. + fn accept_evidence( + &mut self, + kind: EventKind, + evidence: &Evidence, + incidents: &mut [Incident], + now: u64, + ) -> Result<(), RecoveryError> { + self.validate_evidence(kind, evidence)?; + if self.instance.is_empty() + && let Phase::Incident(index) = self.phase + { + incidents + .get_mut(index) + .ok_or_else(|| RecoveryError::Invalid("incident state is absent".into()))? + .baseline = evidence.progress; + } + self.apply_evidence(kind, evidence, now); + Ok(()) + } + + /// Reject retired instances, replayed sequence numbers and unannounced replacement processes. + fn validate_evidence(&self, kind: EventKind, evidence: &Evidence) -> Result<(), RecoveryError> { + let changing_instance = !self.instance.is_empty() && evidence.instance != self.instance; + if self.retired.contains(&evidence.instance) + || (evidence.instance == self.instance && evidence.sequence <= self.sequence) + || (changing_instance && kind != EventKind::Started) + { + return Err(RecoveryError::Invalid( + "stale evidence or instance must announce started first".into(), + )); + } + Ok(()) + } + + /// Update process identity and useful-progress counters after evidence validation succeeds. + fn apply_evidence(&mut self, kind: EventKind, evidence: &Evidence, now: u64) { + if evidence.instance != self.instance { + if self.instance.is_empty() { + self.progress = evidence.progress; + } else { + let _inserted = self.retired.insert(self.instance.clone()); + } + } + self.instance.clone_from(&evidence.instance); + self.sequence = evidence.sequence; + let advances_work = matches!(kind, EventKind::Progress | EventKind::Completed); + if !self.active || (advances_work && evidence.progress > self.progress) { + self.progress_at = now; + } + self.active = true; + if !matches!(kind, EventKind::Checkpoint(_)) { + self.failed = kind == EventKind::Failed; + self.completed = kind == EventKind::Completed; + } + if advances_work { + self.progress = self.progress.max(evidence.progress); + } + } + + /// Distinguish real deployment failures and active-work stalls from idle or completed workloads. + fn unhealthy(&self, health: Health, policy: &crate::config::Policy, now: u64) -> bool { + health == Health::Unready + || self.failed + || (self.active && !self.completed && now.saturating_sub(self.progress_at) >= policy.stall_seconds) + } + + /// Permit a restart only within the configured mode, action budget and cooldown. + fn decision(&self, incident: &Incident, config: &Config, attempts: u32, now: u64) -> Decision { + if incident.status == Status::Detected + && config.mode == Mode::ControllerAssisted + && attempts < config.policy.max_restarts + && now.saturating_sub(self.last_action) >= config.policy.restart_after_seconds + { + Decision::Restart + } else { + Decision::Observe + } + } + + /// Advance an incident through detection, useful recovery or deadline expiry using observer time. + fn advance( + &mut self, + incident: &mut Incident, + health: Health, + unhealthy: bool, + policy: &crate::config::Policy, + now: u64, + ) { + if incident.status == Status::Suspected && !unhealthy { + self.phase = Phase::Healthy; + incident.status = Status::Transient; + return; + } + if incident.status == Status::Suspected { + if now.saturating_sub(incident.onset) < policy.detection_grace_seconds { + return; + } + incident.status = Status::Detected; + incident.detected = ObservationTime::Recorded(now); + self.last_action = now; + } + if now.saturating_sub(incident.onset) >= policy.deadline_seconds { + incident.status = Status::Unrecovered; + } + if health == Health::Ready && !unhealthy && self.progress > incident.baseline { + // Late progress closes the active phase without rewriting a missed recovery deadline as success. + if incident.status != Status::Unrecovered { + incident.recovered = ObservationTime::Recorded(now); + incident.status = Status::Recovered; + } + self.phase = Phase::Healthy; + } + } +} diff --git a/services/recovery-supervisor/src/runtime.rs b/services/recovery-supervisor/src/runtime.rs new file mode 100644 index 00000000..0930282a --- /dev/null +++ b/services/recovery-supervisor/src/runtime.rs @@ -0,0 +1,290 @@ +//! Hub lifecycle transport, durable action budgets, and backend execution. + +use std::time::{Duration, Instant}; + +use edge_toolkit::ws::{ClientMessage, CloudEvent, MessageScope, ServerMessage}; +use et_ws_runner_common::RegisteredSocket; +use fs_err as fs; +use futures_util::{SinkExt as _, StreamExt as _}; +use tokio::sync::mpsc; +use tokio_tungstenite::tungstenite::Message; +use tracing::{info, warn}; + +use crate::RecoveryError; +use crate::config::{Config, Workload}; +use crate::health::Session; +use crate::kubernetes::Backend; +use crate::persistence::StateStore; +use crate::policy::{Action, Budget, Decision, Health, Supervisor}; + +/// Restore the durable budget or create an empty budget only when no prior document exists. +pub fn load_budget(config: &Config) -> Result { + match fs::read(&config.state_file) { + Ok(bytes) => { + let document: serde_json::Value = serde_json::from_slice(&bytes)?; + let budget = document + .get("restart_budget") + .ok_or_else(|| RecoveryError::Invalid("state document has no restart budget".into()))?; + Ok(serde_json::from_value(budget.clone())?) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Budget::new(config.job_id.clone())), + Err(error) => Err(error.into()), + } +} + +/// Apply one observation cycle, recording a budget reservation before requesting an external action. +pub async fn cycle( + config: &Config, + store: &StateStore, + supervisor: &mut Supervisor, + backend: &mut B, + now: u64, +) -> Result<(), RecoveryError> +where + B: Backend, +{ + for (workload, health) in observations(config, backend).await { + apply_observation(store, supervisor, backend, &workload, health, now).await?; + } + store.persist(supervisor.report()).await +} + +/// Collect backend observations without borrowing application policy state. +async fn observations(config: &Config, backend: &mut B) -> Vec<(Workload, Health)> +where + B: Backend, +{ + let mut observations = Vec::new(); + for workload in &config.workloads { + let health = match backend.health(workload).await { + Ok(health) => health, + Err(error) => { + warn!(workload = workload.deployment, %error, "recovery observation unavailable"); + Health::Unknown + } + }; + observations.push((workload.clone(), health)); + } + observations +} + +/// Apply current application evidence to an infrastructure observation, reserving budget before any restart. +async fn apply_observation( + store: &StateStore, + supervisor: &mut Supervisor, + backend: &mut B, + workload: &Workload, + health: Health, + now: u64, +) -> Result<(), RecoveryError> +where + B: Backend, +{ + if supervisor.observe(workload, health, now)? != Decision::Restart { + return Ok(()); + } + supervisor.reserve_restart(workload, now)?; + store.persist(supervisor.report()).await?; + let outcome = match backend.restart(workload).await { + Ok(()) => Action::RestartRequested, + Err(error) => Action::RestartFailed { + error: error.to_string(), + }, + }; + warn!(workload = workload.deployment, ?outcome, "recovery action"); + supervisor.action_result(workload, outcome) +} + +/// Run hub transport and supervision concurrently so backend operations never block WebSocket heartbeats. +#[expect( + clippy::integer_division_remainder_used, + reason = "tokio::select! expands to remainder operations" +)] +pub async fn run(config: Config, mut backend: B) -> Result<(), RecoveryError> +where + B: Backend, +{ + let session = Session::acquire(&config)?; + #[cfg(unix)] + let termination = { + let mut signals = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + async move { + let _signal = signals.recv().await; + } + }; + #[cfg(not(unix))] + let termination = std::future::pending::<()>(); + let store = StateStore::new(config.state_file.clone()); + let mut supervisor = Supervisor::new(config.clone(), load_budget(&config)?)?; + // Fail closed on an unwritable budget path before registering or inspecting deployments. + store.persist(supervisor.report()).await?; + let socket = connect(&config).await?; + let (sender, receiver) = mpsc::channel::<(String, CloudEvent)>(256); + let started = Instant::now(); + let outcome = tokio::select! { + result = hub(socket, sender) => result, + result = control(&config, &mut supervisor, &mut backend, &store, &session, receiver, started) => result, + signal = tokio::signal::ctrl_c() => { signal?; Ok(()) }, + () = termination => Ok(()), + }; + store.persist(supervisor.report()).await?; + outcome +} + +/// Register the configured supervisor identity before accepting lifecycle evidence. +#[expect( + clippy::single_call_fn, + reason = "registration validates hub identity before supervision can begin" +)] +async fn connect(config: &Config) -> Result { + let (socket, identity, _status) = et_ws_runner_common::connect_and_register( + &config.ws_url, + Some(config.agent_id.clone()), + Some(Duration::from_secs(30)), + ) + .await?; + if identity != config.agent_id { + return Err(RecoveryError::Invalid( + "hub did not assign the configured supervisor identity".into(), + )); + } + info!(job = config.job_id, mode = ?config.mode, "recovery supervisor registered"); + Ok(socket) +} + +/// Consume evidence and run bounded observation cycles while the hub handles transport independently. +#[expect( + clippy::integer_division_remainder_used, + clippy::single_call_fn, + reason = "named control loop separates shutdown ownership; tokio::select! expands to remainder operations" +)] +async fn control( + config: &Config, + supervisor: &mut Supervisor, + backend: &mut B, + store: &StateStore, + session: &Session, + mut receiver: mpsc::Receiver<(String, CloudEvent)>, + started: Instant, +) -> Result<(), RecoveryError> +where + B: Backend, +{ + let mut interval = tokio::time::interval(Duration::from_secs(config.poll_seconds)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + item = receiver.recv() => { + let (identity, event) = item + .ok_or_else(|| RecoveryError::Invalid("hub event channel closed".into()))?; + accept(supervisor, &identity, &event, started); + } + _ = interval.tick() => { + poll(config, supervisor, backend, store, &mut receiver, started).await?; + session.advance().await?; + } + } + } +} + +/// Apply queued progress before evaluating failures and persist the completed observation cycle. +#[expect( + clippy::single_call_fn, + reason = "one poll keeps observation, queued evidence and persistence in order" +)] +async fn poll( + config: &Config, + supervisor: &mut Supervisor, + backend: &mut B, + store: &StateStore, + receiver: &mut mpsc::Receiver<(String, CloudEvent)>, + started: Instant, +) -> Result<(), RecoveryError> +where + B: Backend, +{ + for (workload, health) in observations(config, backend).await { + // Apply queued progress without letting a continuous producer starve observation and persistence. + for _ in 0..receiver.len() { + if let Ok((identity, event)) = receiver.try_recv() { + accept(supervisor, &identity, &event, started); + } + } + apply_observation( + store, + supervisor, + backend, + &workload, + health, + started.elapsed().as_secs(), + ) + .await?; + } + store.persist(supervisor.report()).await +} + +/// Log rejected lifecycle evidence without stopping supervision of valid workloads. +fn accept(supervisor: &mut Supervisor, identity: &str, event: &CloudEvent, started: Instant) { + if let Err(error) = supervisor.event(identity, event, started.elapsed().as_secs()) { + warn!(%error, "ignored recovery evidence"); + } +} + +/// Receive direct `CloudEvents` through Core's existing acknowledged agent-message transport. +#[expect( + clippy::integer_division_remainder_used, + reason = "tokio::select! expands to remainder operations" +)] +pub async fn hub( + mut socket: RegisteredSocket, + sender: mpsc::Sender<(String, CloudEvent)>, +) -> Result<(), RecoveryError> { + let mut heartbeat = et_ws_runner_common::heartbeat_interval().await; + loop { + tokio::select! { + _ = heartbeat.tick() => socket.send(Message::Ping(Vec::new().into())).await?, + frame = socket.next() => match frame { + Some(Ok(Message::Text(text))) => { + if let Ok(ServerMessage::AgentMessage { from_agent_id, message_id, scope: MessageScope::Direct, + message, .. }) = ServerMessage::from_text_frame(&text) { + if let Ok(event) = serde_json::from_value::(message) { + enqueue(&mut socket, &sender, (from_agent_id, event), &mut heartbeat).await?; + } + let acknowledgement = ClientMessage::MessageAck { message_id }; + socket.send(Message::Text(serde_json::to_string(&acknowledgement)?.into())).await?; + } + } + Some(Ok(Message::Ping(bytes))) => socket.send(Message::Pong(bytes)).await?, + Some(Ok(Message::Close(_))) | None => { + return Err(RecoveryError::Invalid("hub connection closed".into())); + } + Some(Err(error)) => return Err(error.into()), + Some(Ok(_)) => {} + } + } + } +} + +/// Retain pending evidence until queue admission, sending keepalive pings while data reads are backpressured. +#[expect( + clippy::integer_division_remainder_used, + clippy::single_call_fn, + reason = "separate bounded delivery from frame handling; tokio::select! expands to remainder operations" +)] +async fn enqueue( + socket: &mut RegisteredSocket, + sender: &mpsc::Sender<(String, CloudEvent)>, + evidence: (String, CloudEvent), + heartbeat: &mut tokio::time::Interval, +) -> Result<(), RecoveryError> { + let delivery = sender.send(evidence); + tokio::pin!(delivery); + loop { + tokio::select! { + result = &mut delivery => return result.map_err(|error| { + RecoveryError::Invalid(format!("recovery evidence queue unavailable: {error}")) + }), + _ = heartbeat.tick() => socket.send(Message::Ping(Vec::new().into())).await?, + } + } +} diff --git a/services/recovery-supervisor/tests/config.rs b/services/recovery-supervisor/tests/config.rs new file mode 100644 index 00000000..bae0f72d --- /dev/null +++ b/services/recovery-supervisor/tests/config.rs @@ -0,0 +1,177 @@ +//! Configuration and deployment observation reject ambiguous or unsafe targets before any action. +#![cfg(test)] + +use et_recovery_supervisor::config::{Config, Mode, valid_name}; +use et_recovery_supervisor::kubernetes::{Kubernetes, deployment_health}; +use et_recovery_supervisor::policy::{Budget, Health, Supervisor}; +use et_test_recovery::{config, event, supervisor}; +use fs_err as fs; + +/// Typed construction preserves the complete in-cluster configuration without encoding it through JSON. +#[test] +fn typed_in_cluster_settings_match_the_expected_runtime_configuration() { + let mut expected = config(); + expected.context = "in-cluster".into(); + expected.state_file = "/app/state/recovery.json".into(); + let actual = Config::in_cluster( + expected.ws_url.clone(), + expected.namespace.clone(), + expected.job_id.clone(), + expected.policy.clone(), + expected.workloads.clone(), + ); + actual.validate().unwrap(); + assert_eq!( + serde_json::to_value(actual).unwrap(), + serde_json::to_value(expected).unwrap() + ); +} + +/// Validate the default fixture while rejecting mismatched restored job budgets. +#[test] +fn example_is_valid_and_observe_only_by_default() { + let config = config(); + config.validate().unwrap(); + let _backend = Kubernetes::new(config.clone()).unwrap(); + assert_eq!(config.mode, Mode::ObserveOnly); + let mut policy = supervisor(Mode::ObserveOnly); + policy + .event("example-worker", &event("started", 0, 1, "one"), 0) + .unwrap(); + let mut budget = Budget::new("wrong-job".into()); + let _error = Supervisor::new(config.clone(), budget.clone()).unwrap_err(); + budget.job_id.clone_from(&config.job_id); + budget.schema_version = 2; + let _error = Supervisor::new(config, budget).unwrap_err(); +} + +/// Reject unsafe deployment targets, missing identities and duplicate workload bindings. +#[test] +fn targets_and_identities_are_explicit_and_unique() { + for name in [ + "", + "-flag", + "worker/../other", + "worker/other", + "UPPER", + "worker-", + "label=value", + ] { + assert!(!valid_name(name)); + } + assert!(!valid_name(&"a".repeat(64))); + assert!(valid_name("worker-1")); + for field in ["context", "namespace", "agent_id", "job_id", "state_file", "ws_url"] { + let mut value = config_for_serialization(); + value[field] = serde_json::json!(""); + let invalid: et_recovery_supervisor::config::Config = serde_json::from_value(value).unwrap(); + assert!(invalid.validate().is_err(), "{field}"); + } + let mut duplicate = config(); + duplicate.workloads.push(duplicate.workloads[0].clone()); + let _error = duplicate.validate().unwrap_err(); + duplicate.workloads.clear(); + let _error = duplicate.validate().unwrap_err(); + for field in ["deployment", "agent_id", "source"] { + let mut value = config_for_serialization(); + value["workloads"][0][field] = serde_json::json!(""); + let invalid = serde_json::from_value::(value).unwrap(); + assert!(invalid.validate().is_err(), "{field}"); + } +} + +/// Expose a default configuration as JSON for focused invalid-field mutations. +fn config_for_serialization() -> serde_json::Value { + serde_json::to_value(config()).unwrap() +} + +/// Reject malformed connection URLs without forbidding trusted internal plaintext deployments. +#[test] +fn websocket_endpoints_require_a_valid_host_and_no_embedded_credentials() { + // These malformed URLs are parser inputs only; no network connections are established. + for endpoint in [ + "not a url", + "https://127.0.0.1/ws", + "wss://", + "wss://user@hub/ws", + "wss://user:secret@hub/ws", + "wss://hub/ws#fragment", + ] { + let mut value = config(); + value.ws_url = endpoint.into(); + assert!(value.validate().is_err(), "{endpoint}"); + } + for endpoint in [ + "wss://hub.example/ws", + // Intentionally validate plaintext compatibility for internal deployments; this input never opens a + // connection. + // nosemgrep: config.semgrep.no-semgrep-mentions -- external rule annotation + // nosemgrep: javascript.lang.security.detect-insecure-websocket.detect-insecure-websocket + "ws://ws-server:8080/ws", + ] { + let mut value = config(); + value.ws_url = endpoint.into(); + value.validate().unwrap(); + } +} + +/// Reject disabled timing bounds and policies that leave no time for recovery. +#[test] +fn zero_timings_and_impossible_recovery_deadlines_are_rejected() { + for field in [ + "detection_grace_seconds", + "restart_after_seconds", + "stall_seconds", + "deadline_seconds", + ] { + let mut value = config_for_serialization(); + value["policy"][field] = serde_json::json!(0_u64); + assert!( + serde_json::from_value::(value) + .unwrap() + .validate() + .is_err() + ); + } + let mut invalid = config(); + invalid.poll_seconds = 0; + let _error = Kubernetes::new(invalid).unwrap_err(); + let mut invalid = config(); + invalid.policy.deadline_seconds = 2; + let _error = invalid.validate().unwrap_err(); +} + +/// Require a current single-replica deployment rather than stale or incomplete readiness status. +#[test] +fn readiness_requires_current_generation_and_one_available_updated_replica() { + let path = edge_toolkit::config::get_project_root() + .join("services/recovery-supervisor/tests/fixtures/deployment-ready.yaml"); + let mut value: serde_json::Value = serde_yaml::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!( + deployment_health(&serde_json::to_vec(&value).unwrap()).unwrap(), + Health::Ready + ); + for (field, count) in [ + ("replicas", 2_u64), + ("unavailableReplicas", 1), + ("observedGeneration", 1), + ("availableReplicas", 0), + ("updatedReplicas", 0), + ] { + let mut unready = value.clone(); + unready["status"][field] = serde_json::json!(count); + assert_eq!( + deployment_health(&serde_json::to_vec(&unready).unwrap()).unwrap(), + Health::Unready, + "{field}={count} must prevent readiness" + ); + } + let _status = value.as_object_mut().unwrap().remove("status"); + assert_eq!( + deployment_health(&serde_json::to_vec(&value).unwrap()).unwrap(), + Health::Unready + ); + value["spec"]["replicas"] = serde_json::json!(2_u64); + let _error = deployment_health(&serde_json::to_vec(&value).unwrap()).unwrap_err(); + let _error = deployment_health(b"invalid json").unwrap_err(); +} diff --git a/services/recovery-supervisor/tests/environment.rs b/services/recovery-supervisor/tests/environment.rs new file mode 100644 index 00000000..b3a263bc --- /dev/null +++ b/services/recovery-supervisor/tests/environment.rs @@ -0,0 +1,114 @@ +//! The production environment shape accepts explicit workload lists and rejects malformed or unsafe settings. +#![cfg(test)] + +use et_recovery_supervisor::config::{Config, EnvConfig, Mode}; +use et_test_recovery::environment; + +/// Keep recovery opt-in while allowing unrelated service settings in the process environment. +#[test] +fn example_environment_defaults_to_observation_and_ignores_unrelated_variables() { + let mut values = environment(); + let _mode = values.remove("RECOVERY_MODE"); + let _previous = values.insert("UNRELATED_SETTING".into(), "not recovery configuration".into()); + let parsed: EnvConfig = serde_env::from_iter(values).unwrap(); + parsed.recovery.validate().unwrap(); + assert_eq!(parsed.recovery.mode, Mode::ObserveOnly); + assert_eq!(parsed.recovery.policy.stall_seconds, 180); + assert_eq!(parsed.recovery.workloads.len(), 1); + assert_eq!(parsed.recovery.workloads[0].agent_id, "example-worker"); +} + +/// Load multiple sender bindings and explicit intervention settings without changing process-wide state. +#[test] +fn multiple_workloads_and_assisted_mode_deserialize_without_global_environment_changes() { + let mut values = environment(); + let _previous = values.insert("RECOVERY_MODE".into(), "controller-assisted".into()); + let _previous = values.insert( + "RECOVERY_WORKLOADS".into(), + serde_json::json!([ + {"deployment":"worker-1", "agent_id":"client-1", "source":"/modules/client"}, + {"deployment":"worker-2", "agent_id":"client-2", "source":"/modules/client"} + ]) + .to_string(), + ); + let parsed: EnvConfig = serde_env::from_iter(values).unwrap(); + parsed.recovery.validate().unwrap(); + assert_eq!(parsed.recovery.mode, Mode::ControllerAssisted); + assert_eq!(parsed.recovery.workloads.len(), 2); + assert_eq!(parsed.recovery.workloads[1].deployment, "worker-2"); + let encoded = serde_json::to_value(&parsed.recovery).unwrap(); + let restored: Config = serde_json::from_value(encoded).unwrap(); + assert_eq!(restored.workloads[1].agent_id, "client-2"); +} + +/// Reject missing settings, malformed policy values and invalid encoded workload lists. +#[test] +fn malformed_environment_values_and_missing_fields_fail_deserialization() { + for (key, value) in [ + ("RECOVERY_WORKLOADS", "not json"), + ("RECOVERY_WORKLOADS", "{}"), + ("RECOVERY_WORKLOADS", "[{}]"), + ("RECOVERY_WORKLOADS", "[1]"), + ( + "RECOVERY_WORKLOADS", + r#"[{"deployment":"a","agent_id":"b","source":"c","extra":1}]"#, + ), + ("RECOVERY_MODE", "automatic"), + ("RECOVERY_POLL_SECONDS", "abc"), + ("RECOVERY_POLICY_STALL_SECONDS", "-1"), + ("RECOVERY_POLICY_MAX_RESTARTS", "4294967296"), + ] { + let mut values = environment(); + let _previous = values.insert(key.into(), value.into()); + let parsed: Result = serde_env::from_iter(values); + assert!(parsed.is_err(), "{key}: {value}"); + } + for key in ["RECOVERY_JOB_ID", "RECOVERY_WORKLOADS", "RECOVERY_POLICY_STALL_SECONDS"] { + let mut values = environment(); + let _removed = values.remove(key); + let parsed: Result = serde_env::from_iter(values); + assert!(parsed.is_err(), "{key}"); + } + let mut invalid = serde_json::to_value(et_test_recovery::config()).unwrap(); + invalid["workloads"] = serde_json::json!(1_u64); + let _error = serde_json::from_value::(invalid).unwrap_err(); +} + +/// Reject configuration that parses successfully but cannot support safe recovery. +#[test] +fn parsed_environment_still_requires_semantic_validation() { + for (key, value) in [ + ("RECOVERY_WORKLOADS", "[]"), + ("RECOVERY_CONTEXT", "-unsafe"), + ("RECOVERY_POLICY_DEADLINE_SECONDS", "1"), + ("RECOVERY_POLL_SECONDS", "0"), + ] { + let mut values = environment(); + let _previous = values.insert(key.into(), value.into()); + let parsed: EnvConfig = serde_env::from_iter(values).unwrap(); + assert!(parsed.recovery.validate().is_err(), "{key}"); + } +} + +/// Fail startup before writing state when environment configuration is incomplete. +#[test] +fn binary_rejects_malformed_environment_before_creating_state() { + let directory = tempfile::tempdir().unwrap(); + let state = directory.path().join("state.json"); + for (key, value) in [ + ("RECOVERY_WORKLOADS", "not json"), + ("RECOVERY_POLL_SECONDS", "0"), + ("RECOVERY_MODE", "automatic"), + ] { + // Raw output is intentional: rejection before any external action is the outcome under test. + let output = std::process::Command::new(env!("CARGO_BIN_EXE_et-recovery-supervisor")) + .envs(environment()) + .env("RECOVERY_STATE_FILE", &state) + .env(key, value) + .output() + .unwrap(); + assert!(!output.status.success(), "{key}"); + assert!(!output.stderr.is_empty(), "{key}"); + assert!(!state.exists(), "{key}"); + } +} diff --git a/services/recovery-supervisor/tests/fixtures/deployment-ready.yaml b/services/recovery-supervisor/tests/fixtures/deployment-ready.yaml new file mode 100644 index 00000000..4cb0cc24 --- /dev/null +++ b/services/recovery-supervisor/tests/fixtures/deployment-ready.yaml @@ -0,0 +1,9 @@ +metadata: + generation: 2 +spec: + replicas: 1 +status: + observedGeneration: 2 + replicas: 1 + availableReplicas: 1 + updatedReplicas: 1 diff --git a/services/recovery-supervisor/tests/fixtures/kubeconfig.yaml b/services/recovery-supervisor/tests/fixtures/kubeconfig.yaml new file mode 100644 index 00000000..23c52c6f --- /dev/null +++ b/services/recovery-supervisor/tests/fixtures/kubeconfig.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Config +clusters: [] +contexts: [] +users: [] +current-context: "" diff --git a/services/recovery-supervisor/tests/health.rs b/services/recovery-supervisor/tests/health.rs new file mode 100644 index 00000000..19244175 --- /dev/null +++ b/services/recovery-supervisor/tests/health.rs @@ -0,0 +1,407 @@ +//! Probes require a live lease and fresh evidence from the current supervision session, without contacting Kubernetes. +#![cfg(test)] + +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use edge_toolkit::ws::{ConnectStatus, MessageScope, ServerMessage}; +use et_recovery_supervisor::RecoveryError; +use et_recovery_supervisor::config::Config; +use et_recovery_supervisor::health::{Session, check, silence_limit}; +use et_recovery_supervisor::kubernetes::Backend; +use et_recovery_supervisor::policy::Health; +use et_test_recovery::{config, environment}; +use fs_err as fs; +use futures_util::{SinkExt as _, StreamExt as _}; +use tokio_tungstenite::tungstenite::Message; + +/// Point a default configuration at an isolated temporary state file. +fn configured(directory: &Path) -> Config { + let mut config = config(); + config.state_file = directory.join("state.json"); + config +} + +/// Alter one heartbeat field to exercise health validation without changing the durable budget. +fn replace(config: &Config, field: &str, value: serde_json::Value) { + let path = config.state_file.with_extension("health.json"); + let mut evidence: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + evidence[field] = value; + fs::write(path, serde_json::to_vec(&evidence).unwrap()).unwrap(); +} + +/// Run the actual health-check executable with the fixture's state location. +fn probe(config: &Config) -> std::process::Output { + // The unsuccessful exit is the assertion under test, not a command-launch failure. + std::process::Command::new(env!("CARGO_BIN_EXE_et-recovery-healthcheck")) + .envs(environment()) + .env("RECOVERY_STATE_FILE", &config.state_file) + .output() + .unwrap() +} + +/// Require both the library and executable health checks to reject a stopped owner. +fn assert_stopped(config: &Config) { + assert!(check(config).unwrap_err().to_string().contains("not running")); + assert!(!probe(config).status.success()); +} + +/// Bound the wait for a supervisor to finish and publish a healthy observation cycle. +async fn wait_for_health(config: &Config) { + tokio::time::timeout(Duration::from_secs(5), async { + while check(config).is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +/// Keep startup unhealthy until a completed cycle and ensure probing cannot create or consume a restart budget. +#[tokio::test] +async fn startup_waits_for_a_completed_cycle_and_a_read_only_probe_cannot_spend_the_budget() { + let directory = tempfile::tempdir().unwrap(); + let config = configured(directory.path()); + let _missing = check(&config).unwrap_err(); + assert!(!config.state_file.with_extension("lock").exists()); + let session = Session::acquire(&config).unwrap(); + let _starting = check(&config).unwrap_err(); + assert!(!probe(&config).status.success()); + session.advance().await.unwrap(); + check(&config).unwrap(); + assert!(probe(&config).status.success()); + assert!(!config.state_file.exists()); + drop(session); + assert_stopped(&config); +} + +/// Prevent a replacement supervisor from inheriting another process's healthy status. +#[tokio::test] +async fn a_new_owner_cannot_reuse_a_previous_sessions_fresh_heartbeat() { + let directory = tempfile::tempdir().unwrap(); + let config = configured(directory.path()); + let first = Session::acquire(&config).unwrap(); + first.advance().await.unwrap(); + let identity = fs::read(config.state_file.with_extension("session")).unwrap(); + let _already_owned = Session::acquire(&config).unwrap_err(); + assert_eq!(fs::read(config.state_file.with_extension("session")).unwrap(), identity); + check(&config).unwrap(); + drop(first); + let second = Session::acquire(&config).unwrap(); + assert_ne!(fs::read(config.state_file.with_extension("session")).unwrap(), identity); + assert!( + check(&config) + .unwrap_err() + .to_string() + .contains("current recovery session") + ); + second.advance().await.unwrap(); + check(&config).unwrap(); +} + +/// Keep ownership exclusive while allowing a separate health-check process to read session metadata on every OS. +#[tokio::test] +async fn session_identity_remains_readable_without_reading_the_locked_lease() { + let directory = tempfile::tempdir().unwrap(); + let config = configured(directory.path()); + let session = Session::acquire(&config).unwrap(); + let path = config.state_file.with_extension("session"); + let identity = fs::read_to_string(&path).unwrap(); + let _identity = uuid::Uuid::parse_str(&identity).unwrap(); + assert_eq!(fs::metadata(config.state_file.with_extension("lock")).unwrap().len(), 0); + session.advance().await.unwrap(); + assert!(probe(&config).status.success()); + fs::remove_file(&path).unwrap(); + assert!(check(&config).is_err()); + assert!(!probe(&config).status.success()); + fs::write(path, identity).unwrap(); + check(&config).unwrap(); +} + +/// Release ownership if readable session metadata cannot be published, without advertising old evidence as healthy. +#[test] +fn unavailable_session_storage_releases_the_budget_lease() { + let directory = tempfile::tempdir().unwrap(); + let config = configured(directory.path()); + let path = config.state_file.with_extension("session"); + fs::create_dir(&path).unwrap(); + let _unavailable = Session::acquire(&config).unwrap_err(); + assert_stopped(&config); + fs::remove_dir(path).unwrap(); + let _owner = Session::acquire(&config).unwrap(); +} + +/// Reject corrupted heartbeats and evidence belonging to another job, session or schema. +#[tokio::test] +async fn malformed_foreign_and_unknown_version_evidence_fails_closed() { + let directory = tempfile::tempdir().unwrap(); + let config = configured(directory.path()); + let session = Session::acquire(&config).unwrap(); + for (field, value) in [ + ("schema_version", serde_json::json!(2_u32)), + ("job_id", serde_json::json!("other-job")), + ("session", serde_json::json!("other-session")), + ] { + session.advance().await.unwrap(); + replace(&config, field, value); + assert!( + check(&config) + .unwrap_err() + .to_string() + .contains("current recovery session") + ); + } + fs::write(config.state_file.with_extension("health.json"), b"invalid json").unwrap(); + assert!(check(&config).unwrap_err().to_string().contains("JSON")); + session.advance().await.unwrap(); + check(&config).unwrap(); +} + +/// Reject stale or future-dated evidence while allowing a newly completed cycle to restore health. +#[tokio::test] +async fn stale_cycles_and_future_timestamps_are_unhealthy_but_progress_restores_health() { + let directory = tempfile::tempdir().unwrap(); + let config = configured(directory.path()); + let session = Session::acquire(&config).unwrap(); + for (timestamp, message) in [ + (SystemTime::UNIX_EPOCH, "stale"), + (SystemTime::now() + Duration::from_secs(3600), "future"), + ] { + session.advance().await.unwrap(); + replace(&config, "completed_at", serde_json::to_value(timestamp).unwrap()); + assert!(check(&config).unwrap_err().to_string().contains(message)); + assert!(!probe(&config).status.success()); + } + session.advance().await.unwrap(); + check(&config).unwrap(); +} + +/// Keep a failed heartbeat write from advertising a healthy supervision cycle. +#[tokio::test] +async fn unavailable_evidence_storage_does_not_publish_health() { + let directory = tempfile::tempdir().unwrap(); + let config = configured(directory.path()); + let session = Session::acquire(&config).unwrap(); + fs::create_dir_all(config.state_file.with_extension("health.json")).unwrap(); + let _unwritable = session.advance().await.unwrap_err(); + let _missing = check(&config).unwrap_err(); +} + +/// Reject invalid settings and inaccessible lease files before supervision can claim ownership. +#[test] +fn invalid_configuration_and_unavailable_lock_storage_fail_before_startup() { + let directory = tempfile::tempdir().unwrap(); + let mut config = configured(directory.path()); + config.poll_seconds = 0; + let _invalid_owner = Session::acquire(&config).unwrap_err(); + let _invalid_probe = check(&config).unwrap_err(); + config.poll_seconds = 2; + config.state_file = directory.path().join("absent/state.json"); + let _unavailable = Session::acquire(&config).unwrap_err(); +} + +/// Include every workload's bounded commands in the health allowance without integer overflow. +#[test] +fn silence_budget_covers_all_bounded_commands_and_polling_without_overflow() { + let mut config = config(); + assert_eq!(silence_limit(&config), Duration::from_secs(32)); + config.workloads.push(config.workloads[0].clone()); + assert_eq!(silence_limit(&config), Duration::from_secs(52)); + config.command_timeout_seconds = u64::MAX; + assert_eq!(silence_limit(&config), Duration::from_secs(u64::MAX)); +} + +#[derive(Default)] +struct PausedBackend { + observations: u32, + blocked: Arc, + release: Arc, +} + +impl Backend for PausedBackend { + /// Pause an observation on demand or report the configured API-failure condition. + async fn health(&mut self, _workload: &et_recovery_supervisor::config::Workload) -> Result { + self.observations = self.observations.saturating_add(1); + if self.observations == 2 { + self.blocked.notify_one(); + self.release.notified().await; + } + Err(RecoveryError::Invalid("Kubernetes API unavailable".into())) + } + + /// Reject unexpected recovery actions in health-probe tests. + fn restart( + &mut self, + _workload: &et_recovery_supervisor::config::Workload, + ) -> impl Future> + Send { + std::future::ready(Err(RecoveryError::Invalid( + "unknown observations must not trigger restarts".into(), + ))) + } +} + +/// Distinguish a stalled control loop from a completed cycle reporting an unavailable Kubernetes API. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_blocked_loop_cannot_refresh_health_and_api_failure_is_not_a_supervisor_stall() { + let directory = tempfile::tempdir().unwrap(); + let server = et_ws_test_server::start(); + let mut config = configured(directory.path()); + config.ws_url.clone_from(&server.ws_url); + config.poll_seconds = 1; + let blocked = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let backend = PausedBackend { + observations: 0, + blocked: Arc::clone(&blocked), + release: Arc::clone(&release), + }; + let task = tokio::spawn(et_recovery_supervisor::runtime::run(config.clone(), backend)); + tokio::time::timeout(Duration::from_secs(20), blocked.notified()) + .await + .unwrap(); + check(&config).unwrap(); + // Age the completed cycle deterministically while the actual observation loop is blocked. + replace( + &config, + "completed_at", + serde_json::to_value(SystemTime::UNIX_EPOCH).unwrap(), + ); + assert!(check(&config).unwrap_err().to_string().contains("stale")); + release.notify_one(); + wait_for_health(&config).await; + task.abort(); + let _cancelled = task.await; + assert!(check(&config).is_err()); +} + +/// Release health ownership on transport loss instead of trusting a recently written heartbeat. +#[tokio::test] +async fn graceful_and_abrupt_hub_loss_invalidate_health_even_with_a_fresh_heartbeat() { + for graceful in [false, true] { + assert_transport_failure(graceful).await; + } +} + +/// Verify owner shutdown and probe failure after one chosen hub-disconnection mode. +#[expect( + clippy::single_call_fn, + reason = "one transport-failure fixture exercises both connection shutdown modes" +)] +async fn assert_transport_failure(graceful: bool) { + let directory = tempfile::tempdir().unwrap(); + let mut config = configured(directory.path()); + let (url, close, server) = fake_hub("recovery-supervisor", graceful).await; + config.ws_url = url; + let backend = PausedBackend::default(); + let task = tokio::spawn(et_recovery_supervisor::runtime::run(config.clone(), backend)); + wait_for_health(&config).await; + close.notify_one(); + let error = tokio::time::timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains("hub connection")); + server.await.unwrap(); + assert_stopped(&config); +} + +/// Serve a loopback protocol peer with controllable registration and shutdown behavior. +async fn fake_hub(identity: &str, graceful: bool) -> (String, Arc, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + // Intentionally use plaintext on loopback only to test registration and shutdown, not TLS or production traffic. + // nosemgrep: config.semgrep.no-semgrep-mentions -- external rule annotation + // nosemgrep: javascript.lang.security.detect-insecure-websocket.detect-insecure-websocket + let url = format!("ws://{}/ws", listener.local_addr().unwrap()); + let identity = identity.to_owned(); + let close = Arc::new(tokio::sync::Notify::new()); + let shutdown = Arc::clone(&close); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + let _connect = socket.next().await.unwrap().unwrap(); + let acknowledgement = ServerMessage::ConnectAck { + agent_id: identity.clone(), + status: ConnectStatus::Assigned, + }; + socket + .send(Message::Text(serde_json::to_string(&acknowledgement).unwrap().into())) + .await + .unwrap(); + if identity == "recovery-supervisor" { + let foreign = ServerMessage::AgentMessage { + message_id: "foreign-evidence".into(), + from_agent_id: "not-allowlisted".into(), + scope: MessageScope::Direct, + server_received_at: String::default(), + message: serde_json::to_value(et_test_recovery::event("started", 0, 1, "foreign")).unwrap(), + }; + socket + .send(Message::Text(serde_json::to_string(&foreign).unwrap().into())) + .await + .unwrap(); + } + shutdown.notified().await; + if graceful { + socket.close(None).await.unwrap(); + } + }); + (url, close, server) +} + +/// Release the lease after failed registration or heartbeat persistence. +#[tokio::test] +async fn registration_and_heartbeat_persistence_failures_never_leave_a_healthy_owner() { + for bad_identity in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let mut config = configured(directory.path()); + let identity = if bad_identity { + "wrong-supervisor" + } else { + "recovery-supervisor" + }; + let (url, close, server) = fake_hub(identity, false).await; + config.ws_url = url; + if !bad_identity { + fs::create_dir_all(config.state_file.with_extension("health.json")).unwrap(); + } + let backend = PausedBackend::default(); + let error = tokio::time::timeout( + Duration::from_secs(5), + et_recovery_supervisor::runtime::run(config.clone(), backend), + ) + .await + .unwrap() + .unwrap_err(); + let expected = if bad_identity { + "configured supervisor identity" + } else { + "state persistence failed" + }; + assert!(error.to_string().contains(expected), "{error}"); + close.notify_one(); + server.await.unwrap(); + assert_stopped(&config); + } +} + +/// Retain queue-failure context when converting evidence-delivery errors. +#[test] +fn failed_evidence_delivery_preserves_the_transport_error() { + for full in [false, true] { + let evidence = ( + "example-worker".into(), + et_test_recovery::event("started", 0, 1, "session"), + ); + let delivery = if full { + tokio::sync::mpsc::error::TrySendError::Full(evidence) + } else { + tokio::sync::mpsc::error::TrySendError::Closed(evidence) + }; + let error = RecoveryError::from(delivery); + assert!(error.to_string().contains("recovery evidence queue unavailable")); + } +} diff --git a/services/recovery-supervisor/tests/hub_frames.rs b/services/recovery-supervisor/tests/hub_frames.rs new file mode 100644 index 00000000..bdaeaef5 --- /dev/null +++ b/services/recovery-supervisor/tests/hub_frames.rs @@ -0,0 +1,167 @@ +//! Exercise control frames and malformed application payloads against the supervisor's real WebSocket loop. +#![cfg(test)] + +use std::time::Duration; + +use edge_toolkit::ws::{ClientMessage, MessageScope, ServerMessage}; +use et_recovery_supervisor::RecoveryError; +use et_recovery_supervisor::config::Mode; +use et_recovery_supervisor::policy::{Health, Status}; +use et_recovery_supervisor::runtime::hub; +use et_test_recovery::{config, event, supervisor}; +use et_ws_runner_common::RegisteredSocket; +use futures_util::{SinkExt as _, StreamExt as _}; +use tokio::io::AsyncWriteExt as _; +use tokio::net::TcpStream; +use tokio_tungstenite::WebSocketStream; +use tokio_tungstenite::tungstenite::Message; + +/// Establish a real loopback transport without requiring a hub registry or Kubernetes. +async fn peer() -> (RegisteredSocket, WebSocketStream) { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + // Intentionally use plaintext on loopback only to exercise WebSocket frames without unrelated TLS + // negotiation. + // nosemgrep: config.semgrep.no-semgrep-mentions -- external rule annotation + // nosemgrep: javascript.lang.security.detect-insecure-websocket.detect-insecure-websocket + let endpoint = format!("ws://{}/ws", listener.local_addr().unwrap()); + let client = tokio::spawn(async move { tokio_tungstenite::connect_async(endpoint).await.unwrap().0 }); + let (stream, _) = listener.accept().await.unwrap(); + let server = tokio_tungstenite::accept_async(stream).await.unwrap(); + (client.await.unwrap(), server) +} + +/// Read one protocol response with a bound so missing acknowledgements cannot hang a test. +async fn receive(server: &mut WebSocketStream) -> Message { + tokio::time::timeout(Duration::from_secs(3), server.next()) + .await + .unwrap() + .unwrap() + .unwrap() +} + +/// Require the transport task to finish with a service error rather than hang or panic. +async fn stopped(task: tokio::task::JoinHandle>) -> RecoveryError { + let outcome = tokio::time::timeout(Duration::from_secs(3), task).await.unwrap(); + outcome.unwrap().unwrap_err() +} + +/// Send a direct message without assuming the bounded evidence queue can accept it immediately. +async fn send_direct(server: &mut WebSocketStream, identity: &str, payload: serde_json::Value) { + let message = ServerMessage::AgentMessage { + from_agent_id: "example-worker".into(), + message_id: identity.into(), + scope: MessageScope::Direct, + message: payload, + server_received_at: String::default(), + }; + server + .send(Message::Text(serde_json::to_string(&message).unwrap().into())) + .await + .unwrap(); +} + +/// Require the acknowledgement for this message rather than an earlier successful delivery. +async fn acknowledged(server: &mut WebSocketStream, identity: &str) { + let Message::Text(response) = receive(server).await else { + panic!("expected a direct-message acknowledgement"); + }; + assert!(matches!( + serde_json::from_str::(&response).unwrap(), + ClientMessage::MessageAck { message_id } if message_id == identity + )); +} + +/// Deliver a direct message and require its own acknowledgement, even if its event payload is invalid. +async fn direct(server: &mut WebSocketStream, identity: &str, payload: serde_json::Value) { + send_direct(server, identity, payload).await; + acknowledged(server, identity).await; +} + +/// Preserve replacement startup and later progress under overload while keeping outbound heartbeats active. +#[tokio::test] +async fn queue_backpressure_preserves_replacement_identity_and_recovery_progress() { + let (client, mut server) = peer().await; + let (sender, mut receiver) = tokio::sync::mpsc::channel(1); + sender + .try_send(("occupied".into(), event("started", 0, 1, "old"))) + .unwrap(); + let task = tokio::spawn(hub(client, sender)); + let mut policy = supervisor(Mode::ObserveOnly); + policy + .event("example-worker", &event("started", 0, 1, "old"), 0) + .unwrap(); + let worker = config().workloads.remove(0); + let _suspected = policy.observe(&worker, Health::Unready, 10).unwrap(); + let _detected = policy.observe(&worker, Health::Unready, 13).unwrap(); + for (identity, kind, progress, sequence) in [ + ("replacement-started", "started", 0, 1), + ("replacement-progress", "progress", 1, 2), + ] { + let payload = serde_json::to_value(event(kind, progress, sequence, "new")).unwrap(); + send_direct(&mut server, identity, payload).await; + } + let heartbeat = tokio::time::timeout(Duration::from_secs(15), server.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(matches!(&heartbeat, Message::Ping(_)), "{heartbeat:?}"); + assert!(!task.is_finished()); + assert_eq!(receiver.try_recv().unwrap().0, "occupied"); + for (identity, now) in [("replacement-started", 14), ("replacement-progress", 15)] { + let (sender, evidence) = tokio::time::timeout(Duration::from_secs(3), receiver.recv()) + .await + .unwrap() + .unwrap(); + policy.event(&sender, &evidence, now).unwrap(); + acknowledged(&mut server, identity).await; + } + let _recovered = policy.observe(&worker, Health::Ready, 16).unwrap(); + assert_eq!(policy.incidents()[0].status, Status::Recovered); + server.close(None).await.unwrap(); + assert!(stopped(task).await.to_string().contains("hub connection closed")); +} + +/// Acknowledge malformed direct events, answer pings and ignore unrelated frames without losing later valid events. +#[tokio::test] +async fn malformed_events_and_non_event_frames_do_not_stop_delivery() { + let (client, mut server) = peer().await; + let (sender, mut receiver) = tokio::sync::mpsc::channel(4); + let task = tokio::spawn(hub(client, sender)); + server.send(Message::Text("not-json".into())).await.unwrap(); + server.send(Message::Pong(vec![1_u8].into())).await.unwrap(); + server.send(Message::Binary(vec![2_u8].into())).await.unwrap(); + direct(&mut server, "invalid", serde_json::json!({"not":"a lifecycle event"})).await; + assert_eq!( + receiver.try_recv().unwrap_err(), + tokio::sync::mpsc::error::TryRecvError::Empty + ); + server.send(Message::Ping(vec![3_u8].into())).await.unwrap(); + assert_eq!(receive(&mut server).await, Message::Pong(vec![3_u8].into())); + direct( + &mut server, + "valid", + serde_json::to_value(et_test_recovery::event("progress", 1, 2, "worker")).unwrap(), + ) + .await; + let (identity, event) = receiver.try_recv().unwrap(); + assert_eq!(identity, "example-worker"); + assert_eq!(event.event_type, "et.recovery.progress"); + server.close(None).await.unwrap(); + let error = stopped(task).await; + assert!(error.to_string().contains("hub connection closed")); +} + +/// Preserve protocol errors instead of presenting a malformed WebSocket frame as a clean shutdown. +#[tokio::test] +async fn invalid_websocket_frames_preserve_the_transport_error() { + let (client, mut server) = peer().await; + let (sender, _receiver) = tokio::sync::mpsc::channel(4); + let task = tokio::spawn(hub(client, sender)); + // Opcode 3 is reserved; sending raw bytes deliberately bypasses tungstenite's valid-frame encoder. + server.get_mut().write_all(&[0x83_u8, 0_u8]).await.unwrap(); + let error = stopped(task).await; + assert!(matches!(error, RecoveryError::Socket(_)), "{error}"); +} diff --git a/services/recovery-supervisor/tests/kubernetes.rs b/services/recovery-supervisor/tests/kubernetes.rs new file mode 100644 index 00000000..adee341c --- /dev/null +++ b/services/recovery-supervisor/tests/kubernetes.rs @@ -0,0 +1,262 @@ +//! Exercise the real kubectl subprocess against a loopback API with isolated credentials and discovery cache. +#![cfg(test)] + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use et_recovery_supervisor::RecoveryError; +use et_recovery_supervisor::kubernetes::{Backend as _, Kubernetes}; +use et_recovery_supervisor::policy::Health; +use et_test_helpers::temp_env; +use fs_err as fs; +use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec, DeploymentStatus}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use serde_json::{Value, json}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tracing::subscriber::set_default; + +/// Timestamp fixture activity without changing the command or request deadlines under investigation. +#[derive(Clone, Copy)] +struct Diagnostics { + started: Instant, +} + +impl Diagnostics { + /// Keep progress in captured test stderr, including observations made before a command times out. + fn record(&self, message: impl std::fmt::Display) { + eprintln!( + "[kubectl-fixture +{:.3}s] {message}", + self.started.elapsed().as_secs_f64() + ); + } + + /// Report the failing command and fixture liveness while retaining the real backend error. + fn command(&self, stage: &str, result: Result, fixture: &tokio::task::JoinHandle<()>) -> T { + let value = result.unwrap_or_else(|error| { + panic!( + "kubectl {stage} failed after {:.3}s; fixture_finished={}: {error}", + self.started.elapsed().as_secs_f64(), + fixture.is_finished() + ); + }); + self.record(format_args!("kubectl {stage} completed")); + value + } +} + +/// Provide just the API discovery and Deployment fields used by kubectl get and rollout restart. +#[expect( + clippy::single_call_fn, + reason = "API response fixtures are separate from HTTP transport handling" +)] +fn response(path: &str) -> Value { + match path { + "/api" => json!({"kind":"APIVersions", "apiVersion":"v1", "versions":["v1"]}), + "/api/v1" => json!({"kind":"APIResourceList", "apiVersion":"v1", "groupVersion":"v1", "resources":[]}), + "/apis" => json!({"kind":"APIGroupList", "apiVersion":"v1", "groups":[{ + "name":"apps", "versions":[{"groupVersion":"apps/v1", "version":"v1"}], + "preferredVersion":{"groupVersion":"apps/v1", "version":"v1"} + }]}), + "/apis/apps/v1" => json!({"kind":"APIResourceList", "apiVersion":"v1", "groupVersion":"apps/v1", + "resources":[{"name":"deployments", "singularName":"deployment", "namespaced":true, + "kind":"Deployment", "verbs":["get","patch"]}] + }), + "/apis/apps/v1/namespaces/example/deployments/example-worker" => deployment(), + _ => panic!("unexpected Kubernetes API path: {path}"), + } +} + +/// Serialize an actual Kubernetes Deployment so discovery and restart tests use the API's real wire schema. +#[expect( + clippy::single_call_fn, + reason = "typed deployment data is separate from API discovery routing" +)] +fn deployment() -> Value { + serde_json::to_value(Deployment { + metadata: ObjectMeta { + name: Some("example-worker".into()), + namespace: Some("example".into()), + generation: Some(1), + ..ObjectMeta::default() + }, + spec: Some(DeploymentSpec { + replicas: Some(1), + ..DeploymentSpec::default() + }), + status: Some(DeploymentStatus { + observed_generation: Some(1), + replicas: Some(1), + available_replicas: Some(1), + updated_replicas: Some(1), + ..DeploymentStatus::default() + }), + }) + .unwrap() +} + +/// Read a complete request, including the rollout patch body, before returning a close-delimited response. +#[expect( + clippy::single_call_fn, + reason = "bounded request parsing is distinct from fixture response selection" +)] +async fn request(stream: &mut TcpStream, diagnostics: Diagnostics) -> String { + let mut bytes = Vec::new(); + let mut chunk = [0_u8; 2048]; + while bytes.len() < 0x0001_0000 { + let count = stream.read(&mut chunk).await.unwrap(); + assert_ne!(count, 0, "Kubernetes request ended before its body"); + bytes.extend_from_slice(&chunk[..count]); + diagnostics.record(format_args!("received {count} bytes; request total={}", bytes.len())); + if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..end]); + // Requests without Content-Length have no body in this fixture. + let length = headers + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or_default(); + if bytes.len() >= end.saturating_add(4).saturating_add(length) { + return String::from_utf8(bytes).unwrap(); + } + } + } + panic!("Kubernetes fixture request exceeded its size bound"); +} + +/// Reply to discovery, query and patch requests while recording the exact kubectl request paths and bodies. +#[expect( + clippy::single_call_fn, + reason = "the asynchronous API fixture must run concurrently with real kubectl" +)] +async fn serve(listener: TcpListener, seen: tokio::sync::mpsc::UnboundedSender, diagnostics: Diagnostics) { + while !seen.is_closed() { + diagnostics.record("waiting for a TCP connection"); + let (mut stream, _) = listener.accept().await.unwrap(); + diagnostics.record("accepted a TCP connection; waiting for the complete request"); + let text = tokio::time::timeout(Duration::from_secs(5), request(&mut stream, diagnostics)) + .await + .unwrap(); + let target = text.split_whitespace().nth(1).unwrap(); + let path = target.split('?').next().unwrap(); + diagnostics.record(format_args!("request complete: {}", text.lines().next().unwrap())); + let body = response(path).to_string(); + let headers = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n", + body.len() + ); + let reply = format!("{headers}Connection: close\r\n\r\n{body}"); + seen.send(text).unwrap(); + stream.write_all(reply.as_bytes()).await.unwrap(); + diagnostics.record("response written; closing the connection"); + } +} + +/// Exercise successful command output, readiness decoding and restart acceptance without a real cluster. +#[test] +fn kubectl_queries_and_restarts_only_the_configured_namespace_and_deployment() { + let _logging = set_default(tracing_subscriber::fmt().with_test_writer().finish()); + let diagnostics = Diagnostics { + started: Instant::now(), + }; + let directory = tempfile::tempdir().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let listener = runtime + .block_on(TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))) + .unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + diagnostics.record(format_args!("API listener bound at {endpoint}; request deadline=5s")); + let kubeconfig = kubeconfig(directory.path(), &endpoint); + let cache = directory.path().join("cache"); + temp_env::with_vars( + [ + ("KUBECONFIG", Some(kubeconfig.as_os_str())), + ("KUBECACHEDIR", Some(cache.as_os_str())), + ], + || { + runtime.block_on(async { + let (seen, received) = tokio::sync::mpsc::unbounded_channel(); + let task = tokio::spawn(serve(listener, seen, diagnostics)); + let mut config = et_test_recovery::config(); + config.command_timeout_seconds = 60; + diagnostics.record(format_args!( + "starting kubectl health command; deadline={}s", + config.command_timeout_seconds + )); + let workload = config.workloads[0].clone(); + let mut backend = Kubernetes::new(config).unwrap(); + assert_eq!( + diagnostics.command("health", backend.health(&workload).await, &task), + Health::Ready + ); + diagnostics.record("starting kubectl restart command"); + diagnostics.command("restart", backend.restart(&workload).await, &task); + assert_restart_request(received); + task.abort(); + assert!( + task.await.unwrap_err().is_cancelled(), + "API fixture panicked before teardown" + ); + }); + }, + ); +} + +/// Verify that the recorded restart patch targets only the configured namespaced deployment. +#[expect( + clippy::single_call_fn, + reason = "request assertions are distinct from command execution and fixture setup" +)] +fn assert_restart_request(mut received: tokio::sync::mpsc::UnboundedReceiver) { + let mut patch = String::default(); + while let Ok(text) = received.try_recv() { + if text.starts_with("PATCH ") { + patch = text; + } + } + assert!(patch.starts_with("PATCH /apis/apps/v1/namespaces/example/deployments/example-worker")); + assert!(patch.contains("kubectl.kubernetes.io/restartedAt")); +} + +/// Keep credentials isolated while making kubeconfig preparation distinct from the timed command exchange. +#[expect( + clippy::single_call_fn, + reason = "fixture file preparation is independent of runtime diagnostics" +)] +fn kubeconfig(directory: &Path, endpoint: &str) -> PathBuf { + let path = directory.join("kubeconfig.json"); + let document = json!({"apiVersion":"v1", "kind":"Config", "current-context":"k3d-example", + "clusters":[{"name":"fixture", "cluster":{"server":endpoint}}], + "contexts":[{"name":"k3d-example", "context":{"cluster":"fixture", "user":"fixture"}}], + "users":[{"name":"fixture", "user":{}}] + }); + fs::write(&path, serde_json::to_vec(&document).unwrap()).unwrap(); + path +} + +/// Verify the diagnostic failure path locally without claiming to reproduce the Windows timeout. +#[tokio::test] +#[should_panic(expected = "fixture_finished=false: diagnostic probe")] +async fn command_failure_reports_the_operation_and_live_fixture() { + let diagnostics = Diagnostics { + started: Instant::now(), + }; + diagnostics.record("fixture waiting for its first TCP connection"); + let task = tokio::spawn(std::future::pending::<()>()); + diagnostics.command::<()>("health", Err(RecoveryError::Invalid("diagnostic probe".into())), &task); +} + +/// Preserve target-rejection errors through both public backend methods before launching kubectl. +#[tokio::test] +async fn backend_methods_propagate_non_allowlisted_target_errors() { + let mut backend = Kubernetes::new(et_test_recovery::config()).unwrap(); + let mut workload = et_test_recovery::config().workloads.remove(0); + workload.deployment = "not-allowlisted".into(); + let health = backend.health(&workload).await.unwrap_err(); + let restart = backend.restart(&workload).await.unwrap_err(); + assert!(health.to_string().contains("not allowlisted")); + assert!(restart.to_string().contains("not allowlisted")); +} diff --git a/services/recovery-supervisor/tests/persistence.rs b/services/recovery-supervisor/tests/persistence.rs new file mode 100644 index 00000000..974de2cb --- /dev/null +++ b/services/recovery-supervisor/tests/persistence.rs @@ -0,0 +1,92 @@ +//! Ordered blocking writes preserve state on errors and do not race cancelled async callers. +#![cfg(test)] + +use std::sync::Arc; +use std::time::Duration; + +use et_recovery_supervisor::persistence::StateStore; +use fs_err as fs; +use serde::{Serialize, Serializer}; +use tokio::sync::Notify; + +struct PausedDocument { + started: Arc, + release: std::sync::Mutex>, +} + +impl Serialize for PausedDocument { + /// Pause serialization until the test releases the blocking writer. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.started.notify_one(); + let release = match self.release.lock() { + Ok(release) => release, + Err(error) => return Err(serde::ser::Error::custom(error)), + }; + match release.recv_timeout(Duration::from_secs(5)) { + Ok(()) => serializer.serialize_u64(1), + Err(error) => Err(serde::ser::Error::custom(error)), + } + } +} + +/// Keep cancelled callers from allowing older blocking writes to overtake later state commits. +#[tokio::test] +async fn cancellation_cannot_let_an_older_save_overwrite_the_final_report() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("state.json"); + let store = StateStore::new(path.clone()); + let writer = store.clone(); + let started = Arc::new(Notify::new()); + let (release, receiver) = std::sync::mpsc::channel(); + let paused = PausedDocument { + started: Arc::clone(&started), + release: std::sync::Mutex::new(receiver), + }; + let first = tokio::spawn(async move { writer.persist(paused).await }); + // This single-thread runtime can still process notifications while serialization is deliberately blocked. + tokio::time::timeout(Duration::from_secs(3), started.notified()) + .await + .unwrap(); + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let mut final_save = tokio::spawn(async move { store.persist(2_u64).await }); + let _blocked = tokio::time::timeout(Duration::from_millis(20), &mut final_save) + .await + .unwrap_err(); + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), final_save) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(fs::read(&path).unwrap(), b"2"); +} + +struct InvalidDocument; + +impl Serialize for InvalidDocument { + /// Inject a serialization failure before a replacement document can be committed. + fn serialize(&self, _serializer: S) -> Result + where + S: Serializer, + { + Err(serde::ser::Error::custom("intentional serialization failure")) + } +} + +/// Preserve committed state after serialization failure and permit a subsequent successful save. +#[tokio::test] +async fn failed_serialization_preserves_the_last_document_and_releases_the_writer() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("state.json"); + let store = StateStore::new(path.clone()); + store.persist(1_u64).await.unwrap(); + let error = store.persist(InvalidDocument).await.unwrap_err(); + assert!(error.to_string().contains("intentional serialization failure")); + assert_eq!(fs::read(&path).unwrap(), b"1"); + store.persist(2_u64).await.unwrap(); + assert_eq!(fs::read(&path).unwrap(), b"2"); +} diff --git a/services/recovery-supervisor/tests/policy.rs b/services/recovery-supervisor/tests/policy.rs new file mode 100644 index 00000000..76974af5 --- /dev/null +++ b/services/recovery-supervisor/tests/policy.rs @@ -0,0 +1,335 @@ +//! Recovery policy verifies useful work, enforces action bounds, and rejects stale or foreign evidence. +#![cfg(test)] + +use et_recovery_supervisor::config::Mode; +use et_recovery_supervisor::policy::{Action, Decision, Health, ObservationTime, Status, Supervisor}; +use et_test_recovery::{config, event, report, supervisor}; + +/// Require renewed useful progress rather than deployment readiness alone to close an incident. +#[test] +fn readiness_does_not_establish_recovery_but_progress_does() { + let mut supervisor = supervisor(Mode::ObserveOnly); + let workload = config().workloads.remove(0); + supervisor + .event("example-worker", &event("started", 0, 1, "first"), 0) + .unwrap(); + supervisor + .event("example-worker", &event("progress", 1, 2, "first"), 1) + .unwrap(); + assert_eq!( + supervisor.observe(&workload, Health::Unready, 10).unwrap(), + Decision::Observe + ); + assert_eq!( + supervisor.observe(&workload, Health::Unready, 13).unwrap(), + Decision::Observe + ); + assert_eq!(supervisor.incidents()[0].status, Status::Detected); + assert_eq!(supervisor.incidents()[0].detected, ObservationTime::Recorded(13)); + let _decision = supervisor.observe(&workload, Health::Ready, 14).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Detected); + supervisor + .event("example-worker", &event("started", 1, 1, "replacement"), 15) + .unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 16).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Detected); + supervisor + .event("example-worker", &event("checkpoint-restored", 1, 2, "replacement"), 16) + .unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 17).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Detected); + supervisor + .event("example-worker", &event("progress", 2, 3, "replacement"), 18) + .unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 19).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Recovered); + assert_eq!(report(&supervisor)["recovery_seconds"], serde_json::json!([9_u64])); + assert_eq!(report(&supervisor)["mttr_seconds"].as_f64(), Some(9.0_f64)); +} + +/// Keep unavailable infrastructure observations separate from failures and verified recoveries. +#[test] +fn unknown_observations_do_not_trigger_actions_or_fake_recovery() { + let mut supervisor = supervisor(Mode::ControllerAssisted); + let workload = config().workloads.remove(0); + assert_eq!( + supervisor.observe(&workload, Health::Unknown, 0).unwrap(), + Decision::Observe + ); + assert!(supervisor.incidents().is_empty()); + let _decision = supervisor.observe(&workload, Health::Unready, 1).unwrap(); + let _decision = supervisor.observe(&workload, Health::Unready, 4).unwrap(); + assert_eq!( + supervisor.observe(&workload, Health::Unknown, 100).unwrap(), + Decision::Observe + ); + assert_eq!(supervisor.report().unknown_observations, 2_u64); + assert_eq!(supervisor.incidents()[0].status, Status::Detected); +} + +/// Use the workload's first committed progress counter as the incident baseline. +#[test] +fn initial_committed_progress_becomes_the_failure_baseline() { + let mut policy = supervisor(Mode::ObserveOnly); + let worker = config().workloads.remove(0); + policy + .event("example-worker", &event("started", 7, 1, "restored"), 0) + .unwrap(); + let _first = policy.observe(&worker, Health::Unready, 10).unwrap(); + let _detected = policy.observe(&worker, Health::Unready, 13).unwrap(); + policy + .event("example-worker", &event("progress", 7, 2, "restored"), 14) + .unwrap(); + let _unchanged = policy.observe(&worker, Health::Ready, 15).unwrap(); + assert_eq!(policy.incidents()[0].baseline, 7); + assert_eq!(policy.incidents()[0].status, Status::Detected); + policy + .event("example-worker", &event("progress", 8, 3, "restored"), 16) + .unwrap(); + let _resumed = policy.observe(&worker, Health::Ready, 17).unwrap(); + assert_eq!(policy.incidents()[0].status, Status::Recovered); +} + +/// Enforce intervention delay and durable restart limits across observer replacement. +#[test] +fn assisted_restarts_are_delayed_bounded_and_survive_supervisor_replacement() { + let mut supervisor = supervisor(Mode::ControllerAssisted); + let workload = config().workloads.remove(0); + let _error = supervisor.reserve_restart(&workload, 0).unwrap_err(); + let _decision = supervisor.observe(&workload, Health::Unready, 0).unwrap(); + let _decision = supervisor.observe(&workload, Health::Unready, 3).unwrap(); + let _error = supervisor.reserve_restart(&workload, 4).unwrap_err(); + assert_eq!( + supervisor.observe(&workload, Health::Unready, 22).unwrap(), + Decision::Observe + ); + for now in [23, 43] { + assert_eq!( + supervisor.observe(&workload, Health::Unready, now).unwrap(), + Decision::Restart + ); + supervisor.reserve_restart(&workload, now).unwrap(); + supervisor.action_result(&workload, Action::RestartRequested).unwrap(); + } + assert_eq!( + supervisor.observe(&workload, Health::Unready, 63).unwrap(), + Decision::Observe + ); + let _error = supervisor.reserve_restart(&workload, 63).unwrap_err(); + let mut restored_config = config(); + restored_config.mode = Mode::ControllerAssisted; + let mut restored = Supervisor::new(restored_config, supervisor.budget().clone()).unwrap(); + let _decision = restored.observe(&workload, Health::Unready, 0).unwrap(); + let _decision = restored.observe(&workload, Health::Unready, 3).unwrap(); + assert_eq!( + restored.observe(&workload, Health::Unready, 23).unwrap(), + Decision::Observe + ); +} + +/// Detect active-work stalls without treating idle or completed applications as stalled. +#[test] +fn stalled_work_can_fail_without_a_failed_container_and_completed_work_does_not_stall() { + let mut supervisor = supervisor(Mode::ObserveOnly); + let workload = config().workloads.remove(0); + let _decision = supervisor.observe(&workload, Health::Ready, 300).unwrap(); + assert!(supervisor.incidents().is_empty()); + supervisor + .event("example-worker", &event("started", 0, 1, "one"), 300) + .unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 480).unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 483).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Detected); + supervisor + .event("example-worker", &event("completed", 3, 2, "one"), 490) + .unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 491).unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 1000).unwrap(); + assert_eq!(supervisor.incidents().len(), 1); +} + +/// Keep missed recovery deadlines distinct from successful or instantaneous recoveries. +#[test] +fn expired_recovery_is_not_a_zero_duration_success() { + let mut supervisor = supervisor(Mode::ControllerAssisted); + let workload = config().workloads.remove(0); + let _decision = supervisor.observe(&workload, Health::Unready, 0).unwrap(); + let _decision = supervisor.observe(&workload, Health::Unready, 3).unwrap(); + assert_eq!( + supervisor.observe(&workload, Health::Unready, 240).unwrap(), + Decision::Observe + ); + assert_eq!(supervisor.incidents()[0].status, Status::Unrecovered); + let _error = supervisor.reserve_restart(&workload, 260).unwrap_err(); + supervisor + .event("example-worker", &event("started", 0, 1, "late"), 260) + .unwrap(); + supervisor + .event("example-worker", &event("progress", 1, 2, "late"), 261) + .unwrap(); + let _late = supervisor.observe(&workload, Health::Ready, 262).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Unrecovered); + assert_eq!(supervisor.incidents()[0].recovered, ObservationTime::Pending); + assert_eq!(supervisor.report().recovered_incidents, 0); + assert_eq!(report(&supervisor)["mttr_seconds"], serde_json::Value::Null); + assert_eq!(supervisor.report().recovery_seconds, Vec::::new()); + let _new_failure = supervisor.observe(&workload, Health::Unready, 263).unwrap(); + assert_eq!(supervisor.incidents().len(), 2); + assert_eq!(supervisor.incidents()[1].onset, 263); +} + +/// Apply the recovery deadline even when the first post-deadline observation already contains useful progress. +#[test] +fn recovery_is_verified_only_before_the_deadline() { + for (now, expected) in [ + (239, Status::Recovered), + (240, Status::Unrecovered), + (241, Status::Unrecovered), + ] { + let mut supervisor = supervisor(Mode::ObserveOnly); + let workload = config().workloads.remove(0); + supervisor + .event("example-worker", &event("started", 0, 1, "worker"), 0) + .unwrap(); + let _suspected = supervisor.observe(&workload, Health::Unready, 0).unwrap(); + let _detected = supervisor.observe(&workload, Health::Unready, 3).unwrap(); + supervisor + .event("example-worker", &event("progress", 1, 2, "worker"), now) + .unwrap(); + let _ready = supervisor.observe(&workload, Health::Ready, now).unwrap(); + assert_eq!(supervisor.incidents()[0].status, expected, "observer time {now}"); + assert_eq!( + supervisor.report().recovered_incidents, + usize::from(expected == Status::Recovered) + ); + } +} + +/// Reject foreign senders and stale process evidence before they can alter recovery state. +#[test] +fn foreign_replayed_and_out_of_order_events_cannot_override_current_evidence() { + let mut supervisor = supervisor(Mode::ObserveOnly); + let start = event("started", 0, 1, "one"); + let _error = supervisor.event("foreign-worker", &start, 0).unwrap_err(); + for (field, value) in [ + ("job_id", serde_json::json!("foreign")), + ("schema_version", serde_json::json!(2_u64)), + ("instance", serde_json::json!("")), + ("sequence", serde_json::json!(0_u64)), + ] { + let mut invalid = start.clone(); + invalid.data[field] = value; + let _error = supervisor.event("example-worker", &invalid, 0).unwrap_err(); + } + for kind in ["other", "et.recovery.unknown"] { + let mut invalid = start.clone(); + invalid.event_type = kind.into(); + let _error = supervisor.event("example-worker", &invalid, 0).unwrap_err(); + } + supervisor.event("example-worker", &start, 0).unwrap(); + let _error = supervisor + .event("example-worker", &event("progress", 1, 2, "unannounced"), 0) + .unwrap_err(); + let _error = supervisor.event("example-worker", &start, 1).unwrap_err(); + supervisor + .event("example-worker", &event("started", 0, 1, "two"), 1) + .unwrap(); + assert!( + supervisor + .event("example-worker", &event("progress", 50, 2, "one"), 2) + .is_err() + ); + supervisor + .event("example-worker", &event("failed", 0, 2, "two"), 2) + .unwrap(); + let workload = config().workloads.remove(0); + let _decision = supervisor.observe(&workload, Health::Ready, 2).unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 5).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Detected); +} + +/// Filter short readiness disruptions without consuming recovery actions. +#[test] +fn transient_unreadiness_below_grace_does_not_trigger_recovery() { + let mut supervisor = supervisor(Mode::ObserveOnly); + let mut workload = config().workloads.remove(0); + let _decision = supervisor.observe(&workload, Health::Unready, 0).unwrap(); + let _decision = supervisor.observe(&workload, Health::Ready, 1).unwrap(); + assert_eq!(supervisor.incidents()[0].status, Status::Transient); + let _error = supervisor + .action_result(&workload, Action::RestartRequested) + .unwrap_err(); + workload.deployment = "not-allowlisted".into(); + let _error = supervisor.observe(&workload, Health::Ready, 2).unwrap_err(); + let _error = supervisor.reserve_restart(&workload, 2).unwrap_err(); + let _error = supervisor + .action_result(&workload, Action::RestartRequested) + .unwrap_err(); +} + +/// Treat an adopted process's existing progress as a baseline rather than new recovery evidence. +#[test] +fn adopting_a_running_instance_does_not_count_old_work_as_recovery() { + let mut observer = supervisor(Mode::ObserveOnly); + let target = config().workloads.remove(0); + let _suspected = observer.observe(&target, Health::Unready, 0).unwrap(); + let _detected = observer.observe(&target, Health::Unready, 3).unwrap(); + observer + .event("example-worker", &event("progress", 7, 100, "existing"), 4) + .unwrap(); + let _ready = observer.observe(&target, Health::Ready, 5).unwrap(); + assert_eq!(observer.incidents()[0].baseline, 7); + assert_eq!(observer.incidents()[0].status, Status::Detected); + observer + .event("example-worker", &event("progress", 8, 101, "existing"), 6) + .unwrap(); + let _recovered = observer.observe(&target, Health::Ready, 7).unwrap(); + assert_eq!(observer.incidents()[0].status, Status::Recovered); +} + +/// Preserve checkpoint measurements and distinguish unobserved timestamps from observer second zero. +#[test] +fn report_preserves_typed_checkpoint_measurements_and_distinguishes_pending_times() { + use et_recovery_supervisor::policy::CheckpointKind; + + let mut policy = supervisor(Mode::ObserveOnly); + let target = config().workloads.remove(0); + policy + .event("example-worker", &event("started", 0, 1, "one"), 0) + .unwrap(); + let _decision = policy.observe(&target, Health::Unready, 0).unwrap(); + assert_eq!(policy.incidents()[0].detected, ObservationTime::Pending); + assert_eq!(policy.incidents()[0].recovered, ObservationTime::Pending); + assert_eq!(report(&policy)["incidents"][0]["detected"], serde_json::Value::Null); + for (sequence, kind, expected) in [ + (2, "checkpoint-saved", CheckpointKind::Saved), + (3, "checkpoint-restored", CheckpointKind::Restored), + (4, "resume-completed", CheckpointKind::ResumeCompleted), + ] { + let mut measurement = event(kind, 0, sequence, "one"); + measurement.data["bytes_count"] = serde_json::json!(1024_u64); + measurement.data["duration_millis"] = serde_json::json!(12_u64); + policy.event("example-worker", &measurement, sequence).unwrap(); + let snapshot = policy.report(); + let latest = snapshot.checkpoint_measurements.last().unwrap(); + assert_eq!(latest.kind, expected); + assert_eq!(latest.bytes, 1024); + assert_eq!(latest.duration_millis, 12); + assert_eq!( + report(&policy)["checkpoint_measurements"] + .as_array() + .unwrap() + .last() + .unwrap()["kind"], + kind + ); + } + for time in [ + ObservationTime::Pending, + ObservationTime::Recorded(0), + ObservationTime::Recorded(7), + ] { + let serialized = serde_json::to_value(time).unwrap(); + assert_eq!(serde_json::from_value::(serialized).unwrap(), time); + } +} diff --git a/services/recovery-supervisor/tests/runtime.rs b/services/recovery-supervisor/tests/runtime.rs new file mode 100644 index 00000000..3ba0bf17 --- /dev/null +++ b/services/recovery-supervisor/tests/runtime.rs @@ -0,0 +1,509 @@ +//! Recovery actions are backend-neutral, persisted before execution, and carried over the real hub transport. +#![cfg(test)] + +use std::time::Duration; + +use edge_toolkit::ws::{ClientMessage, ServerMessage}; +use et_recovery_supervisor::RecoveryError; +use et_recovery_supervisor::config::{Mode, Workload}; +use et_recovery_supervisor::kubernetes::{Backend, Kubernetes}; +use et_recovery_supervisor::persistence::StateStore; +use et_recovery_supervisor::policy::{Action, Health, Supervisor}; +use et_recovery_supervisor::runtime::{cycle, hub, load_budget, run}; +use et_test_recovery::{config, environment, event, report, supervisor}; +use fs_err as fs; +use futures_util::{SinkExt as _, StreamExt as _}; +use tracing::field::Field; +use tracing::instrument::WithSubscriber as _; +use tracing::subscriber::set_default; +use tracing::{Event, Subscriber}; +use tracing_subscriber::layer::SubscriberExt as _; + +/// Observe rejected evidence on the service task without installing a process-wide logging subscriber. +struct RejectedEvidence(tokio::sync::mpsc::UnboundedSender); + +impl tracing_subscriber::Layer for RejectedEvidence { + /// Forward runtime error fields so the test can wait for policy rejection, not just transport acknowledgement. + fn on_event(&self, event: &Event<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>) { + if event.metadata().target() != "et_recovery_supervisor::runtime" { + return; + } + event.record(&mut |field: &Field, value: &dyn std::fmt::Debug| { + if field.name() == "error" { + self.0.send(format!("{value:?}")).unwrap(); + } + }); + } +} + +struct FakeBackend { + config: et_recovery_supervisor::config::Config, + restarts: u32, + unknown: bool, + fail_restart: bool, +} + +impl Backend for FakeBackend { + /// Return a controlled unhealthy or unavailable observation for policy tests. + fn health(&mut self, _workload: &Workload) -> impl Future> + Send { + if self.unknown { + return std::future::ready(Err(RecoveryError::Invalid("API unavailable".into()))); + } + std::future::ready(Ok(Health::Unready)) + } + + /// Verify the durable reservation before recording a simulated restart attempt. + fn restart(&mut self, workload: &Workload) -> impl Future> + Send { + // The observable file must already contain the consumed budget before the backend can mutate anything. + let restored = load_budget(&self.config).unwrap(); + assert_eq!( + restored.restarts.get(&workload.deployment).copied(), + Some(self.restarts.saturating_add(1)) + ); + self.restarts = self.restarts.saturating_add(1); + if self.fail_restart { + return std::future::ready(Err(RecoveryError::Invalid("restart denied".into()))); + } + std::future::ready(Ok(())) + } +} + +/// Persist restart reservations before backend calls and retain attempts even when execution fails. +#[tokio::test] +async fn cycle_persists_budget_before_action_and_retains_failed_attempts() { + let directory = tempfile::tempdir().unwrap(); + let (config, mut supervisor, mut backend) = assisted_fixture(directory.path()); + let store = StateStore::new(config.state_file.clone()); + for observed_at in [0, 3, 23] { + cycle(&config, &store, &mut supervisor, &mut backend, observed_at) + .await + .unwrap(); + } + backend.fail_restart = true; + cycle(&config, &store, &mut supervisor, &mut backend, 43).await.unwrap(); + assert_eq!(backend.restarts, 2); + let saved = load_budget(&config).unwrap(); + assert_eq!(saved.restarts["example-worker"], 2); + assert_eq!( + supervisor.incidents()[0].actions.last(), + Some(&Action::RestartFailed { + error: "restart denied".into() + }) + ); + assert_eq!( + report(&supervisor)["incidents"][0]["actions"][3]["kind"], + "restart-failed" + ); + backend.unknown = true; + cycle(&config, &store, &mut supervisor, &mut backend, 63).await.unwrap(); + assert_eq!(supervisor.report().unknown_observations, 1_u64); + assert_eq!(backend.restarts, 2); +} + +/// Prevent external recovery actions when the durable budget cannot be committed. +#[tokio::test] +async fn persistence_failure_prevents_external_restart() { + let directory = tempfile::tempdir().unwrap(); + let (config, mut supervisor, mut backend) = assisted_fixture(directory.path()); + let store = StateStore::new(config.state_file.clone()); + for observed_at in [0, 3] { + cycle(&config, &store, &mut supervisor, &mut backend, observed_at) + .await + .unwrap(); + } + fs::remove_file(&config.state_file).unwrap(); + fs::create_dir_all(&config.state_file).unwrap(); + let _error = cycle(&config, &store, &mut supervisor, &mut backend, 23) + .await + .unwrap_err(); + assert_eq!(backend.restarts, 0); + let _error = load_budget(&config).unwrap_err(); +} + +/// Reject invalid restored budgets and verify that complete reports replace their predecessors. +#[tokio::test] +async fn malformed_state_fails_closed_and_atomic_updates_replace_previous_documents() { + let directory = tempfile::tempdir().unwrap(); + let mut config = config(); + config.state_file = directory.path().join("state.json"); + let store = StateStore::new(config.state_file.clone()); + let mut supervisor = supervisor(Mode::ObserveOnly); + supervisor + .event("example-worker", &event("started", 0, 1, "one"), 0) + .unwrap(); + for value in [serde_json::json!({}), serde_json::json!({"restart_budget":{}})] { + store.persist(value).await.unwrap(); + let _error = load_budget(&config).unwrap_err(); + } + fs::write(&config.state_file, b"invalid json").unwrap(); + let _error = load_budget(&config).unwrap_err(); + store.persist(supervisor.report()).await.unwrap(); + assert_eq!(load_budget(&config).unwrap().job_id, "example-job"); +} + +/// Preserve sender identity and acknowledge direct lifecycle messages over the real hub. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn hub_delivers_sender_bound_cloudevents_and_acknowledges_receipt() { + let _telemetry = et_otlp::init_guarded(None).unwrap(); + let server = et_ws_test_server::start(); + let (observer, _) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("observer")).await; + let (mut producer, producer_id) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("example-worker")).await; + let (sender, mut receiver) = tokio::sync::mpsc::channel(8); + let task = tokio::spawn(hub(observer, sender)); + let message = ClientMessage::SendAgentMessage { + to_agent_id: "observer".into(), + message: serde_json::to_value(event("started", 0, 1, "one")).unwrap(), + }; + send_message(&mut producer, &message).await; + let (sender_id, evidence) = tokio::time::timeout(Duration::from_secs(3), receiver.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(sender_id, producer_id); + assert_eq!(evidence.event_type, "et.recovery.started"); + assert_acknowledged(&mut producer).await; + task.abort(); +} + +/// Reject undeclared deployments before launching any Kubernetes command. +#[tokio::test] +async fn kubernetes_rejects_non_allowlisted_targets_before_running_a_command() { + let backend = Kubernetes::new(config()).unwrap(); + let mut workload = config().workloads.remove(0); + workload.deployment = "not-allowlisted".into(); + assert!( + backend + .execute(&workload, &["get", "-o", "json"]) + .await + .unwrap_err() + .to_string() + .contains("allowlisted") + ); +} + +struct ReadyBackend; + +struct GatedBackend { + observing: tokio::sync::mpsc::Sender<()>, + release: tokio::sync::mpsc::Receiver<()>, +} + +impl Backend for GatedBackend { + /// Hold infrastructure observation until the test has queued acknowledged application evidence. + async fn health(&mut self, _workload: &Workload) -> Result { + self.observing.send(()).await.unwrap(); + self.release.recv().await.unwrap(); + Ok(Health::Ready) + } + + /// Preserve the healthy fixture's rejection of unexpected recovery actions. + async fn restart(&mut self, workload: &Workload) -> Result<(), RecoveryError> { + ReadyBackend.restart(workload).await + } +} + +/// Retain evidence until capacity returns without requiring the producer to resend it, and fail on consumer loss. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn full_evidence_queue_waits_for_capacity_without_requiring_a_resend() { + let server = et_ws_test_server::start(); + let (observer, _) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("observer")).await; + let (mut producer, _) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("example-worker")).await; + let (sender, mut receiver) = tokio::sync::mpsc::channel(1); + sender + .try_send(("occupied".into(), event("started", 0, 1, "one"))) + .unwrap(); + let task = tokio::spawn(hub(observer, sender)); + let message = ClientMessage::SendAgentMessage { + to_agent_id: "observer".into(), + message: serde_json::to_value(event("progress", 1, 2, "one")).unwrap(), + }; + send_message(&mut producer, &message).await; + assert!(!task.is_finished()); + assert_eq!(receiver.try_recv().unwrap().0, "occupied"); + assert_acknowledged(&mut producer).await; + let (identity, evidence) = receiver.try_recv().unwrap(); + assert_eq!(identity, "example-worker"); + assert_eq!(evidence.event_type, "et.recovery.progress"); + drop(receiver); + send_message(&mut producer, &message).await; + let error = tokio::time::timeout(Duration::from_secs(3), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains("recovery evidence queue unavailable")); +} + +/// Exercise real kubectl context errors using an isolated fixture without contacting a cluster. +#[tokio::test] +async fn kubernetes_reports_command_errors_without_contacting_a_cluster() { + let _logging = set_default(tracing_subscriber::fmt().with_test_writer().finish()); + let mut config = config(); + config.command_timeout_seconds = 60; + let workload = config.workloads[0].clone(); + let backend = Kubernetes::new(config).unwrap(); + let path = + edge_toolkit::config::get_project_root().join("services/recovery-supervisor/tests/fixtures/kubeconfig.yaml"); + let fixture = path.to_str().unwrap(); + let error = backend + .execute(&workload, &["--kubeconfig", fixture, "get", "-o", "json"]) + .await + .unwrap_err(); + assert!( + matches!(error, RecoveryError::Invalid(_)), + "expected a kubectl context error, not a process startup or timeout failure: {error}" + ); + assert!(error.to_string().contains("kubectl")); + assert!(error.to_string().contains("context")); +} + +/// Reject an unusable budget directory before creating state or supervising workloads. +#[test] +fn binary_fails_closed_on_an_unavailable_state_directory() { + let directory = tempfile::tempdir().unwrap(); + let mut config = config(); + config.state_file = directory.path().join("absent").join("state.json"); + // Raw output is intentional: an unsuccessful exit is the outcome under test. + let output = std::process::Command::new(env!("CARGO_BIN_EXE_et-recovery-supervisor")) + .envs(environment()) + .env("RECOVERY_STATE_FILE", &config.state_file) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8(output.stderr).unwrap().contains("Io(")); + assert!(!config.state_file.exists()); +} + +impl Backend for ReadyBackend { + /// Report a ready deployment while testing lifecycle transport. + fn health(&mut self, _workload: &Workload) -> impl Future> + Send { + std::future::ready(Ok(Health::Ready)) + } + + /// Reject any unexpected restart of a healthy fixture. + fn restart(&mut self, _workload: &Workload) -> impl Future> + Send { + std::future::ready(Err(RecoveryError::Invalid("unexpected restart of healthy work".into()))) + } +} + +/// Reject foreign and replayed evidence while accepting valid work between polls and persisting it on disconnect. +#[tokio::test] +async fn idle_control_rejects_invalid_evidence_without_losing_subsequent_valid_work() { + let directory = tempfile::tempdir().unwrap(); + let server = et_ws_test_server::start(); + let config = connected_config(&server, directory.path(), 3600); + let (sender, mut rejected) = tokio::sync::mpsc::unbounded_channel(); + let subscriber = tracing_subscriber::registry().with(RejectedEvidence(sender)); + let task = tokio::spawn(run(config.clone(), ReadyBackend).with_subscriber(subscriber)); + let (mut producer, _) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("example-worker")).await; + wait_for_agent(&mut producer, &config.agent_id).await; + wait_for_completed_cycle(&config).await; + let mut foreign = event("checkpoint-restored", 999, 1, "foreign"); + foreign.data["job_id"] = "another-job".into(); + send_evidence(&mut producer, &config.agent_id, foreign).await; + assert_rejection(&mut rejected, "invalid lifecycle identity or job").await; + let started = event("started", 0, 1, "one"); + let checkpoint = event("checkpoint-restored", 3, 2, "one"); + send_evidence(&mut producer, &config.agent_id, started).await; + send_evidence(&mut producer, &config.agent_id, checkpoint.clone()).await; + send_evidence(&mut producer, &config.agent_id, checkpoint).await; + assert_rejection(&mut rejected, "stale evidence").await; + assert!(!task.is_finished()); + server.stop().await; + let stopped = tokio::time::timeout(Duration::from_secs(5), task).await.unwrap(); + let error = stopped.unwrap().unwrap_err(); + assert!(matches!(error, RecoveryError::Socket(_) | RecoveryError::Invalid(_))); + let saved: serde_json::Value = serde_json::from_slice(&fs::read(&config.state_file).unwrap()).unwrap(); + let checkpoints = saved["checkpoint_measurements"].as_array().unwrap(); + assert_eq!(checkpoints.len(), 1); + assert_eq!(checkpoints[0]["progress"], 3_u64); + assert_eq!(checkpoints[0]["instance"], "one"); + assert_eq!(saved["incidents"], serde_json::json!([])); +} + +/// Complete registration and the initial poll before sending events that must be handled by the idle control loop. +#[expect( + clippy::single_call_fn, + reason = "completed-cycle waiting separates poll and idle-receive coverage" +)] +async fn wait_for_completed_cycle(config: &et_recovery_supervisor::config::Config) { + tokio::time::timeout(Duration::from_secs(5), async { + while et_recovery_supervisor::health::check(config).is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +/// Require a fresh rejection for each invalid event rather than reusing an earlier warning. +async fn assert_rejection(receiver: &mut tokio::sync::mpsc::UnboundedReceiver, expected: &str) { + let error = tokio::time::timeout(Duration::from_secs(5), receiver.recv()) + .await + .unwrap() + .unwrap(); + assert!(error.contains(expected), "expected {expected}: {error}"); +} + +/// Deliver one lifecycle event and wait for its transport acknowledgement independently of policy acceptance. +async fn send_evidence( + producer: &mut et_ws_runner_common::RegisteredSocket, + identity: &str, + evidence: edge_toolkit::ws::CloudEvent, +) { + let message = ClientMessage::SendAgentMessage { + to_agent_id: identity.into(), + message: serde_json::to_value(evidence).unwrap(), + }; + send_message(producer, &message).await; + assert_acknowledged(producer).await; +} + +/// Give each real-hub test an isolated budget path and an explicit observation interval. +fn connected_config( + server: &et_ws_test_server::TestServer, + directory: &std::path::Path, + poll_seconds: u64, +) -> et_recovery_supervisor::config::Config { + let mut config = config(); + config.ws_url.clone_from(&server.ws_url); + config.poll_seconds = poll_seconds; + config.state_file = directory.join("state.json"); + config +} + +/// Consume real hub evidence while preventing concurrent ownership of the same durable budget. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn service_consumes_lifecycle_evidence_and_excludes_a_second_budget_owner() { + let directory = tempfile::tempdir().unwrap(); + let server = et_ws_test_server::start(); + let config = connected_config(&server, directory.path(), 1); + let (observing, mut observed) = tokio::sync::mpsc::channel(1); + let (release, gate) = tokio::sync::mpsc::channel(1); + let task = tokio::spawn(run( + config.clone(), + GatedBackend { + observing, + release: gate, + }, + )); + let (mut producer, _) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("example-worker")).await; + wait_for_agent(&mut producer, &config.agent_id).await; + tokio::time::timeout(Duration::from_secs(5), observed.recv()) + .await + .unwrap() + .unwrap(); + assert!( + run(config.clone(), ReadyBackend) + .await + .unwrap_err() + .to_string() + .contains("budget lock") + ); + for (kind, sequence) in [("started", 1), ("checkpoint-restored", 2), ("progress", 3)] { + let message = ClientMessage::SendAgentMessage { + to_agent_id: config.agent_id.clone(), + message: serde_json::to_value(event(kind, 1, sequence, "one")).unwrap(), + }; + send_message(&mut producer, &message).await; + assert_acknowledged(&mut producer).await; + } + release.send(()).await.unwrap(); + wait_for_checkpoint(&config).await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); +} + +/// Require evidence queued during the blocked observation to appear in the first persisted cycle. +#[expect( + clippy::single_call_fn, + reason = "bounded report waiting is separate from transport and ownership assertions" +)] +async fn wait_for_checkpoint(config: &et_recovery_supervisor::config::Config) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let document: serde_json::Value = serde_json::from_slice(&fs::read(&config.state_file).unwrap()).unwrap(); + if document["checkpoint_measurements"] + .as_array() + .is_some_and(|values| values.len() == 1) + { + assert_eq!(document["checkpoint_measurements"][0]["progress"], 1_u64); + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap(); +} + +/// Build a controller-assisted policy and backend sharing one isolated durable budget. +fn assisted_fixture(directory: &std::path::Path) -> (et_recovery_supervisor::config::Config, Supervisor, FakeBackend) { + let mut config = config(); + config.mode = Mode::ControllerAssisted; + config.state_file = directory.join("state.json"); + let supervisor = Supervisor::new(config.clone(), load_budget(&config).unwrap()).unwrap(); + let backend = FakeBackend { + config: config.clone(), + restarts: 0, + unknown: false, + fail_restart: false, + }; + (config, supervisor, backend) +} + +/// Serialize and send one typed client message through an established hub connection. +async fn send_message(socket: &mut et_ws_runner_common::RegisteredSocket, message: &ClientMessage) { + socket + .send(tokio_tungstenite::tungstenite::Message::Text( + serde_json::to_string(message).unwrap().into(), + )) + .await + .unwrap(); +} + +/// Require an acknowledged delivery status within a bounded transport wait. +async fn assert_acknowledged(producer: &mut et_ws_runner_common::RegisteredSocket) { + let acknowledged = tokio::time::timeout(Duration::from_secs(3), async { + while let Some(Ok(frame)) = producer.next().await { + if let tokio_tungstenite::tungstenite::Message::Text(text) = frame + && matches!( + ServerMessage::from_text_frame(&text), + Ok(ServerMessage::MessageStatus { + status: edge_toolkit::ws::MessageDeliveryStatus::Acknowledged, + .. + }) + ) + { + return true; + } + } + false + }) + .await + .unwrap(); + assert!(acknowledged); +} + +/// Wait for the supervisor's registered identity before sending workload evidence. +async fn wait_for_agent(producer: &mut et_ws_runner_common::RegisteredSocket, identity: &str) { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + producer + .send(tokio_tungstenite::tungstenite::Message::Text( + serde_json::to_string(&ClientMessage::ListAgents).unwrap().into(), + )) + .await + .unwrap(); + if let tokio_tungstenite::tungstenite::Message::Text(text) = et_ws_test_server::next_payload(producer).await + && let Ok(ServerMessage::ListAgentsResponse { agents }) = ServerMessage::from_text_frame(&text) + && agents.iter().any(|agent| agent.agent_id == identity) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} diff --git a/services/recovery-supervisor/tests/shutdown.rs b/services/recovery-supervisor/tests/shutdown.rs new file mode 100644 index 00000000..24f4e2e7 --- /dev/null +++ b/services/recovery-supervisor/tests/shutdown.rs @@ -0,0 +1,139 @@ +//! Unix termination signals must save application evidence received since the last completed observation cycle. +#![cfg(test)] +#![cfg(unix)] + +use std::time::Duration; + +use edge_toolkit::ws::{ClientMessage, MessageDeliveryStatus, ServerMessage}; +use et_recovery_supervisor::config::Config; +use et_test_recovery::{config, environment, event}; +use fs_err as fs; +use futures_util::{SinkExt as _, StreamExt as _}; +use tokio::io::{AsyncBufReadExt as _, BufReader}; +use tokio::process::{Child, Command}; +use tokio_tungstenite::tungstenite::Message; + +/// Launch the production executable with isolated state and a kubeconfig that cannot contact a cluster. +#[expect( + clippy::single_call_fn, + reason = "keep subprocess setup separate from shutdown assertions" +)] +fn launch(config: &Config) -> Child { + let fixture = + edge_toolkit::config::get_project_root().join("services/recovery-supervisor/tests/fixtures/kubeconfig.yaml"); + Command::new(env!("CARGO_BIN_EXE_et-recovery-supervisor")) + .envs(environment()) + .env("RECOVERY_WS_URL", &config.ws_url) + .env("RECOVERY_STATE_FILE", &config.state_file) + .env("RECOVERY_POLL_SECONDS", config.poll_seconds.to_string()) + .env("KUBECONFIG", fixture) + .env("RUST_LOG", "warn") + .stdout(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap() +} + +/// Wait for the first completed poll so later evidence cannot be saved by a scheduled cycle. +#[expect( + clippy::single_call_fn, + reason = "isolate the bounded startup wait from the termination test" +)] +async fn wait_for_initial_poll(config: &Config, child: &mut Child) { + tokio::time::timeout(Duration::from_secs(20), async { + while et_recovery_supervisor::health::check(config).is_err() { + assert!( + child.try_wait().unwrap().is_none(), + "supervisor exited before its initial poll" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +/// Require the child to process a replay after the valid checkpoint, rather than relying on queue admission alone. +#[expect( + clippy::single_call_fn, + reason = "log synchronization proves evidence was applied before signaling" +)] +async fn wait_for_replay_rejection(child: &mut Child) { + let stdout = child.stdout.take().unwrap(); + let mut lines = BufReader::new(stdout).lines(); + tokio::time::timeout(Duration::from_secs(10), async { + while let Some(line) = lines.next_line().await.unwrap() { + if line.contains("stale evidence") { + return; + } + } + panic!("supervisor exited before rejecting the replay"); + }) + .await + .unwrap(); +} + +/// Deliver lifecycle evidence and require its own transport acknowledgement. +async fn send(socket: &mut et_ws_runner_common::RegisteredSocket, kind: &str, progress: u64, sequence: u64) { + let message = ClientMessage::SendAgentMessage { + to_agent_id: "recovery-supervisor".into(), + message: serde_json::to_value(event(kind, progress, sequence, "one")).unwrap(), + }; + socket + .send(Message::Text(serde_json::to_string(&message).unwrap().into())) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + while let Some(Ok(Message::Text(text))) = socket.next().await { + if matches!( + ServerMessage::from_text_frame(&text), + Ok(ServerMessage::MessageStatus { + status: MessageDeliveryStatus::Acknowledged, + .. + }) + ) { + return; + } + } + panic!("lifecycle delivery was not acknowledged"); + }) + .await + .unwrap(); +} + +/// Save accepted but not yet polled checkpoint evidence and release the lease on the normal container-stop signal. +#[tokio::test] +async fn sigterm_saves_the_final_report_before_a_successful_exit() { + let directory = tempfile::tempdir().unwrap(); + let server = et_ws_test_server::start(); + let mut config = config(); + config.ws_url.clone_from(&server.ws_url); + config.state_file = directory.path().join("state.json"); + config.poll_seconds = 3600; + let mut child = launch(&config); + wait_for_initial_poll(&config, &mut child).await; + let (mut producer, _) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("example-worker")).await; + send(&mut producer, "started", 0, 1).await; + send(&mut producer, "checkpoint-restored", 3, 2).await; + send(&mut producer, "checkpoint-restored", 3, 2).await; + wait_for_replay_rejection(&mut child).await; + let before: serde_json::Value = serde_json::from_slice(&fs::read(&config.state_file).unwrap()).unwrap(); + assert_eq!(before["checkpoint_measurements"], serde_json::json!([])); + let signal = Command::new("coreutils") + .args(["kill", "-TERM", &child.id().unwrap().to_string()]) + .output() + .await + .unwrap(); + assert!(signal.status.success(), "{signal:?}"); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .unwrap() + .unwrap(); + assert!( + status.success(), + "supervisor terminated without a graceful exit: {status}" + ); + let after: serde_json::Value = serde_json::from_slice(&fs::read(&config.state_file).unwrap()).unwrap(); + assert_eq!(after["checkpoint_measurements"][0]["progress"], 3_u64); + assert!(et_recovery_supervisor::health::check(&config).is_err()); +} diff --git a/services/recovery-supervisor/tests/tls.rs b/services/recovery-supervisor/tests/tls.rs new file mode 100644 index 00000000..28c4c736 --- /dev/null +++ b/services/recovery-supervisor/tests/tls.rs @@ -0,0 +1,34 @@ +//! Standalone supervisor builds negotiate TLS for secure WebSocket endpoints. +#![cfg(test)] + +use std::time::Duration; + +use tokio::io::AsyncReadExt as _; + +/// Require a real TLS `ClientHello` from the supervisor's independently enabled secure transport. +#[tokio::test] +async fn secure_websocket_sends_a_tls_handshake_instead_of_rejecting_the_scheme() { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("wss://{}/ws", listener.local_addr().unwrap()); + let client = tokio::spawn(async move { + tokio::time::timeout(Duration::from_secs(10), tokio_tungstenite::connect_async(endpoint)) + .await + .unwrap() + }); + let (mut peer, _) = tokio::time::timeout(Duration::from_secs(10), listener.accept()) + .await + .unwrap() + .unwrap(); + let mut record_type = [0_u8; 1]; + let _read = tokio::time::timeout(Duration::from_secs(10), peer.read_exact(&mut record_type)) + .await + .unwrap() + .unwrap(); + // TLS handshake records start with 0x16. Without TLS compiled in, the client closes before sending a record. + assert_eq!(record_type, [0x16_u8]); + drop(peer); + // This peer deliberately stops after ClientHello; it cannot complete a secure WebSocket connection. + let _handshake_error = client.await.unwrap().unwrap_err(); +} diff --git a/utilities/cli/Cargo.toml b/utilities/cli/Cargo.toml index 78483a70..bf99f587 100644 --- a/utilities/cli/Cargo.toml +++ b/utilities/cli/Cargo.toml @@ -22,6 +22,7 @@ croner.workspace = true edge-toolkit.workspace = true et-org.workspace = true et-path.workspace = true +et-recovery-supervisor.workspace = true fake.workspace = true fs-err.workspace = true jiff.workspace = true @@ -41,6 +42,7 @@ toml.workspace = true command-error.workspace = true et-test-helpers.workspace = true et-ws-test-server.workspace = true +serde-env.workspace = true tempfile.workspace = true [lints] diff --git a/utilities/cli/src/deployment_types/k3s.rs b/utilities/cli/src/deployment_types/k3s.rs index 4f68a5fb..eb3240af 100644 --- a/utilities/cli/src/deployment_types/k3s.rs +++ b/utilities/cli/src/deployment_types/k3s.rs @@ -42,6 +42,7 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, ObjectMeta}; use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use super::document; +use super::k3s_recovery; use crate::error::CliError; use crate::input::{ArtifactSource, ClusterInput}; use crate::limits::{CpuCap, Limits, MAX_CPU_SHARES, ResolvedLimits, k8s_bytes, k8s_cpus}; @@ -60,10 +61,10 @@ use crate::{ /// `IfNotPresent` is what stops every restart re-pulling a tag the node already holds. It also lets a published-image /// scenario be tested against a local build of the same tag, which is how CI checks a change to an image before it /// is published. -const IMAGE_PULL_POLICY: &str = "IfNotPresent"; +pub(super) const IMAGE_PULL_POLICY: &str = "IfNotPresent"; /// Tag the generated manifests reference, matching what the README's build-and-import step produces. -const IMAGE_TAG: &str = "latest"; +pub(super) const IMAGE_TAG: &str = "latest"; /// Collector image, matching the one `compose.yaml` runs. const OPENOBSERVE_IMAGE: &str = "openobserve/openobserve:v1.1.1"; @@ -202,6 +203,7 @@ pub fn generate_k3s_deployment(cluster: &ClusterInput, output_dir: &Path) -> Res ), cluster, )?; + crate::recovery::validate(cluster, &runners)?; let serves_a_page = super::serves_a_page(cluster); let module_paths = docker_image_module_paths(&cluster_module_names(cluster), serves_a_page, &cluster.module_paths)?; // Empty for a headless cluster, which is not given the page: the hub rejects a root it cannot find among the @@ -213,9 +215,16 @@ pub fn generate_k3s_deployment(cluster: &ClusterInput, output_dir: &Path) -> Res }; let mut docs = fixed_documents(&namespace, cluster, &module_paths, &root_module, &hub_site(cluster))?; - for runner in &runners { - docs.push(runner_document(&namespace, runner, cluster.artifact_source)?); + for mut runner in runners { + let supervised = k3s_recovery::configure_runner(cluster, &mut runner); + docs.push(runner_document( + &namespace, + &runner, + cluster.artifact_source, + supervised, + )?); } + docs.extend(k3s_recovery::documents(cluster, &namespace)?); let plugins = cluster_plugins(cluster); docs.extend(runtime_plugin_documents(&plugins)?); @@ -281,7 +290,7 @@ fn labels(name: &str) -> BTreeMap { } /// Metadata for a namespaced object, labelled so its Service can select it. -fn meta(namespace: &str, name: &str) -> ObjectMeta { +pub(super) fn meta(namespace: &str, name: &str) -> ObjectMeta { ObjectMeta { labels: Some(labels(name)), name: Some(name.to_string()), @@ -291,7 +300,7 @@ fn meta(namespace: &str, name: &str) -> ObjectMeta { } /// Metadata for an object that holds no pods, so nothing ever selects it by label. -fn plain_meta(namespace: &str, name: &str) -> ObjectMeta { +pub(super) fn plain_meta(namespace: &str, name: &str) -> ObjectMeta { ObjectMeta { name: Some(name.to_string()), namespace: Some(namespace.to_string()), @@ -330,7 +339,7 @@ fn collector_config(namespace: &str) -> ConfigMap { } /// A one-gigabyte `ReadWriteOnce` claim, standing in for the named volumes `compose.yaml` declares. -fn claim(namespace: &str, name: &str) -> PersistentVolumeClaim { +pub(super) fn claim(namespace: &str, name: &str) -> PersistentVolumeClaim { PersistentVolumeClaim { metadata: plain_meta(namespace, name), spec: Some(PersistentVolumeClaimSpec { @@ -368,7 +377,7 @@ fn from_secret(namespace: &str) -> EnvFromSource { } /// One literal environment entry. -fn env(name: &str, value: String) -> EnvVar { +pub(super) fn env(name: &str, value: String) -> EnvVar { EnvVar { name: name.to_string(), value: Some(value), @@ -392,7 +401,7 @@ fn probe(exec: Option, http_get: Option) -> Probe { /// Drops every capability, forbids privilege escalation, and takes the root filesystem read-only; anything a container /// genuinely has to write gets an explicit volume instead. `runAsNonRoot` is belt-and-braces next to the pod-level ids /// -- it makes the kubelet refuse to start a container that would resolve to uid 0 anyway. -fn hardened_container_context() -> SecurityContext { +pub(super) fn hardened_container_context() -> SecurityContext { SecurityContext { allow_privilege_escalation: Some(false), capabilities: Some(Capabilities { @@ -561,9 +570,14 @@ fn cluster_plugins(cluster: &ClusterInput) -> Vec<&'static RuntimePlugin> { .filter(|agent| agent.runner.is_some()) .map(|agent| &agent.limits); let services = &cluster.services; + let recovery_limits = match &cluster.recovery { + crate::recovery::Recovery::Configured(settings) => vec![&settings.limits], + crate::recovery::Recovery::Disabled => Vec::new(), + }; let mut needed: Vec<&RuntimePlugin> = [&services.collector.limits, &services.hub.limits] .into_iter() .chain(agents) + .chain(recovery_limits) .flat_map(|limits| required_plugins(&limits.resolve())) .collect(); needed.sort_unstable_by_key(|plugin| plugin.name); @@ -746,7 +760,7 @@ fn mount_shm(container: &mut Container, volumes: &mut Vec, limits: &Limi } /// The container half of a volume pair, which is the same whatever the volume is backed by. -fn volume_mount(name: &str, path: &str) -> VolumeMount { +pub(super) fn volume_mount(name: &str, path: &str) -> VolumeMount { VolumeMount { mount_path: path.to_string(), name: name.to_string(), @@ -760,7 +774,7 @@ fn volume_mount(name: &str, path: &str) -> VolumeMount { /// so a caller passing `/tmp` is naming the containerised program's own temp directory rather than creating a world- /// writable file on the host. `DeepSource RS-S1003` reads the literal as the latter, which is why every call site /// passing `/tmp` carries a `skipcq` for that one rule. -fn scratch(name: &str, path: &str) -> (VolumeMount, Volume) { +pub(super) fn scratch(name: &str, path: &str) -> (VolumeMount, Volume) { let volume = Volume { empty_dir: Some(EmptyDirVolumeSource::default()), name: name.to_string(), @@ -770,7 +784,7 @@ fn scratch(name: &str, path: &str) -> (VolumeMount, Volume) { } /// Mount a claim of the same name at `path`. -fn mount(name: &str, path: &str) -> (VolumeMount, Volume) { +pub(super) fn mount(name: &str, path: &str) -> (VolumeMount, Volume) { let volume = Volume { name: name.to_string(), persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource { @@ -786,7 +800,7 @@ fn mount(name: &str, path: &str) -> (VolumeMount, Volume) { /// /// A single-host scenario pins nothing and leaves placement to the scheduler, which on one node has no choice to make. /// `limits` are applied here rather than by each caller, so no component can be built without them. -fn deployment( +pub(super) fn deployment( namespace: &str, name: &str, container: Container, @@ -1053,7 +1067,12 @@ fn runner_env(runner: &RunnerInstance) -> Vec { /// One workload per runner, each hosting a single module exactly as the other formats arrange it. /// /// A `Deployment` for a runner that runs continuously, a `CronJob` for a scheduled one. -fn runner_document(namespace: &str, runner: &RunnerInstance, images: ArtifactSource) -> Result { +fn runner_document( + namespace: &str, + runner: &RunnerInstance, + images: ArtifactSource, + supervised: bool, +) -> Result { // A runner fetches its module to a scratch directory and, for the web runner, lets Deno cache there, so a read-only // root needs somewhere writable even though nothing is meant to persist. skipcq: RS-S1003 let (runtime_mount, runtime_volume) = scratch("runner-tmp", "/tmp"); @@ -1072,14 +1091,18 @@ fn runner_document(namespace: &str, runner: &RunnerInstance, images: ArtifactSou }; let volumes = vec![runtime_volume]; if runner.schedule.is_unset() { - return document(&deployment( + let mut object = deployment( namespace, &runner.name, container, volumes, &runner.site, &runner.limits, - )); + ); + if supervised { + k3s_recovery::recreate(&mut object); + } + return document(&object); } let template = pod_template(&runner.name, container, volumes, &runner.site, &runner.limits); document(&cron_job(namespace, &runner.name, template, &runner.schedule)) diff --git a/utilities/cli/src/deployment_types/k3s_recovery.rs b/utilities/cli/src/deployment_types/k3s_recovery.rs new file mode 100644 index 00000000..cdc6b625 --- /dev/null +++ b/utilities/cli/src/deployment_types/k3s_recovery.rs @@ -0,0 +1,238 @@ +//! Kubernetes recovery deployment: explicit credentials, named-target RBAC and a durable single-owner budget. + +use std::collections::BTreeMap; + +use et_recovery_supervisor::config::Mode; +use k8s_openapi::api::apps::v1::{Deployment, DeploymentStrategy}; +use k8s_openapi::api::core::v1::{ + ConfigMap, ConfigMapVolumeSource, Container, ExecAction, Probe, ServiceAccount, Volume, +}; +use k8s_openapi::api::rbac::v1::{PolicyRule, Role, RoleBinding, RoleRef, Subject}; + +use super::document; +use super::k3s::{ + IMAGE_PULL_POLICY, IMAGE_TAG, claim, deployment, env, hardened_container_context, meta, mount, plain_meta, scratch, + volume_mount, +}; +use crate::placement::hub_site; +use crate::recovery::{Recovery, RecoverySettings}; +use crate::{CliError, ClusterInput, RunnerInstance, image_prefix}; + +const NAME: &str = "recovery-supervisor"; +const CONFIG: &str = "recovery-credentials"; +const STATE: &str = "recovery-state"; + +/// Inject only the bindings of allowlisted native-Python workloads. +pub(super) fn configure_runner(cluster: &ClusterInput, runner: &mut RunnerInstance) -> bool { + if let Recovery::Configured(settings) = &cluster.recovery + && let Some(workload) = settings + .workloads + .iter() + .find(|workload| workload.deployment == runner.name) + { + runner.env.extend(crate::recovery::bindings(settings, workload)); + return true; + } + false +} + +/// Single-writer replacement for the supervisor budget and supervised application checkpoints. +pub(super) fn recreate(object: &mut Deployment) { + if let Some(spec) = object.spec.as_mut() { + spec.strategy = Some(DeploymentStrategy { + type_: Some("Recreate".into()), + ..DeploymentStrategy::default() + }); + } +} + +/// Generate nothing unless the scenario explicitly requests recovery. +pub(super) fn documents(cluster: &ClusterInput, namespace: &str) -> Result, CliError> { + let Recovery::Configured(settings) = &cluster.recovery else { + return Ok(Vec::new()); + }; + let account = ServiceAccount { + metadata: plain_meta(namespace, NAME), + ..ServiceAccount::default() + }; + let credentials = ConfigMap { + metadata: plain_meta(namespace, CONFIG), + data: Some(BTreeMap::from([("kubeconfig".into(), kubeconfig(namespace)?)])), + ..ConfigMap::default() + }; + let mut variables = settings.environment(namespace)?; + let _previous = variables.insert("RUST_LOG".into(), "info".into()); + let _previous = variables.insert("KUBECONFIG".into(), "/app/credentials/kubeconfig".into()); + let object = supervisor_deployment(cluster, namespace, variables, settings)?; + Ok(vec![ + document(&account)?, + document(&role(settings, namespace))?, + document(&binding(namespace))?, + document(&credentials)?, + document(&claim(namespace, STATE))?, + document(&object)?, + ]) +} + +/// Grant access only to named workloads, with mutation reserved for controller-assisted mode. +#[expect( + clippy::single_call_fn, + reason = "keep the least-privilege authorization definition explicit" +)] +fn role(settings: &RecoverySettings, namespace: &str) -> Role { + let verbs = if settings.mode == Mode::ControllerAssisted { + vec!["get", "patch"] + } else { + vec!["get"] + }; + Role { + metadata: plain_meta(namespace, NAME), + rules: Some(vec![PolicyRule { + api_groups: Some(vec!["apps".into()]), + resources: Some(vec!["deployments".into()]), + resource_names: Some( + settings + .workloads + .iter() + .map(|workload| workload.deployment.clone()) + .collect(), + ), + verbs: verbs.into_iter().map(str::to_owned).collect(), + ..PolicyRule::default() + }]), + } +} + +/// Bind the scoped role to this supervisor's service account. +#[expect( + clippy::single_call_fn, + reason = "separate role binding from configuration and deployment generation" +)] +fn binding(namespace: &str) -> RoleBinding { + RoleBinding { + metadata: plain_meta(namespace, NAME), + role_ref: RoleRef { + api_group: "rbac.authorization.k8s.io".into(), + kind: "Role".into(), + name: NAME.into(), + }, + subjects: Some(vec![Subject { + kind: "ServiceAccount".into(), + name: NAME.into(), + namespace: Some(namespace.into()), + ..Subject::default() + }]), + } +} + +/// Use the projected, rotating `ServiceAccount` token and cluster CA, never the operator's host kubeconfig. +fn kubeconfig(namespace: &str) -> Result { + let service_account = "/var/run/secrets/kubernetes.io/serviceaccount"; + Ok(serde_yaml::to_string(&serde_json::json!({ + "apiVersion": "v1", "kind": "Config", "current-context": "in-cluster", + "clusters": [{"name": "cluster", "cluster": { + "server": "https://kubernetes.default.svc", "certificate-authority": format!("{service_account}/ca.crt") + }}], + "users": [{"name": "supervisor", "user": {"tokenFile": format!("{service_account}/token")}}], + "contexts": [{"name": "in-cluster", "context": { + "cluster": "cluster", "user": "supervisor", "namespace": namespace + }}] + }))?) +} + +/// Mount persistent state separately from credentials and writable kubectl discovery caches. +fn supervisor_deployment( + cluster: &ClusterInput, + namespace: &str, + variables: BTreeMap, + settings: &RecoverySettings, +) -> Result { + let (state_mount, state_volume) = mount(STATE, "/app/state"); + let (runtime_mount, runtime_volume) = scratch("recovery-runtime", "/app/runtime"); + let credentials = credentials_volume(); + let (probe, startup) = health_probes(&settings.config(namespace)?)?; + let container = Container { + name: NAME.into(), + image: Some(format!( + "{}et-recovery-supervisor:{IMAGE_TAG}", + image_prefix(cluster.artifact_source) + )), + image_pull_policy: Some(IMAGE_PULL_POLICY.into()), + env: Some(variables.into_iter().map(|(name, value)| env(&name, value)).collect()), + security_context: Some(hardened_container_context()), + startup_probe: Some(startup), + readiness_probe: Some(probe.clone()), + liveness_probe: Some(probe), + volume_mounts: Some(vec![ + state_mount, + runtime_mount, + volume_mount(CONFIG, "/app/credentials"), + ]), + ..Container::default() + }; + let mut object = deployment( + namespace, + NAME, + container, + vec![state_volume, runtime_volume, credentials], + &hub_site(cluster), + &settings.limits, + ); + recreate(&mut object); + if let Some(spec) = object.spec.as_mut().and_then(|spec| spec.template.spec.as_mut()) { + spec.service_account_name = Some(NAME.into()); + spec.automount_service_account_token = Some(true); + } + object.metadata = meta(namespace, NAME); + Ok(object) +} + +/// Project the generated kubeconfig independently of writable state and runtime directories. +#[expect( + clippy::single_call_fn, + reason = "the read-only credential volume is a distinct deployment resource" +)] +fn credentials_volume() -> Volume { + Volume { + name: CONFIG.into(), + config_map: Some(ConfigMapVolumeSource { + name: CONFIG.into(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + } +} + +/// Allow registration and a full bounded cycle before liveness checks can restart a starting supervisor. +#[expect( + clippy::single_call_fn, + reason = "probe construction is a distinct deployment-hardening step" +)] +fn health_probes(config: &et_recovery_supervisor::config::Config) -> Result<(Probe, Probe), CliError> { + let probe = Probe { + exec: Some(ExecAction { + command: Some(vec!["et-recovery-healthcheck".into()]), + }), + period_seconds: Some(10_i32), + timeout_seconds: Some(3_i32), + failure_threshold: Some(3_i32), + ..Probe::default() + }; + let startup_seconds = et_recovery_supervisor::health::silence_limit(config) + .as_secs() + .saturating_add(30); + let failure_threshold = match i32::try_from(startup_seconds) { + Ok(seconds) => seconds, + Err(error) => { + return Err(CliError::Recovery(format!( + "supervisor startup probe duration exceeds Kubernetes limits: {error}" + ))); + } + }; + let startup = Probe { + period_seconds: Some(1_i32), + failure_threshold: Some(failure_threshold), + ..probe.clone() + }; + Ok((probe, startup)) +} diff --git a/utilities/cli/src/deployment_types/mod.rs b/utilities/cli/src/deployment_types/mod.rs index eac8de21..b552094c 100644 --- a/utilities/cli/src/deployment_types/mod.rs +++ b/utilities/cli/src/deployment_types/mod.rs @@ -2,6 +2,7 @@ mod dekit; mod docker_compose; mod hub; mod k3s; +mod k3s_recovery; mod mise; mod process_compose; mod scenario_image; diff --git a/utilities/cli/src/error.rs b/utilities/cli/src/error.rs index 0d6633dc..d7ba5092 100644 --- a/utilities/cli/src/error.rs +++ b/utilities/cli/src/error.rs @@ -10,6 +10,15 @@ use thiserror::Error; #[derive(Debug, Error)] #[non_exhaustive] pub enum CliError { + /// Recovery settings cannot safely supervise the declared scenario. + #[error("Invalid recovery scenario: {0}")] + Recovery(String), + /// The supervisor's configuration validation rejected the generated policy or bindings. + #[error(transparent)] + RecoveryConfig(#[from] et_recovery_supervisor::RecoveryError), + /// Recovery runtime settings could not be encoded. + #[error("Recovery configuration encoding failed: {0}")] + RecoveryEncoding(#[from] serde_json::Error), #[error(transparent)] Io(#[from] std::io::Error), diff --git a/utilities/cli/src/input.rs b/utilities/cli/src/input.rs index dbc3ec18..2e112a7c 100644 --- a/utilities/cli/src/input.rs +++ b/utilities/cli/src/input.rs @@ -139,6 +139,9 @@ pub struct ClusterInput { /// Settings for the processes every deployment runs whatever its agents, from `services:`. #[serde(default, skip_serializing_if = "Services::is_unset")] pub services: Services, + /// Optional Kubernetes recovery supervision, from `recovery:`; absent leaves deployment behavior unchanged. + #[serde(default, skip_serializing_if = "crate::recovery::Recovery::is_unset")] + pub recovery: crate::recovery::Recovery, } /// The processes every deployment runs alongside the agents' runners, each with its own settings. @@ -215,6 +218,7 @@ impl Default for ClusterInput { module_paths: Vec::new(), machines: Vec::new(), services: Services::default(), + recovery: crate::recovery::Recovery::default(), } } } diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index 12dc601c..eec7c6bd 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -21,6 +21,7 @@ mod input; mod limits; mod module_package_json; mod placement; +mod recovery; mod scenario_password; mod schedule; @@ -44,6 +45,7 @@ pub use self::input::{ }; pub use self::limits::{Cpus, Limits, LimitsError, Ulimit}; pub use self::module_package_json::generate_module_package_json; +pub use self::recovery::{Recovery, RecoverySettings}; pub(crate) use self::scenario_password::{scenario_password, scenario_seed}; pub use self::schedule::Schedule; @@ -502,6 +504,7 @@ pub fn regenerate_verification( .iter() .copied() .filter(|output_type| !scheduled || output_type.runs_schedules()) + .filter(|output_type| cluster.recovery.is_unset() || *output_type == OutputType::K3s) .collect(); if output_types.is_empty() { continue; @@ -531,6 +534,13 @@ const fn deployment_summary( } } +/// Validate the requested formats and recovery bindings before writing or removing any deployment files. +fn validate_deployment_outputs(cluster: &ClusterInput, output_types: &[OutputType]) -> Result<(), CliError> { + refuse_unschedulable_formats(cluster, output_types)?; + recovery::validate_formats(cluster, output_types)?; + recovery::validate(cluster, &scenario_runners(cluster)?) +} + /// Write every file a deployment of `output_types` needs into `output_dir`, with its credential and README. fn generate_deployment_outputs( cluster: &ClusterInput, @@ -538,7 +548,7 @@ fn generate_deployment_outputs( output_types: &[OutputType], seed: u64, ) -> Result<(), CliError> { - refuse_unschedulable_formats(cluster, output_types)?; + validate_deployment_outputs(cluster, output_types)?; if !output_dir.exists() { fs::create_dir_all(output_dir)?; } @@ -552,7 +562,8 @@ fn generate_deployment_outputs( for output_type in output_types { generate_format(cluster, output_dir, *output_type, with_process_compose)?; } - write_readme(cluster, output_dir, output_types) + write_readme(cluster, output_dir, output_types)?; + recovery::remove_unsupported_outputs(cluster, output_dir) } /// Refuse a scheduled scenario in a format that cannot schedule it. @@ -1209,6 +1220,13 @@ fn k3s_machine_imports(cluster: &ClusterInput, server: &input::Machine) -> Strin cluster.cluster_name, server.address ); if matches!(cluster.artifact_source, ArtifactSource::Local) { + if !cluster.recovery.is_unset() { + let _write_result = writeln!( + imports, + "docker save et-recovery-supervisor:latest | ssh {} sudo k3s ctr images import -", + server.address + ); + } for machine in &cluster.machines { for kind in machine_runner_kinds(cluster, &machine.name) { let _write_result = writeln!( @@ -1414,6 +1432,7 @@ fn k3s_run_instructions( "docker save \"$image\" | sudo k3s ctr images import -\n", "```\n\n", "{runner_note}", + "{recovery_note}", "{prepare}", "{apply}", ), @@ -1429,7 +1448,8 @@ fn k3s_run_instructions( name = cluster.cluster_name, preamble = hub.preamble, secret = secret, - runner_note = runner_image_note(runners, images) + runner_note = runner_image_note(runners, images), + recovery_note = recovery::image_note(cluster) ) } diff --git a/utilities/cli/src/recovery.rs b/utilities/cli/src/recovery.rs new file mode 100644 index 00000000..50cc054b --- /dev/null +++ b/utilities/cli/src/recovery.rs @@ -0,0 +1,234 @@ +//! Optional scenario recovery configuration, validated against the actual runner deployments before writing output. + +use std::collections::BTreeMap; +use std::path::Path; + +use et_recovery_supervisor::config::{Config, Mode, Policy, Workload}; +use fs_err as fs; +use serde::{Deserialize, Serialize}; + +use crate::limits::Limits; +use crate::{CliError, ClusterInput, OutputType, RunnerInstance, hub_service_ws_url}; + +/// Absence is a normal scenario without recovery, not an implicit policy or permission grant. +#[derive(Clone, Debug, Default, Serialize)] +#[serde(untagged)] +#[non_exhaustive] +pub enum Recovery { + /// Explicit recovery settings. + Configured(Box), + /// No supervisor is deployed. + #[default] + Disabled, +} + +impl Recovery { + /// Whether recovery was omitted from the scenario. + #[must_use] + pub const fn is_unset(&self) -> bool { + matches!(self, Self::Disabled) + } +} + +impl<'de> Deserialize<'de> for Recovery { + /// Preserve detailed errors when decoding optional scenario recovery settings. + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(RecoveryVisitor) + } +} + +/// Preserve field-specific configuration errors rather than hiding them behind untagged-enum matching. +struct RecoveryVisitor; + +impl<'de> serde::de::Visitor<'de> for RecoveryVisitor { + type Value = Recovery; + + /// Describe the accepted mapping-or-null shape in configuration errors. + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a recovery configuration mapping or null") + } + + /// Treat an explicit null recovery block as disabled supervision. + fn visit_unit(self) -> Result { + Ok(Recovery::Disabled) + } + + /// Decode an explicit recovery mapping without hiding field-specific validation errors. + fn visit_map(self, map: M) -> Result + where + M: serde::de::MapAccess<'de>, + { + let settings = RecoverySettings::deserialize(serde::de::value::MapAccessDeserializer::new(map))?; + Ok(Recovery::Configured(Box::new(settings))) + } +} + +/// Recovery policy and application bindings; Kubernetes credentials and storage paths are generated, not user input. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +#[non_exhaustive] +pub struct RecoverySettings { + /// Job identity emitted by all participating application modules. + pub job_id: String, + /// Observation is the default; intervention must be explicitly enabled. + #[serde(default)] + pub mode: Mode, + /// Infrastructure observation interval. + pub poll_seconds: u64, + /// Deadline for one Kubernetes command. + pub command_timeout_seconds: u64, + /// Shared, bounded recovery policy. + pub policy: Policy, + /// Existing, continuous native-Python deployments and their application evidence identities. + pub workloads: Vec, + /// Limits for the supervisor's own container. + #[serde(default, skip_serializing_if = "Limits::is_unset")] + pub limits: Limits, +} + +impl RecoverySettings { + /// Construct the same validated runtime configuration a standalone operator supplies through the environment. + pub fn config(&self, namespace: &str) -> Result { + let mut config = Config::in_cluster( + hub_service_ws_url(), + namespace.into(), + self.job_id.clone(), + self.policy.clone(), + self.workloads.clone(), + ); + config.mode = self.mode; + config.poll_seconds = self.poll_seconds; + config.command_timeout_seconds = self.command_timeout_seconds; + config.validate()?; + Ok(config) + } + + /// Environment variables consumed by the supervisor image. + pub fn environment(&self, namespace: &str) -> Result, CliError> { + let config = self.config(namespace)?; + let mut env = BTreeMap::new(); + let values: BTreeMap = serde_json::from_value(serde_json::to_value(config)?)?; + for (key, value) in values { + if key == "policy" { + let policy: BTreeMap = serde_json::from_value(value)?; + for (setting, number) in policy { + let _previous = env.insert( + format!("RECOVERY_POLICY_{}", setting.to_uppercase()), + number.to_string(), + ); + } + } else { + let text = value.as_str().map_or_else(|| value.to_string(), str::to_owned); + let _previous = env.insert(format!("RECOVERY_{}", key.to_uppercase()), text); + } + } + Ok(env) + } +} + +/// Refuse unsupported output formats instead of silently omitting the requested recovery service. +pub(crate) fn validate_formats(cluster: &ClusterInput, formats: &[OutputType]) -> Result<(), CliError> { + if !cluster.recovery.is_unset() && formats.iter().any(|format| *format != OutputType::K3s) { + return Err(CliError::Recovery("recovery currently requires k3s output".into())); + } + Ok(()) +} + +/// Remove only obsolete generated launchers after a recovery-enabled K3s deployment has been written successfully. +pub(crate) fn remove_unsupported_outputs(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { + if cluster.recovery.is_unset() { + return Ok(()); + } + for format in OutputType::ALL.iter().filter(|format| **format != OutputType::K3s) { + if let Err(error) = fs::remove_file(output_dir.join(format.output_file_name())) + && error.kind() != std::io::ErrorKind::NotFound + { + return Err(error.into()); + } + } + Ok(()) +} + +/// Bind intervention to declared deployments with stable agent identities, never arbitrary Kubernetes names. +pub(crate) fn validate(cluster: &ClusterInput, runners: &[RunnerInstance]) -> Result<(), CliError> { + let Recovery::Configured(settings) = &cluster.recovery else { + return Ok(()); + }; + let _config = settings.config(&format!("et-{}", cluster.cluster_name))?; + crate::limits::validate_limits("recovery-supervisor", &settings.limits)?; + if runners.iter().any(|runner| runner.name == "recovery-supervisor") { + return Err(CliError::Recovery( + "recovery-supervisor is reserved when recovery is enabled".into(), + )); + } + for workload in &settings.workloads { + validate_workload(settings, workload, runners)?; + } + Ok(()) +} + +/// Check one declared runner's kind and identity without mixing scenario-wide validation into it. +#[expect( + clippy::single_call_fn, + reason = "validate each workload binding as a distinct scenario-generation step" +)] +fn validate_workload( + settings: &RecoverySettings, + workload: &Workload, + runners: &[RunnerInstance], +) -> Result<(), CliError> { + let runner = runners + .iter() + .find(|runner| runner.name == workload.deployment) + .ok_or_else(|| CliError::Recovery(format!("{} is not a declared runner", workload.deployment)))?; + if runner.runner != "pyo3" || !runner.schedule.is_unset() { + return Err(CliError::Recovery( + "supervised runners must be continuous native-Python deployments".into(), + )); + } + for (name, expected) in bindings(settings, workload) { + if runner.env.get(&name).is_some_and(|value| *value != expected) { + return Err(CliError::Recovery(format!( + "{} conflicts with generated {name}", + runner.name + ))); + } + } + Ok(()) +} + +/// Stable runner identity and shared job settings made available to recovery-aware modules. +pub(crate) fn bindings(settings: &RecoverySettings, workload: &Workload) -> BTreeMap { + BTreeMap::from([ + ("PYO3_AGENT_ID".into(), workload.agent_id.clone()), + ("RECOVERY_JOB_ID".into(), settings.job_id.clone()), + ("RECOVERY_SUPERVISOR_AGENT_ID".into(), "recovery-supervisor".into()), + ]) +} + +/// Include the service image in the ordinary image preparation steps before Kubernetes applies the scenario. +pub(crate) fn image_note(cluster: &ClusterInput) -> String { + if cluster.recovery.is_unset() { + return String::default(); + } + if matches!(cluster.artifact_source, crate::ArtifactSource::Published) { + return format!( + "The cluster also pulls `{}/et-recovery-supervisor:latest`.\n\n", + crate::IMAGE_REGISTRY + ); + } + concat!( + "Build and import the recovery service image from the repository root:\n\n", + "```bash\n", + "image=et-recovery-supervisor:latest\n", + "docker build -t \"$image\" -f services/recovery-supervisor/Dockerfile .\n", + "docker save \"$image\" | sudo k3s ctr images import -\n", + "```\n\n", + "The supervisor starts automatically with scoped ServiceAccount credentials and persistent state.\n", + "Application modules must implement lifecycle reporting and any required checkpoint restoration.\n\n" + ) + .into() +} diff --git a/utilities/cli/tests/recovery.rs b/utilities/cli/tests/recovery.rs new file mode 100644 index 00000000..e38c6a5d --- /dev/null +++ b/utilities/cli/tests/recovery.rs @@ -0,0 +1,451 @@ +//! Recovery scenarios generate deployable, scoped supervision without changing scenarios that do not request it. +#![cfg(test)] + +use std::path::{Path, PathBuf}; + +use et_cli::{OutputType, RecoverySettings, generate_deployment}; +use et_recovery_supervisor::config::{EnvConfig, Mode}; +use fs_err as fs; +use serde::Deserialize as _; +use serde_json::{Value, json}; + +#[derive(serde::Deserialize)] +struct ResourceHeader { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, +} + +/// Read the Kubernetes metadata name through a typed resource header. +fn object_name(object: &Value) -> String { + let header: ResourceHeader = serde_json::from_value(object.clone()).unwrap(); + header.metadata.name.unwrap() +} + +/// Load the checked-in recovery scenario used by deployment-generation tests. +fn scenario() -> Value { + let file = edge_toolkit::config::get_project_root().join("verification/local/input/pyo3-math1-recovery.yaml"); + serde_yaml::from_str(&fs::read_to_string(file).unwrap()).unwrap() +} + +/// Serialize a scenario mutation into an isolated input file. +fn write_input(root: &Path, input: &Value) -> PathBuf { + let file = root.join("scenario.yaml"); + fs::write(&file, serde_yaml::to_string(input).unwrap()).unwrap(); + file +} + +/// Generate recovery manifests and decode every resulting Kubernetes document. +fn generate(input: &Value) -> (tempfile::TempDir, Vec) { + let root = tempfile::tempdir().unwrap(); + let file = write_input(root.path(), input); + let output = root.path().join("output"); + let _summary = generate_deployment(&file, &output, Some(OutputType::K3s)).unwrap(); + let text = fs::read_to_string(output.join("k3s.yaml")).unwrap(); + let objects = serde_yaml::Deserializer::from_str(&text) + .map(|doc| Value::deserialize(doc).unwrap()) + .collect(); + (root, objects) +} + +/// Return the generation failure for an invalid scenario mutation. +fn generation_error(input: &Value) -> et_cli::CliError { + let root = tempfile::tempdir().unwrap(); + let file = write_input(root.path(), input); + generate_deployment(&file, &root.path().join("output"), Some(OutputType::K3s)).unwrap_err() +} + +/// Find one required Kubernetes document by kind and metadata name. +fn object<'objects>(objects: &'objects [Value], kind: &str, name: &str) -> &'objects Value { + objects + .iter() + .find(|object| object["kind"] == kind && object_name(object) == name) + .unwrap() +} + +/// Extract a deployment's rendered container settings for serde-env round trips. +fn environment(deployment: &Value) -> Vec<(String, String)> { + deployment["spec"]["template"]["spec"]["containers"][0]["env"] + .as_array() + .unwrap() + .iter() + .map(|item| { + ( + item["name"].as_str().unwrap().into(), + item["value"].as_str().unwrap().into(), + ) + }) + .collect() +} + +/// Ensure scenarios without recovery do not acquire a supervisor deployment. +fn assert_no_supervisor(objects: &[Value]) { + assert!( + !objects + .iter() + .any(|object| object_name(object) == "recovery-supervisor") + ); +} + +/// Round-trip generated settings and verify least-privilege in-cluster credentials. +#[test] +fn generates_valid_supervisor_settings_and_scoped_in_cluster_credentials() { + let (_root, objects) = generate(&scenario()); + let deployment = object(&objects, "Deployment", "recovery-supervisor"); + let env = environment(deployment); + let config: EnvConfig = serde_env::from_iter(env).unwrap(); + config.recovery.validate().unwrap(); + assert_eq!(config.recovery.context, "in-cluster"); + assert_eq!(config.recovery.mode, Mode::ObserveOnly); + assert_eq!(config.recovery.namespace, "et-pyo3-math1-recovery"); + assert_eq!(config.recovery.workloads[0].deployment, "example-worker"); + let role = object(&objects, "Role", "recovery-supervisor"); + assert_eq!( + role["rules"], + json!([{ + "apiGroups": ["apps"], "resources": ["deployments"], + "resourceNames": ["example-worker"], "verbs": ["get"] + }]) + ); + let binding = object(&objects, "RoleBinding", "recovery-supervisor"); + assert_eq!(binding["roleRef"]["kind"], "Role"); + assert_eq!(binding["subjects"][0]["namespace"], "et-pyo3-math1-recovery"); + let credentials = object(&objects, "ConfigMap", "recovery-credentials")["data"]["kubeconfig"] + .as_str() + .unwrap(); + let kubeconfig: Value = serde_yaml::from_str(credentials).unwrap(); + assert_eq!(kubeconfig["current-context"], "in-cluster"); + assert_eq!( + kubeconfig["users"][0]["user"]["tokenFile"], + "/var/run/secrets/kubernetes.io/serviceaccount/token" + ); + assert!(kubeconfig["users"][0]["user"]["token"].is_null()); + assert!( + !objects + .iter() + .any(|object| object["kind"] == "ClusterRole" || object["kind"] == "ClusterRoleBinding") + ); +} + +/// Preserve recovery state and prevent overlapping supervisor or checkpoint-writer replacements. +#[test] +fn budgets_persist_and_replacements_do_not_overlap_during_rollouts() { + let (_root, objects) = generate(&scenario()); + let deployment = object(&objects, "Deployment", "recovery-supervisor"); + assert_eq!(deployment["spec"]["replicas"], 1_u32); + assert_eq!(deployment["spec"]["strategy"]["type"], "Recreate"); + let pod = &deployment["spec"]["template"]["spec"]; + assert_eq!(pod["serviceAccountName"], "recovery-supervisor"); + assert_eq!(pod["automountServiceAccountToken"].as_bool(), Some(true)); + assert_eq!(pod["securityContext"]["fsGroup"], 10_001_i64); + assert_eq!( + pod["containers"][0]["securityContext"]["readOnlyRootFilesystem"].as_bool(), + Some(true) + ); + assert_eq!( + pod["volumes"][0]["persistentVolumeClaim"]["claimName"], + "recovery-state" + ); + let _claim = object(&objects, "PersistentVolumeClaim", "recovery-state"); + let worker = object(&objects, "Deployment", "example-worker"); + assert_eq!(worker["spec"]["strategy"]["type"], "Recreate"); + let vars: std::collections::BTreeMap<_, _> = environment(worker).into_iter().collect(); + assert_eq!(vars["PYO3_AGENT_ID"], "example-worker"); + assert_eq!(vars["RECOVERY_JOB_ID"], "example-job"); + assert_eq!(vars["RECOVERY_SUPERVISOR_AGENT_ID"], "recovery-supervisor"); +} + +/// Use the session-bound health executable for every generated Kubernetes probe. +#[test] +fn probes_use_the_same_session_check_for_startup_readiness_and_liveness() { + let (_root, objects) = generate(&scenario()); + let deployment = object(&objects, "Deployment", "recovery-supervisor"); + let container = &deployment["spec"]["template"]["spec"]["containers"][0]; + for name in ["startupProbe", "readinessProbe", "livenessProbe"] { + assert_eq!(container[name]["exec"]["command"], json!(["et-recovery-healthcheck"])); + assert_eq!(container[name]["timeoutSeconds"], 3_i32); + } + assert_eq!(container["startupProbe"]["failureThreshold"], 62_i32); +} + +/// Scale startup allowance with policy bounds and reject durations Kubernetes cannot represent. +#[test] +fn startup_probe_budget_scales_with_workload_settings_and_rejects_unrepresentable_durations() { + let mut input = scenario(); + input["recovery"]["poll_seconds"] = json!(5_u64); + input["recovery"]["command_timeout_seconds"] = json!(20_u64); + let (_root, objects) = generate(&input); + let deployment = object(&objects, "Deployment", "recovery-supervisor"); + assert_eq!( + deployment["spec"]["template"]["spec"]["containers"][0]["startupProbe"]["failureThreshold"], + 85_i32 + ); + input["recovery"]["poll_seconds"] = json!(u64::MAX); + assert!( + generation_error(&input) + .to_string() + .contains("startup probe duration exceeds Kubernetes limits") + ); +} + +/// Grant mutation rights only in assisted mode and leave unrelated agents unmodified. +#[test] +fn intervention_permissions_are_opt_in_and_other_agents_remain_unchanged() { + let mut input = scenario(); + input["recovery"]["mode"] = json!("controller-assisted"); + input["recovery"]["limits"] = json!({"cpus": 1_u32, "memory": "256 MiB"}); + let mut other = input["agents"][0].clone(); + other["name"] = json!("other-worker"); + input["agents"].as_array_mut().unwrap().push(other); + let (_root, objects) = generate(&input); + assert_eq!( + object(&objects, "Role", "recovery-supervisor")["rules"][0]["verbs"], + json!(["get", "patch"]) + ); + let other = object(&objects, "Deployment", "other-worker"); + assert!(other["spec"]["strategy"].is_null()); + assert!(!environment(other).iter().any(|(key, _)| key == "PYO3_AGENT_ID")); + let pod = &object(&objects, "Deployment", "recovery-supervisor")["spec"]["template"]["spec"]; + assert_eq!(pod["containers"][0]["resources"]["limits"]["memory"], "256Mi"); +} + +/// Render supervised continuous workers and unrelated scheduled runners through the same k3s generation loop. +#[test] +fn scheduled_non_supervised_runners_keep_their_cronjob_without_recovery_bindings() { + let mut input = scenario(); + let mut auxiliary = input["agents"][0].clone(); + auxiliary["name"] = json!("auxiliary-worker"); + input["agents"].as_array_mut().unwrap().push(auxiliary); + let mut scheduled = input["agents"][0].clone(); + scheduled["name"] = json!("scheduled-worker"); + scheduled["schedule"] = json!({"cron":"*/5 * * * *"}); + input["agents"].as_array_mut().unwrap().push(scheduled); + let (_root, objects) = generate(&input); + let cronjob = object(&objects, "CronJob", "scheduled-worker"); + assert_eq!(cronjob["spec"]["schedule"], "*/5 * * * *"); + let pod = &cronjob["spec"]["jobTemplate"]["spec"]["template"]["spec"]; + assert_eq!(pod["containers"][0]["name"], "scheduled-worker"); + assert_eq!(pod["containers"][0]["image"], "et-ws-pyo3-runner:latest"); + assert!( + pod["containers"][0]["env"] + .as_array() + .unwrap() + .iter() + .all(|item| { item["name"] != "RECOVERY_SUPERVISOR_AGENT_ID" && item["name"] != "RECOVERY_JOB_ID" }) + ); + assert_eq!( + object(&objects, "Deployment", "example-worker")["spec"]["strategy"]["type"], + "Recreate" + ); + assert!( + environment(object(&objects, "Deployment", "auxiliary-worker")) + .iter() + .all(|(name, _value)| !name.starts_with("RECOVERY_")) + ); + let config: EnvConfig = + serde_env::from_iter(environment(object(&objects, "Deployment", "recovery-supervisor"))).unwrap(); + assert_eq!(config.recovery.workloads.len(), 1); + assert_eq!(config.recovery.workloads[0].deployment, "example-worker"); +} + +/// A bare hub must still emit its infrastructure when the runner-generation loop has no workloads. +#[test] +fn no_runners_generate_only_hub_infrastructure_without_supervision() { + let mut input = scenario(); + input["agents"] = json!([]); + let _recovery = input.as_object_mut().unwrap().remove("recovery"); + let (_root, objects) = generate(&input); + assert_no_supervisor(&objects); + let deployments: Vec<_> = objects + .iter() + .filter(|object| object["kind"] == "Deployment") + .map(object_name) + .collect(); + assert_eq!(deployments, ["openobserve", "ws-server"]); + assert!(!objects.iter().any(|object| object["kind"] == "CronJob")); +} + +/// Validate the public runtime-settings conversion directly, including failures before manifest generation. +#[test] +fn runtime_settings_preserve_scenario_policy_and_reject_invalid_namespaces() { + let mut settings: RecoverySettings = serde_json::from_value(scenario()["recovery"].clone()).unwrap(); + settings.mode = Mode::ControllerAssisted; + settings.poll_seconds = 5; + settings.command_timeout_seconds = 20; + let config = settings.config("custom-namespace").unwrap(); + assert_eq!(config.mode, Mode::ControllerAssisted); + assert_eq!(config.namespace, "custom-namespace"); + assert_eq!(config.job_id, settings.job_id); + assert_eq!(config.poll_seconds, settings.poll_seconds); + assert_eq!(config.command_timeout_seconds, settings.command_timeout_seconds); + assert_eq!( + serde_json::to_value(&config.policy).unwrap(), + serde_json::to_value(&settings.policy).unwrap() + ); + assert_eq!( + serde_json::to_value(&config.workloads).unwrap(), + serde_json::to_value(&settings.workloads).unwrap() + ); + assert!( + settings + .config("invalid namespace") + .unwrap_err() + .to_string() + .contains("namespace") + ); + let mut invalid = settings; + invalid.poll_seconds = 0; + assert!(matches!( + invalid.config("custom-namespace"), + Err(et_cli::CliError::RecoveryConfig(_)) + )); +} + +/// Keep omitted recovery inert and use published supervisor images for released scenarios. +#[test] +fn omitted_recovery_emits_no_supervisor_and_published_recovery_uses_released_image_names() { + let mut input = scenario(); + let _removed = input.as_object_mut().unwrap().remove("recovery"); + let (_root, objects) = generate(&input); + assert_no_supervisor(&objects); + let mut input = scenario(); + input["artifact_source"] = json!("published"); + let (root, objects) = generate(&input); + assert_eq!( + object(&objects, "Deployment", "recovery-supervisor")["spec"]["template"]["spec"]["containers"][0]["image"], + "ghcr.io/edge-toolkit/core/et-recovery-supervisor:latest" + ); + let readme = fs::read_to_string(root.path().join("output/README.md")).unwrap(); + assert!(readme.contains("et-recovery-supervisor:latest")); +} + +/// Reject invalid policies, unsupported runner types and conflicting bindings before writing output. +#[test] +fn invalid_bindings_and_policies_fail_before_creating_output() { + for (key, value) in [ + ("poll_seconds", json!(0_u64)), + ("workloads", json!([])), + ( + "workloads", + json!([{"deployment":"outside","agent_id":"a","source":"/modules/a"}]), + ), + ( + "policy", + json!({"detection_grace_seconds": 3_u32, "restart_after_seconds":20_u32, + "stall_seconds":180_u32, "deadline_seconds":1_u32, "max_restarts":2_u32}), + ), + ] { + let mut input = scenario(); + input["recovery"][key] = value; + rejects(&input, OutputType::K3s); + } + for name in ["PYO3_AGENT_ID", "RECOVERY_JOB_ID", "RECOVERY_SUPERVISOR_AGENT_ID"] { + let mut input = scenario(); + input["agents"][0]["env"] = json!({name: "conflicting"}); + rejects(&input, OutputType::K3s); + } + for kind in ["wasi", "web"] { + let mut input = scenario(); + input["agents"][0]["runner"] = json!(kind); + rejects(&input, OutputType::K3s); + } + let mut input = scenario(); + input["agents"][0]["schedule"] = json!({"cron":"*/5 * * * *"}); + rejects(&input, OutputType::K3s); + let mut input = scenario(); + input["agents"][0]["name"] = json!("recovery-supervisor"); + rejects(&input, OutputType::K3s); + let mut input = scenario(); + let duplicate = input["recovery"]["workloads"][0].clone(); + input["recovery"]["workloads"].as_array_mut().unwrap().push(duplicate); + rejects(&input, OutputType::K3s); + for kind in [ + OutputType::Mise, + OutputType::DockerCompose, + OutputType::ProcessCompose, + OutputType::Dekit, + ] { + rejects(&scenario(), kind); + } +} + +/// Preserve compatible explicit identities and prepare requested container-limit support. +#[test] +fn matching_identity_settings_are_preserved_and_supervisor_limits_prepare_plugins() { + let mut input = scenario(); + input["agents"][0]["env"] = json!({"PYO3_AGENT_ID":"example-worker"}); + input["recovery"]["limits"] = json!({"pids": 100_u64}); + let (_root, objects) = generate(&input); + assert!(objects.iter().any(|item| item["kind"] == "DaemonSet")); + let vars: std::collections::BTreeMap<_, _> = environment(object(&objects, "Deployment", "example-worker")) + .into_iter() + .collect(); + assert_eq!(vars["PYO3_AGENT_ID"], "example-worker"); +} + +/// Require unsupported scenario generation to fail without creating its output directory. +fn rejects(input: &Value, kind: OutputType) { + let root = tempfile::tempdir().unwrap(); + let file = write_input(root.path(), input); + let output = root.path().join("output"); + let _error = generate_deployment(&file, &output, Some(kind)).unwrap_err(); + assert!(!output.exists()); +} + +/// Preserve useful field errors and distinguish explicit null from malformed recovery settings. +#[test] +fn malformed_recovery_fields_keep_actionable_errors_and_null_remains_disabled() { + let mut input = scenario(); + input["recovery"]["limits"] = json!({"pids_limit": 100_u64}); + assert!(generation_error(&input).to_string().contains("pids_limit")); + input["recovery"] = Value::Null; + let (_root, objects) = generate(&input); + assert_no_supervisor(&objects); + input["recovery"] = json!(true); + assert!( + generation_error(&input) + .to_string() + .contains("recovery configuration mapping") + ); +} + +/// Place the supervisor on the hub machine and include its image in deployment instructions. +#[test] +fn multi_machine_instructions_include_the_supervisor_image_on_the_server() { + let mut input = scenario(); + input["machines"] = json!([ + {"name":"server", "address":"192.168.1.10"}, + {"name":"worker", "address":"192.168.1.11", "agents":["example-worker"]} + ]); + let (root, objects) = generate(&input); + let readme = fs::read_to_string(root.path().join("output/README.md")).unwrap(); + assert!(readme.contains("docker save et-recovery-supervisor:latest | ssh 192.168.1.10")); + let supervisor = object(&objects, "Deployment", "recovery-supervisor"); + let label = format!("{}/machine", et_org::org!()); + assert_eq!(supervisor["spec"]["template"]["spec"]["nodeSelector"][&label], "server"); +} + +/// Bind multiple declared workers to one supervisor without implicitly enabling intervention. +#[test] +fn one_supervisor_binds_multiple_workloads_and_defaults_to_observation() { + let mut input = scenario(); + let _mode = input["recovery"].as_object_mut().unwrap().remove("mode"); + input["agents"].as_array_mut().unwrap().push(json!({ + "name":"second-worker", "runner":"pyo3", "resources":[{"type":"pyo3-math1"}] + })); + input["recovery"]["workloads"].as_array_mut().unwrap().push(json!({ + "deployment":"second-worker", "agent_id":"second-client", "source":"/modules/second" + })); + let (_root, objects) = generate(&input); + let role = object(&objects, "Role", "recovery-supervisor"); + assert_eq!( + role["rules"][0]["resourceNames"], + json!(["example-worker", "second-worker"]) + ); + let config: EnvConfig = + serde_env::from_iter(environment(object(&objects, "Deployment", "recovery-supervisor"))).unwrap(); + assert_eq!(config.recovery.mode, Mode::ObserveOnly); + assert_eq!(config.recovery.workloads.len(), 2); + let vars: std::collections::BTreeMap<_, _> = environment(object(&objects, "Deployment", "second-worker")) + .into_iter() + .collect(); + assert_eq!(vars["PYO3_AGENT_ID"], "second-client"); +} diff --git a/utilities/cli/tests/recovery_regeneration.rs b/utilities/cli/tests/recovery_regeneration.rs new file mode 100644 index 00000000..9b002964 --- /dev/null +++ b/utilities/cli/tests/recovery_regeneration.rs @@ -0,0 +1,138 @@ +//! Recovery regeneration replaces unsupported launchers without removing unrelated user files. +#![cfg(test)] + +use std::path::{Path, PathBuf}; + +use et_cli::{OutputType, regenerate_verification}; +use fs_err as fs; +use serde_json::{Value, json}; + +/// Decode the reference scenario as a mutable document for regeneration transitions. +fn scenario() -> Value { + let root = edge_toolkit::config::get_project_root(); + let document = fs::read_to_string(root.join("verification/local/input/pyo3-math1-recovery.yaml")).unwrap(); + serde_yaml::from_str(&document).unwrap() +} + +/// Write a scenario document into the verification input tree. +fn write_input(root: &Path, input: &Value) -> PathBuf { + let path = root.join("scenario.yaml"); + let document = serde_yaml::to_string(input).unwrap(); + fs::write(&path, document).unwrap(); + path +} + +/// Generate only supported verification formats for recovery-enabled scenarios. +#[test] +fn verification_generation_limits_recovery_scenarios_to_supported_formats() { + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("local/input"); + fs::create_dir_all(&input).unwrap(); + let _file = write_input(&input, &scenario()); + let generated = regenerate_verification(root.path(), None).unwrap(); + assert_eq!(generated.len(), 1); + let output = &generated[0].output_dir; + assert!(output.join("k3s.yaml").is_file()); + assert!(!output.join("mise.toml").exists()); + let readme = fs::read_to_string(output.join("README.md")).unwrap(); + assert!(readme.contains("services/recovery-supervisor/Dockerfile")); +} + +/// Start with actual outputs from every generator, rather than fabricated files that may not match their layout. +fn verification_without_recovery() -> (tempfile::TempDir, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let input_dir = root.path().join("local/input"); + fs::create_dir_all(&input_dir).unwrap(); + let mut input = scenario(); + input["recovery"] = Value::Null; + let _file = write_input(&input_dir, &input); + let generated = regenerate_verification(root.path(), None).unwrap(); + let output = generated.first().unwrap().output_dir.clone(); + for kind in OutputType::ALL { + assert!(output.join(kind.output_file_name()).is_file()); + } + (root, output) +} + +/// Snapshot the launcher files that a recovery-free K3s-only regeneration must leave untouched. +fn non_k3s_snapshot(output: &Path) -> Vec<(PathBuf, Vec)> { + OutputType::ALL + .iter() + .filter(|kind| **kind != OutputType::K3s) + .map(|kind| { + let path = output.join(kind.output_file_name()); + let bytes = fs::read(&path).unwrap(); + (path, bytes) + }) + .collect() +} + +/// Enabling recovery removes obsolete launchers but preserves shared image files and unrelated user content. +#[test] +fn recovery_regeneration_removes_only_stale_non_k3s_launchers() { + let (root, output) = verification_without_recovery(); + fs::write(output.join("notes.txt"), b"keep notes").unwrap(); + let custom = output.join("custom"); + fs::create_dir(&custom).unwrap(); + fs::write(custom.join("compose.yaml"), b"keep custom deployment").unwrap(); + let _file = write_input(&root.path().join("local/input"), &scenario()); + let regenerated = regenerate_verification(root.path(), None).unwrap(); + assert_eq!(regenerated.len(), 1); + for kind in OutputType::ALL { + assert_eq!(output.join(kind.output_file_name()).is_file(), *kind == OutputType::K3s); + } + for name in [ + "Dockerfile", + "Dockerfile.dockerignore", + "README.md", + "secrets.env", + ".gitignore", + ] { + assert!(output.join(name).is_file(), "missing shared artifact {name}"); + } + assert_eq!(fs::read(output.join("notes.txt")).unwrap(), b"keep notes"); + assert_eq!( + fs::read(custom.join("compose.yaml")).unwrap(), + b"keep custom deployment" + ); + let text = fs::read_to_string(output.join("k3s.yaml")).unwrap(); + assert!(text.contains("recovery-supervisor")); + let _regenerated = regenerate_verification(root.path(), Some(OutputType::K3s)).unwrap(); + assert!(output.join("k3s.yaml").is_file()); +} + +/// Selecting K3s alone does not remove the other generated formats when recovery remains disabled. +#[test] +fn regeneration_preserves_other_formats_without_recovery() { + let (root, output) = verification_without_recovery(); + let before = non_k3s_snapshot(&output); + let _regenerated = regenerate_verification(root.path(), Some(OutputType::K3s)).unwrap(); + assert_eq!(non_k3s_snapshot(&output), before); +} + +/// Invalid recovery settings must fail validation before existing output files can be removed or overwritten. +#[test] +fn invalid_recovery_does_not_remove_existing_launchers() { + let (root, output) = verification_without_recovery(); + let before = non_k3s_snapshot(&output); + let mut input = scenario(); + input["recovery"]["job_id"] = json!(""); + let _file = write_input(&root.path().join("local/input"), &input); + let _error = regenerate_verification(root.path(), None).unwrap_err(); + assert_eq!(non_k3s_snapshot(&output), before); +} + +/// A blocked obsolete path is an error, not permission to remove a directory or silently retain a bypass launcher. +#[test] +fn recovery_cleanup_reports_blocked_paths_without_removing_directory_contents() { + let (root, output) = verification_without_recovery(); + let blocked = output.join("mise.toml"); + fs::remove_file(&blocked).unwrap(); + fs::create_dir(&blocked).unwrap(); + fs::write(blocked.join("notes.txt"), b"keep nested data").unwrap(); + let _file = write_input(&root.path().join("local/input"), &scenario()); + let error = regenerate_verification(root.path(), None).unwrap_err(); + assert!(matches!(error, et_cli::CliError::Io(_))); + assert!(error.to_string().contains("mise.toml")); + assert_eq!(fs::read(blocked.join("notes.txt")).unwrap(), b"keep nested data"); +} diff --git a/utilities/int-gen/Cargo.toml b/utilities/int-gen/Cargo.toml index 5f196b78..0a46a770 100644 --- a/utilities/int-gen/Cargo.toml +++ b/utilities/int-gen/Cargo.toml @@ -24,6 +24,7 @@ edge-toolkit = { workspace = true, features = ["schema-export"] } et-cli.workspace = true et-modules-service = { workspace = true, features = ["openapi-spec"] } et-onnx.workspace = true +et-recovery-supervisor = { workspace = true, features = ["env-schema"] } et-storage-service = { workspace = true, features = ["openapi-spec"] } et-ws-runner-common = { workspace = true, features = ["env-schema"] } et-ws-server = { workspace = true, features = ["env-schema", "openapi-spec"] } diff --git a/utilities/int-gen/src/help.rs b/utilities/int-gen/src/help.rs index de18fd93..48ccc2cc 100644 --- a/utilities/int-gen/src/help.rs +++ b/utilities/int-gen/src/help.rs @@ -1,12 +1,12 @@ //! Render each binary crate's `HELP.md`: its clap command tree, and the environment variables its config reads. //! -//! The crates are found from the workspace manifests rather than listed by hand. A crate with a binary target that -//! depends on `clap` gets the clap section, which clap-markdown writes from the command tree linked in -//! [`clap_commands`]. One that loads its config with serde-env -- directly, or through `et-otlp`'s -//! `load_telemetered` -- gets the environment section, rendered here from the config's JSON Schema linked in -//! [`env_schemas`], with every nested field named the way serde-env reads it: `ws.max_frame_size` is -//! `WS_MAX_FRAME_SIZE`. A crate this finds but has no tree or schema for fails the run, naming what is missing, so a -//! new binary cannot quietly go without a `HELP.md`; so does an entry left behind for a crate that no longer needs one. +//! The crates are found from the workspace manifests rather than listed by hand. A crate with a binary target +//! that depends on `clap` gets the clap section, which clap-markdown writes from the command tree linked in +//! [`clap_commands`]. One that loads its config with serde-env -- directly, or through `et-otlp`'s `load_telemetered` +//! -- gets the environment section, rendered here from the config's JSON Schema linked in [`env_schemas`], with every +//! nested field named the way serde-env reads it: `ws.max_frame_size` is `WS_MAX_FRAME_SIZE`. A crate this finds +//! but has no tree or schema for fails the run, naming what is missing, so a new binary cannot quietly go without a +//! `HELP.md`; so does an entry left behind for a crate that no longer needs one. //! //! The result goes through the repo's markdown formatter, so each file is byte-identical to what that formatter would //! leave on disk. @@ -69,6 +69,10 @@ pub fn env_schemas() -> Result, Error> { use et_ws_runner_common::{pyo3_config, wasi_config, web_config}; use schemars::schema_for; Ok(BTreeMap::from([ + ( + "et-recovery-supervisor", + serde_json::to_value(schema_for!(et_recovery_supervisor::config::EnvConfig))?, + ), ( "et-ws-pyo3-runner", serde_json::to_value(schema_for!(pyo3_config::Config))?, @@ -88,7 +92,7 @@ pub fn env_schemas() -> Result, Error> { ])) } -/// Every workspace member with a binary target, read from the manifests and sorted by name. +/// Every workspace member with a documented binary target, read from the manifests and sorted by name. /// /// The manifests are read directly rather than through `cargo metadata`, which takes the workspace lock. Only normal /// dependencies count, target-specific ones included, so a crate using clap or serde-env in its tests alone is left @@ -138,33 +142,65 @@ fn read_manifest(path: &Path) -> Result { Ok(toml::from_str(&fs_err::read_to_string(path)?)?) } -/// The names of a crate's binary targets, in the order cargo lists them. +/// The names of a crate's documented binary targets, in the order cargo lists them. /// -/// Explicit `[[bin]]` tables come first; otherwise cargo's auto-discovery applies, which names `src/main.rs` after the -/// package and each `src/bin/*.rs` after its file stem. +/// Explicit `[[bin]]` tables honour `doc = false` and override matching discovered targets. Modern editions also +/// discover conventional binary paths unless `autobins = false` disables discovery. fn binary_names(manifest: &toml::Table, dir: &Path, package: &str) -> Result, Error> { - let explicit: Vec = manifest + let binaries = manifest .get("bin") .and_then(toml::Value::as_array) - .into_iter() - .flatten() + .map_or(&[][..], Vec::as_slice); + let mut names: Vec = binaries + .iter() + .filter(|bin| bin.get("doc").and_then(toml::Value::as_bool) != Some(false)) .filter_map(|bin| bin.get("name").and_then(toml::Value::as_str)) .map(str::to_owned) .collect(); - if !explicit.is_empty() { - return Ok(explicit); + let settings = &manifest["package"]; + let legacy = settings + .get("edition") + .is_none_or(|edition| edition.as_str() == Some("2015")); + let default_autobins = !legacy || binaries.is_empty(); + let discover = settings + .get("autobins") + .and_then(toml::Value::as_bool) + .unwrap_or(default_autobins); + if discover { + names.extend( + discovered_binaries(dir, package)? + .into_iter() + .filter_map(|(name, path)| { + let overridden = binaries.iter().any(|bin| { + bin.get("name").and_then(toml::Value::as_str) == Some(name.as_str()) + || bin + .get("path") + .and_then(toml::Value::as_str) + .is_some_and(|explicit| Path::new(explicit) == path) + }); + (!overridden).then_some(name) + }), + ); } + Ok(names) +} + +/// Discover conventional binary names and their package-relative paths for explicit-target overrides. +fn discovered_binaries(dir: &Path, package: &str) -> Result, Error> { let mut discovered = Vec::new(); if dir.join("src/main.rs").is_file() { - discovered.push(package.to_owned()); + discovered.push((package.to_owned(), PathBuf::from("src/main.rs"))); } let bin_dir = dir.join("src/bin"); if bin_dir.is_dir() { - let mut stems: Vec = fs_err::read_dir(&bin_dir)? + let mut stems: Vec<(String, PathBuf)> = fs_err::read_dir(&bin_dir)? .filter_map(Result::ok) .map(|entry| entry.path()) .filter(|path| path.extension().is_some_and(|extension| extension == "rs")) - .filter_map(|path| path.file_stem().and_then(|stem| stem.to_str()).map(str::to_owned)) + .filter_map(|path| { + let name = path.file_stem()?.to_str()?.to_owned(); + Some((name, path.strip_prefix(dir).ok()?.to_path_buf())) + }) .collect(); stems.sort(); discovered.extend(stems); @@ -246,8 +282,8 @@ fn describe(krate: &BinaryCrate, command: Option, schema: Option< /// The clap-markdown rendering of `command`, reshaped into markdown that needs no inline HTML. /// /// The footer crediting clap-markdown is dropped, since it is the document's only HTML. Each `###### **Options:**` -/// style label becomes a `### Options` heading one level under its command's `##`, where clap-markdown otherwise -/// jumps straight from the second level to the sixth. +/// style label becomes a `### Options` heading one level under its command's `##`, where clap-markdown otherwise jumps +/// straight from the second level to the sixth. #[must_use] pub fn clap_tree(command: &clap::Command) -> String { let options = clap_markdown::MarkdownOptions::new().show_footer(false); diff --git a/utilities/int-gen/tests/help_md.rs b/utilities/int-gen/tests/help_md.rs index 8e7fa987..ddc1f0c5 100644 --- a/utilities/int-gen/tests/help_md.rs +++ b/utilities/int-gen/tests/help_md.rs @@ -4,6 +4,7 @@ use edge_toolkit::config::get_project_root; use et_int_gen::first_difference; use et_int_gen::help::{binary_crates, clap_tree, render_env_section}; +use fs_err as fs; use serde_json::json; #[test] @@ -52,6 +53,7 @@ fn detects_every_binary_using_clap_or_serde_env() { ("et-cli", true, false), ("et-int-gen", true, false), ("et-onnx", true, false), + ("et-recovery-supervisor", false, true), ("et-ws-pyo3-runner", false, true), ("et-ws-server", false, true), ("et-ws-wasi-runner", false, true), @@ -60,6 +62,45 @@ fn detects_every_binary_using_clap_or_serde_env() { ); } +/// Merge modern auto-discovery with explicit targets without rediscovering undocumented or overridden binaries. +#[test] +fn undocumented_fixture_binaries_do_not_enter_the_help_inventory() { + let root = tempfile::tempdir().unwrap(); + let fixture = root.path().join("fixture"); + fs::create_dir_all(fixture.join("src/bin")).unwrap(); + fs::write(root.path().join("Cargo.toml"), "[workspace]\nmembers = ['fixture']\n").unwrap(); + fs::write(fixture.join("src/main.rs"), "fn main() {}\n").unwrap(); + fs::write(fixture.join("src/bin/probe.rs"), "fn main() {}\n").unwrap(); + for (settings, extra, expected) in [ + ("edition = '2024'\n", "", vec!["fixture"]), + ("edition = '2024'\nautobins = false\n", "", vec![]), + ("", "", vec![]), + ("edition = '2024'\n", "[[bin]]\nname = 'visible'\n", vec!["visible"]), + ("edition = '2024'\n", "[[bin]]\nname = 'fixture'\ndoc = false\n", vec![]), + ( + "edition = '2024'\n", + "[[bin]]\nname = 'hidden-main'\npath = 'src/main.rs'\ndoc = false\n", + vec![], + ), + ] { + let manifest = [ + "[package]\nname = 'fixture'\n", + settings, + "[dependencies]\nclap = '4'\n", + "[[bin]]\nname = 'probe'\ndoc = false\n", + extra, + ] + .concat(); + fs::write(fixture.join("Cargo.toml"), manifest).unwrap(); + let names: Vec<_> = binary_crates(root.path()) + .unwrap() + .into_iter() + .map(|krate| krate.bin) + .collect(); + assert_eq!(names, expected); + } +} + #[test] fn renders_each_leaf_under_its_serde_env_name() { let long = ["word"; 30].join(" "); diff --git a/verification/local/input/pyo3-math1-recovery.yaml b/verification/local/input/pyo3-math1-recovery.yaml new file mode 100644 index 00000000..82911807 --- /dev/null +++ b/verification/local/input/pyo3-math1-recovery.yaml @@ -0,0 +1,25 @@ +cluster_name: "pyo3-math1-recovery" +deployment_type: "k3s" + +# Wiring example only: math1 has no recovery lifecycle reporting or checkpoint/resume implementation. +agents: + - name: "example-worker" + runner: "pyo3" + resources: + - type: "pyo3-math1" + +recovery: + job_id: "example-job" + mode: "observe-only" + poll_seconds: 2 + command_timeout_seconds: 10 + policy: + detection_grace_seconds: 3 + restart_after_seconds: 20 + stall_seconds: 180 + deadline_seconds: 240 + max_restarts: 2 + workloads: + - deployment: "example-worker" + agent_id: "example-worker" + source: "/modules/@edge-toolkit/et-ws-pyo3-math1" diff --git a/verification/local/output/pyo3-math1-recovery/.gitignore b/verification/local/output/pyo3-math1-recovery/.gitignore new file mode 100644 index 00000000..4b22f893 --- /dev/null +++ b/verification/local/output/pyo3-math1-recovery/.gitignore @@ -0,0 +1,7 @@ +# Written by `et-cli`. +# What the hub writes into the directory it runs from: its self-signed TLS pair (the key is a +# credential too), the agent registry it saves on shutdown, and agent storage. +cert.pem +key.pem +registry.yaml +storage/ diff --git a/verification/local/output/pyo3-math1-recovery/Dockerfile b/verification/local/output/pyo3-math1-recovery/Dockerfile new file mode 100644 index 00000000..fec8399c --- /dev/null +++ b/verification/local/output/pyo3-math1-recovery/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 + +# AUTO-GENERATED by `et-cli` for the `pyo3-math1-recovery` scenario -- do not edit. +# Regenerate with `mise run regen-verification`. +# +# The hub image (services/ws-server/Dockerfile) carries no workflow modules at all, because which +# modules a deployment serves is a per-cluster decision. This image is that decision made concrete: it +# stages exactly the modules this cluster's agents declare, plus the packages those modules depend on, +# into the `/app` paths the hub already scans. +# +# The sibling `Dockerfile.dockerignore` is what makes the COPYs below possible. The repository's root +# `.dockerignore` drops every `*.wasm` and `*.onnx` so the CI images built from `./Dockerfile` stay +# small; BuildKit uses this Dockerfile's own ignore file instead, which re-admits the built `pkg/` +# directories and nothing else. + +# `hub` is a named build context, not a stage defined here. +# The generated compose file wires it to its `hub-image-build` service, so compose builds the hub image +# first and hands the result over under this name. That is also why the version-tag rule is +# suppressed on the next line: hadolint reads `hub` as an untagged image, but there is no tag to give a +# build context, and the image it resolves to is pinned by the hub Dockerfile rather than here. +# hadolint ignore=DL3006 +FROM hub + +LABEL org.opencontainers.image.source="https://github.com/edge-toolkit/core" + +COPY --chown=10001:10001 services/ws-modules/pyo3-math1/pkg /app/services/ws-modules/pyo3-math1/pkg + +USER 10001 diff --git a/verification/local/output/pyo3-math1-recovery/Dockerfile.dockerignore b/verification/local/output/pyo3-math1-recovery/Dockerfile.dockerignore new file mode 100644 index 00000000..92c3adbb --- /dev/null +++ b/verification/local/output/pyo3-math1-recovery/Dockerfile.dockerignore @@ -0,0 +1,83 @@ +# AUTO-GENERATED by `et-cli` for the `pyo3-math1-recovery` scenario -- do not edit. +# Regenerate with `mise run regen-verification`. +# +# BuildKit reads this file instead of the repository root `.dockerignore` when building the +# Dockerfile beside it, so the root file keeps governing the CI images built from `./Dockerfile` +# and only this build sees the re-includes at the end. + +# AUTO-GENERATED from .gitignore by 'mise run gen:dockerignore' -- do not edit. +# Docker reads only this file; patterns are **/-prefixed to match at any depth +# like .gitignore. Edit .gitignore and regenerate. + +# `.dockerignore` is generated from `.gitignore` by `mise run gen:dockerignore`. +# After editing `.gitignore`, regenerate it and commit both, or the dockerignore-check task fails on drift. +**/.claude/ +**/*.wasm +**/*.onnx +**/target/ +**/.DS_Store +services/ws-wasm-agent/pkg/ +services/ws-modules/pywasm1/pkg/ +services/ws-modules/rdata1/pkg/webr/ +services/ws-modules/rcomm1/pkg/webr/ +services/ws-modules/rmath1/pkg/webr/ +services/ws-modules/js-data1/pkg/et_ws_js_data1.js +services/ws-server/static/models/ +**/.zig-cache/ +**/zig-out/ +**/*.o +**/*.pem +**/mprocs.log +**/__pycache__/ +**/.pytest_cache/ +**/.python-version +**/node_modules/ +**/.venv/ +# .NET build output. +# `obj/` is safe globally (nothing tracked is named obj/), but `bin/` is scoped to the module so it never +# matches a Rust crate's `src/bin/` (e.g. utilities/int-gen/src/bin/). +**/obj/ +services/ws-modules/dotnet-data1/bin/ +# Gradle project caches and build output plus the Kotlin session dir, all regenerated by any Gradle run. +**/.gradle/ +**/.kotlin/ +services/ws-modules/kotlin-data1/build/ +services/ws-modules/kotlin-math1/build/ +# Coverage-report artifacts produced by the coverage workflow, uploaded to Codecov and never committed. +# Covers cargo-llvm-cov's lcov, pytest-cov's Cobertura xml, and pytest-cov's .coverage data file. +lcov.info +**/coverage-python.xml +**/.coverage +# minicov/llvm raw profiles dropped by instrumented binaries run during coverage builds. +**/*.profraw +# Editor dir, tool caches, and the ws-server's runtime file storage. +# The .vscode dir is ignored except its shared recommended-extensions list (negated below). +.vscode/* +!.vscode/extensions.json +**/.ruff_cache/ +**/.lycheecache +# Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). +**/.dart_tool/ +services/ws-server/storage/ +# Lock files a generated scenario leaves behind when it is run. +# `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario +# drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather +# than part of it: `regen-verification` never writes one, the drift check never reads one, and what it pins is +# whatever the machine that ran it happened to resolve. One rule here rather than a `.gitignore` generated into +# each scenario, so a scenario added later is covered without anything having to be written into it. +verification/**/*.lock +# The npm dependency graphs `mise lock` records beside that lockfile, a by-product for the same reason. +verification/**/.mise/locks/ +# No rule for the per-scenario credential file, deliberately. +# Under verification/ it is committed like every other generated artifact: the password is derived from an +# input this repository already carries, so the file reproduces from public material rather than keeping a +# secret, and the drift check has to be able to read it. A deployment generated anywhere else gets its own +# `.gitignore` written beside it by `et-cli`, which is what keeps a real credential out of a real repository. +**/.git/ +**/Dockerfile* +README.md + +# Re-admit the built module artifacts this scenario serves. +# The rules above drop every `*.wasm` and `*.onnx`, which is what keeps the CI images small; these +# negations put back the `pkg/` directories this image copies, and nothing else. +!services/ws-modules/pyo3-math1/pkg/** diff --git a/verification/local/output/pyo3-math1-recovery/README.md b/verification/local/output/pyo3-math1-recovery/README.md new file mode 100644 index 00000000..b9000c80 --- /dev/null +++ b/verification/local/output/pyo3-math1-recovery/README.md @@ -0,0 +1,69 @@ +# pyo3-math1-recovery + +This directory contains the generated `k3s.yaml` for the `pyo3-math1-recovery` scenario. + +The scenario exposes these workflow modules: pyo3-math1. + +`secrets.env` holds the scenario's derived OpenObserve and OTLP credentials. It is derived from the +scenario input, so regenerating this deployment rewrites it; if it is missing, regenerate before +starting the stack. A deployment generated outside the repository is written with a `.gitignore` +covering it, so its credential is not committed by whatever repository it lands in. + +## Run With k3s + +The manifests reference images by name and never build them, so build and import each one first. +This scenario's image layers its modules onto the module-less hub image and takes that hub as a +_named build context_ rather than building it, so the hub has to exist first -- a plain +`docker build` of the scenario Dockerfile fails on `FROM hub`. From the repository root: + +```bash +scenario=pyo3-math1-recovery +hub=et-ws-server-hub:latest +image="et-ws-server-$scenario:latest" +dockerfile="verification/local/output/$scenario/Dockerfile" +docker build -t "$hub" -f services/ws-server/Dockerfile . +docker build --build-context "hub=docker-image://$hub" -t "$image" -f "$dockerfile" . +docker save "$image" | sudo k3s ctr images import - +``` + +Then this scenario's runner images, which need no build context of their own: + +```bash +docker build -t et-ws-pyo3-runner:latest -f services/ws-pyo3-runner/Dockerfile . +docker save et-ws-pyo3-runner:latest | sudo k3s ctr images import - +``` + +Build and import the recovery service image from the repository root: + +```bash +image=et-recovery-supervisor:latest +docker build -t "$image" -f services/recovery-supervisor/Dockerfile . +docker save "$image" | sudo k3s ctr images import - +``` + +The supervisor starts automatically with scoped ServiceAccount credentials and persistent state. +Application modules must implement lifecycle reporting and any required checkpoint restoration. + +### Load The Credential + +The credential reaches the pods as a `Secret` created from `secrets.env`, rather than written into +`k3s.yaml` where it would be committed alongside the manifests. From this directory: + +```bash +ns=et-pyo3-math1-recovery +kubectl create namespace "$ns" --save-config +kubectl create secret generic "$ns-secrets" --from-env-file=secrets.env -n "$ns" +``` + +### Apply + +```bash +kubectl apply -f k3s.yaml +``` + +The runners exit and restart until the hub reports ready, so `CrashLoopBackOff` while the hub +starts is expected here rather than a fault. Watch it settle with: + +```bash +kubectl get pods -n "$ns" --watch +``` diff --git a/verification/local/output/pyo3-math1-recovery/k3s.yaml b/verification/local/output/pyo3-math1-recovery/k3s.yaml new file mode 100644 index 00000000..f32a4447 --- /dev/null +++ b/verification/local/output/pyo3-math1-recovery/k3s.yaml @@ -0,0 +1,477 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: et-pyo3-math1-recovery +--- +apiVersion: v1 +kind: ConfigMap +data: + RUST_LOG: warn + ZO_DATA_DIR: /data + ZO_ROOT_USER_EMAIL: root@example.com +metadata: + name: openobserve-config + namespace: et-pyo3-math1-recovery +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: openobserve-data + namespace: et-pyo3-math1-recovery +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: openobserve + name: openobserve + namespace: et-pyo3-math1-recovery +spec: + replicas: 1 + selector: + matchLabels: + app: openobserve + template: + metadata: + labels: + app: openobserve + spec: + containers: + - envFrom: + - configMapRef: + name: openobserve-config + - secretRef: + name: et-pyo3-math1-recovery-secrets + image: openobserve/openobserve:v1.1.1 + imagePullPolicy: IfNotPresent + name: openobserve + ports: + - containerPort: 5080 + readinessProbe: + exec: + command: + - /openobserve + - node + - status + failureThreshold: 20 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /data + name: openobserve-data + - mountPath: /tmp + name: openobserve-tmp + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - name: openobserve-data + persistentVolumeClaim: + claimName: openobserve-data + - emptyDir: {} + name: openobserve-tmp +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: openobserve + name: openobserve + namespace: et-pyo3-math1-recovery +spec: + ports: + - name: http + port: 5080 + targetPort: 5080 + selector: + app: openobserve +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ws-server-storage + namespace: et-pyo3-math1-recovery +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ws-server + name: ws-server + namespace: et-pyo3-math1-recovery +spec: + replicas: 1 + selector: + matchLabels: + app: ws-server + template: + metadata: + labels: + app: ws-server + spec: + containers: + - env: + - name: MODULES_PATHS + value: /app/services/ws-modules/pyo3-math1 + - name: MODULES_ROOT + value: "" + - name: OTLP_AUTH_USERNAME + value: root@example.com + - name: OTLP_COLLECTOR_URL + value: http://openobserve:5080/api/default/v1 + - name: STORAGE_URL + value: file:///app/storage + - name: TLS_CERT_FILE + value: /app/runtime/cert.pem + - name: TLS_KEY_FILE + value: /app/runtime/key.pem + - name: WS_AGENT_REGISTRY + value: /app/runtime/registry.yaml + envFrom: + - secretRef: + name: et-pyo3-math1-recovery-secrets + image: et-ws-server-pyo3-math1-recovery:latest + imagePullPolicy: IfNotPresent + name: ws-server + ports: + - containerPort: 8080 + - containerPort: 8443 + readinessProbe: + failureThreshold: 20 + httpGet: + path: /health + port: 8080 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /app/storage + name: ws-server-storage + - mountPath: /app/runtime + name: ws-server-runtime + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - name: ws-server-storage + persistentVolumeClaim: + claimName: ws-server-storage + - emptyDir: {} + name: ws-server-runtime +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: ws-server + name: ws-server + namespace: et-pyo3-math1-recovery +spec: + ports: + - name: ws + port: 8080 + targetPort: 8080 + - name: wss + port: 8443 + targetPort: 8443 + selector: + app: ws-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: example-worker + name: example-worker + namespace: et-pyo3-math1-recovery +spec: + replicas: 1 + selector: + matchLabels: + app: example-worker + strategy: + type: Recreate + template: + metadata: + labels: + app: example-worker + spec: + containers: + - env: + - name: RUNNER_MODULE + value: "@edge-toolkit/et-ws-pyo3-math1" + - name: WS_SERVER_URL + value: ws://ws-server:8080/ws + - name: PYO3_AGENT_ID + value: example-worker + - name: RECOVERY_JOB_ID + value: example-job + - name: RECOVERY_SUPERVISOR_AGENT_ID + value: recovery-supervisor + image: et-ws-pyo3-runner:latest + imagePullPolicy: IfNotPresent + name: example-worker + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /tmp + name: runner-tmp + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - emptyDir: {} + name: runner-tmp +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: recovery-supervisor + namespace: et-pyo3-math1-recovery +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: recovery-supervisor + namespace: et-pyo3-math1-recovery +rules: + - apiGroups: + - apps + resourceNames: + - example-worker + resources: + - deployments + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: recovery-supervisor + namespace: et-pyo3-math1-recovery +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: recovery-supervisor +subjects: + - kind: ServiceAccount + name: recovery-supervisor + namespace: et-pyo3-math1-recovery +--- +apiVersion: v1 +kind: ConfigMap +data: + kubeconfig: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://kubernetes.default.svc + name: cluster + contexts: + - context: + cluster: cluster + namespace: et-pyo3-math1-recovery + user: supervisor + name: in-cluster + current-context: in-cluster + kind: Config + users: + - name: supervisor + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +metadata: + name: recovery-credentials + namespace: et-pyo3-math1-recovery +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: recovery-state + namespace: et-pyo3-math1-recovery +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: recovery-supervisor + name: recovery-supervisor + namespace: et-pyo3-math1-recovery +spec: + replicas: 1 + selector: + matchLabels: + app: recovery-supervisor + strategy: + type: Recreate + template: + metadata: + labels: + app: recovery-supervisor + spec: + automountServiceAccountToken: true + containers: + - env: + - name: KUBECONFIG + value: /app/credentials/kubeconfig + - name: RECOVERY_AGENT_ID + value: recovery-supervisor + - name: RECOVERY_COMMAND_TIMEOUT_SECONDS + value: "10" + - name: RECOVERY_CONTEXT + value: in-cluster + - name: RECOVERY_JOB_ID + value: example-job + - name: RECOVERY_MODE + value: observe-only + - name: RECOVERY_NAMESPACE + value: et-pyo3-math1-recovery + - name: RECOVERY_POLICY_DEADLINE_SECONDS + value: "240" + - name: RECOVERY_POLICY_DETECTION_GRACE_SECONDS + value: "3" + - name: RECOVERY_POLICY_MAX_RESTARTS + value: "2" + - name: RECOVERY_POLICY_RESTART_AFTER_SECONDS + value: "20" + - name: RECOVERY_POLICY_STALL_SECONDS + value: "180" + - name: RECOVERY_POLL_SECONDS + value: "2" + - name: RECOVERY_STATE_FILE + value: /app/state/recovery.json + - name: RECOVERY_WORKLOADS + value: '[{"agent_id":"example-worker","deployment":"example-worker","source":"/modules/@edge-toolkit/et-ws-pyo3-math1"}]' + - name: RECOVERY_WS_URL + value: ws://ws-server:8080/ws + - name: RUST_LOG + value: info + image: et-recovery-supervisor:latest + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - et-recovery-healthcheck + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 3 + name: recovery-supervisor + readinessProbe: + exec: + command: + - et-recovery-healthcheck + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + startupProbe: + exec: + command: + - et-recovery-healthcheck + failureThreshold: 62 + periodSeconds: 1 + timeoutSeconds: 3 + volumeMounts: + - mountPath: /app/state + name: recovery-state + - mountPath: /app/runtime + name: recovery-runtime + - mountPath: /app/credentials + name: recovery-credentials + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + serviceAccountName: recovery-supervisor + volumes: + - name: recovery-state + persistentVolumeClaim: + claimName: recovery-state + - emptyDir: {} + name: recovery-runtime + - configMap: + name: recovery-credentials + name: recovery-credentials diff --git a/verification/local/output/pyo3-math1-recovery/secrets.env b/verification/local/output/pyo3-math1-recovery/secrets.env new file mode 100644 index 00000000..5c1fa399 --- /dev/null +++ b/verification/local/output/pyo3-math1-recovery/secrets.env @@ -0,0 +1,5 @@ +# Generated from the scenario input -- regenerating this deployment rewrites it. +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +ZO_ROOT_USER_PASSWORD=Atque!et9249 +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +OTLP_AUTH_PASSWORD=Atque!et9249